diff --git a/gluster/swift/__init__.py b/gluster/swift/__init__.py index 14bb4e8..fa360b6 100644 --- a/gluster/swift/__init__.py +++ b/gluster/swift/__init__.py @@ -1,5 +1,6 @@ """ Gluster for Swift """ + class PkgInfo(object): def __init__(self, canonical_version, name, final): self.canonical_version = canonical_version diff --git a/gluster/swift/account/server.py b/gluster/swift/account/server.py index aeaabc9..3c6d616 100644 --- a/gluster/swift/account/server.py +++ b/gluster/swift/account/server.py @@ -17,7 +17,7 @@ # Simply importing this monkey patches the constraint handling to fit our # needs -import gluster.swift.common.constraints +import gluster.swift.common.constraints # noqa from swift.account import server from gluster.swift.common.DiskDir import DiskAccount diff --git a/gluster/swift/common/DiskDir.py b/gluster/swift/common/DiskDir.py index 18d08cc..5d67303 100644 --- a/gluster/swift/common/DiskDir.py +++ b/gluster/swift/common/DiskDir.py @@ -13,36 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, errno +import os -from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \ - mkdirs, validate_account, validate_container, is_marker, \ - get_container_details, get_account_details, get_container_metadata, \ - create_container_metadata, create_account_metadata, DEFAULT_GID, \ - DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \ - write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \ - X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ - X_CONTAINER_COUNT, CONTAINER, os_path +from gluster.swift.common.fs_utils import dir_empty, rmdirs, mkdirs, os_path +from gluster.swift.common.utils import clean_metadata, validate_account, \ + validate_container, get_container_details, get_account_details, \ + create_container_metadata, create_account_metadata, DEFAULT_GID, \ + DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \ + write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \ + X_PUT_TIMESTAMP, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ + X_CONTAINER_COUNT from gluster.swift.common import Glusterfs -from swift.common.constraints import CONTAINER_LISTING_LIMIT -from swift.common.utils import normalize_timestamp, TRUE_VALUES - DATADIR = 'containers' -# Create a dummy db_file in /etc/swift -_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no') -if _unittests_enabled in TRUE_VALUES: - _tmp_dir = '/tmp/gluster_unit_tests' - try: - os.mkdir(_tmp_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - _db_file = os.path.join(_tmp_dir, 'db_file.db') -else: - _db_file = '/etc/swift/db_file.db' +# Create a dummy db_file in Glusterfs.RUN_DIR +_db_file = os.path.join(Glusterfs.RUN_DIR, 'db_file.db') if not os.path.exists(_db_file): file(_db_file, 'w+') @@ -91,7 +78,7 @@ class DiskCommon(object): Accept sorted list. Objects should start with prefix. """ - filtered_objs=[] + filtered_objs = [] for object_name in objects: tmp_obj = object_name.replace(prefix, '', 1) sufix = tmp_obj.split(delimiter, 1) @@ -106,8 +93,7 @@ class DiskCommon(object): TODO: We can traverse in reverse order to optimize. Accept sorted list. """ - filtered_objs=[] - found = 0 + filtered_objs = [] if objects[-1] < marker: return filtered_objs for object_name in objects: @@ -120,7 +106,7 @@ class DiskCommon(object): """ Accept sorted list. """ - filtered_objs=[] + filtered_objs = [] for object_name in objects: if object_name < end_marker: filtered_objs.append(object_name) @@ -130,7 +116,7 @@ class DiskCommon(object): return filtered_objs def filter_limit(self, objects, limit): - filtered_objs=[] + filtered_objs = [] for i in range(0, limit): filtered_objs.append(objects[i]) @@ -232,7 +218,8 @@ class DiskDir(DiskCommon): self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp) self.metadata[X_PUT_TIMESTAMP] = timestamp bused = self.metadata[X_BYTES_USED][0] - self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp) + self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), + timestamp) #TODO: define update_metadata instad of writing whole metadata again. self.put_metadata(self.metadata) @@ -240,10 +227,12 @@ class DiskDir(DiskCommon): ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0] self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp) bused, timestamp = self.metadata[X_BYTES_USED] - self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp) + self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), + timestamp) self.put_metadata(self.metadata) - def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used): + def put_container(self, container, put_timestamp, del_timestamp, + object_count, bytes_used): """ For account server. """ @@ -363,20 +352,22 @@ class DiskDir(DiskCommon): # update the object counts in case they changed behind our back. self.update_object_count() - data = {'account' : self.account, 'container' : self.container, - 'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0], - 'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0], - 'hash': '', 'id' : '', 'created_at' : '1', - 'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0], - 'delete_timestamp' : '1', - 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1', - 'reported_object_count' : '1', 'reported_bytes_used' : '1'} + data = {'account': self.account, 'container': self.container, + 'object_count': self.metadata.get( + X_OBJECTS_COUNT, ('0', 0))[0], + 'bytes_used': self.metadata.get(X_BYTES_USED, ('0', 0))[0], + 'hash': '', 'id': '', 'created_at': '1', + 'put_timestamp': self.metadata.get( + X_PUT_TIMESTAMP, ('0', 0))[0], + 'delete_timestamp': '1', + 'reported_put_timestamp': '1', + 'reported_delete_timestamp': '1', + 'reported_object_count': '1', 'reported_bytes_used': '1'} if include_metadata: data['metadata'] = self.metadata return data - def put_object(self, name, timestamp, size, content_type, - etag, deleted=0): + def put_object(self, name, timestamp, size, content_type, etag, deleted=0): # TODO: Implement the specifics of this func. pass @@ -401,7 +392,8 @@ class DiskDir(DiskCommon): self.unlink() def update_metadata(self, metadata): - assert self.metadata, "Valid container/account metadata should have been created by now" + assert self.metadata, "Valid container/account metadata should have" \ + " been created by now" if metadata: new_metadata = self.metadata.copy() new_metadata.update(metadata) @@ -478,12 +470,13 @@ class DiskAccount(DiskDir): # update the container counts in case they changed behind our back. self.update_container_count() - data = {'account' : self.account, 'created_at' : '1', - 'put_timestamp' : '1', 'delete_timestamp' : '1', - 'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0], - 'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0], - 'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0], - 'hash' : '', 'id' : ''} + data = {'account': self.account, 'created_at': '1', + 'put_timestamp': '1', 'delete_timestamp': '1', + 'container_count': self.metadata.get( + X_CONTAINER_COUNT, (0, 0))[0], + 'object_count': self.metadata.get(X_OBJECTS_COUNT, (0, 0))[0], + 'bytes_used': self.metadata.get(X_BYTES_USED, (0, 0))[0], + 'hash': '', 'id': ''} if include_metadata: data['metadata'] = self.metadata @@ -493,4 +486,4 @@ class DiskAccount(DiskDir): cont_path = os.path.join(self.datadir, container) metadata = read_metadata(cont_path) - return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None + return int(metadata.get(X_PUT_TIMESTAMP, ('0', 0))[0]) or None diff --git a/gluster/swift/common/DiskFile.py b/gluster/swift/common/DiskFile.py index 900bd49..577147a 100644 --- a/gluster/swift/common/DiskFile.py +++ b/gluster/swift/common/DiskFile.py @@ -18,16 +18,16 @@ import errno import random from hashlib import md5 from contextlib import contextmanager -from swift.common.utils import normalize_timestamp, renamer +from swift.common.utils import renamer from swift.common.exceptions import DiskFileNotExist from gluster.swift.common.exceptions import AlreadyExistsAsDir -from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \ - create_object_metadata, do_open, do_close, do_unlink, do_chown, \ - do_listdir, read_metadata, write_metadata, os_path, do_fsync -from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \ - X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \ - X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \ - FILE_TYPE, DEFAULT_UID, DEFAULT_GID +from gluster.swift.common.fs_utils import mkdirs, rmdirs, do_open, do_close, \ + do_unlink, do_chown, os_path, do_fsync +from gluster.swift.common.utils import read_metadata, write_metadata, \ + validate_object, create_object_metadata +from gluster.swift.common.utils import X_CONTENT_LENGTH, X_CONTENT_TYPE, \ + X_TIMESTAMP, X_TYPE, X_OBJECT_TYPE, FILE, MARKER_DIR, OBJECT, DIR_TYPE, \ + FILE_TYPE, DEFAULT_UID, DEFAULT_GID import logging from swift.obj.server import DiskFile @@ -42,17 +42,18 @@ def _adjust_metadata(metadata): # Fix up the metadata to ensure it has a proper value for the # Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE # metadata values. - content_type = metadata['Content-Type'] + content_type = metadata[X_CONTENT_TYPE] if not content_type: # FIXME: How can this be that our caller supplied us with metadata # that has a content type that evaluates to False? # # FIXME: If the file exists, we would already know it is a # directory. So why are we assuming it is a file object? - metadata['Content-Type'] = FILE_TYPE + metadata[X_CONTENT_TYPE] = FILE_TYPE x_object_type = FILE else: - x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE + x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE \ + else FILE metadata[X_TYPE] = OBJECT metadata[X_OBJECT_TYPE] = x_object_type return metadata @@ -184,7 +185,8 @@ class Gluster_DiskFile(DiskFile): if tombstone: # We don't write tombstone files. So do nothing. return - assert self.data_file is not None, "put_metadata: no file to put metadata into" + assert self.data_file is not None, \ + "put_metadata: no file to put metadata into" metadata = _adjust_metadata(metadata) write_metadata(self.data_file, metadata) self.metadata = metadata @@ -192,8 +194,8 @@ class Gluster_DiskFile(DiskFile): def put(self, fd, metadata, extension='.data'): """ - Finalize writing the file on disk, and renames it from the temp file to - the real location. This should be called after the data has been + Finalize writing the file on disk, and renames it from the temp file + to the real location. This should be called after the data has been written to the temp file. :param fd: file descriptor of the temp file @@ -202,7 +204,6 @@ class Gluster_DiskFile(DiskFile): """ # Our caller will use '.data' here; we just ignore it since we map the # URL directly to the file system. - extension = '' metadata = _adjust_metadata(metadata) @@ -220,7 +221,6 @@ class Gluster_DiskFile(DiskFile): msg = 'File object exists as a directory: %s' % self.data_file raise AlreadyExistsAsDir(msg) - timestamp = normalize_timestamp(metadata[X_TIMESTAMP]) write_metadata(self.tmppath, metadata) if X_CONTENT_LENGTH in metadata: self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH])) @@ -248,7 +248,7 @@ class Gluster_DiskFile(DiskFile): :param timestamp: timestamp to compare with each file """ - if not self.metadata or self.metadata['X-Timestamp'] >= timestamp: + if not self.metadata or self.metadata[X_TIMESTAMP] >= timestamp: return assert self.data_file, \ @@ -257,7 +257,8 @@ class Gluster_DiskFile(DiskFile): if self._is_dir: # Marker directory object if not rmdirs(self.data_file): - logging.error('Unable to delete dir object: %s', self.data_file) + logging.error('Unable to delete dir object: %s', + self.data_file) return else: # File object @@ -283,7 +284,7 @@ class Gluster_DiskFile(DiskFile): file_size = 0 if self.data_file: file_size = os_path.getsize(self.data_file) - if X_CONTENT_LENGTH in self.metadata: + if X_CONTENT_LENGTH in self.metadata: metadata_size = int(self.metadata[X_CONTENT_LENGTH]) if file_size != metadata_size: self.metadata[X_CONTENT_LENGTH] = file_size @@ -314,11 +315,11 @@ class Gluster_DiskFile(DiskFile): path = self._container_path subdir_list = self._obj_path.split(os.path.sep) for i in range(len(subdir_list)): - path = os.path.join(path, subdir_list[i]); + path = os.path.join(path, subdir_list[i]) if not os_path.exists(path): self._create_dir_object(path) - tmpfile = '.' + self._obj + '.' + md5(self._obj + \ + tmpfile = '.' + self._obj + '.' + md5(self._obj + str(random.random())).hexdigest() self.tmppath = os.path.join(self.datadir, tmpfile) diff --git a/gluster/swift/common/Glusterfs.py b/gluster/swift/common/Glusterfs.py index ba6bcf1..819a747 100644 --- a/gluster/swift/common/Glusterfs.py +++ b/gluster/swift/common/Glusterfs.py @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import fcntl +import time +import errno import logging -import os, sys, fcntl, time, errno + from ConfigParser import ConfigParser, NoSectionError, NoOptionError from swift.common.utils import TRUE_VALUES, search_tree from gluster.swift.common.fs_utils import mkdirs @@ -25,26 +29,29 @@ from gluster.swift.common.fs_utils import mkdirs _fs_conf = ConfigParser() MOUNT_IP = 'localhost' OBJECT_ONLY = False -RUN_DIR='/var/run/swift' +RUN_DIR = '/var/run/swift' SWIFT_DIR = '/etc/swift' _do_getsize = False -if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')): +if _fs_conf.read(os.path.join(SWIFT_DIR, 'fs.conf')): try: - MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost') + MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', MOUNT_IP) except (NoSectionError, NoOptionError): pass try: - OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES + OBJECT_ONLY = _fs_conf.get('DEFAULT', + 'object_only', + "no") in TRUE_VALUES except (NoSectionError, NoOptionError): pass try: - RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', '/var/run/swift') + RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', RUN_DIR) except (NoSectionError, NoOptionError): pass try: - _do_getsize = _fs_conf.get('DEFAULT', 'accurate_size_in_listing', \ - "no") in TRUE_VALUES + _do_getsize = _fs_conf.get('DEFAULT', + 'accurate_size_in_listing', + "no") in TRUE_VALUES except (NoSectionError, NoOptionError): pass @@ -58,9 +65,11 @@ def _busy_wait(full_mount_path): if os.path.ismount(full_mount_path): return True time.sleep(2) - logging.error('Busy wait for mount timed out for mount %s', full_mount_path) + logging.error('Busy wait for mount timed out for mount %s', + full_mount_path) return False + def mount(root, drive): # FIXME: Possible thundering herd problem here @@ -77,15 +86,15 @@ def mount(root, drive): if not os.path.isdir(full_mount_path): mkdirs(full_mount_path) - lck_file = os.path.join(RUN_DIR, '%s.lock' %drive); + lck_file = os.path.join(RUN_DIR, '%s.lock' % drive) if not os.path.exists(RUN_DIR): mkdirs(RUN_DIR) - fd = os.open(lck_file, os.O_CREAT|os.O_RDWR) + fd = os.open(lck_file, os.O_CREAT | os.O_RDWR) with os.fdopen(fd, 'r+b') as f: try: - fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) + fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as ex: if ex.errno in (errno.EACCES, errno.EAGAIN): # This means that some other process is mounting the @@ -93,13 +102,14 @@ def mount(root, drive): return _busy_wait(full_mount_path) else: raise ex - mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \ + mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, full_mount_path) if os.system(mnt_cmd) or not _busy_wait(full_mount_path): logging.error('Mount failed %s: %s', NAME, mnt_cmd) return False return True + def unmount(full_mount_path): # FIXME: Possible thundering herd problem here @@ -107,6 +117,7 @@ def unmount(full_mount_path): if os.system(umnt_cmd): logging.error('Unable to unmount %s %s' % (full_mount_path, NAME)) + def _get_export_list(): cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP @@ -126,6 +137,7 @@ def _get_export_list(): return export_list + def get_mnt_point(vol_name, conf_dir=SWIFT_DIR, conf_file="object-server*"): """Read the object-server's configuration file and return the device value""" diff --git a/gluster/swift/common/constraints.py b/gluster/swift/common/constraints.py index 11f626b..d1c990d 100644 --- a/gluster/swift/common/constraints.py +++ b/gluster/swift/common/constraints.py @@ -23,11 +23,12 @@ from gluster.swift.common import Glusterfs, ring if hasattr(swift.common.constraints, 'constraints_conf_int'): MAX_OBJECT_NAME_COMPONENT_LENGTH = \ - swift.common.constraints.constraints_conf_int( - 'max_object_name_component_length', 255) + swift.common.constraints.constraints_conf_int( + 'max_object_name_component_length', 255) else: MAX_OBJECT_NAME_COMPONENT_LENGTH = 255 + def validate_obj_name_component(obj): if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH: return 'too long (%d)' % len(obj) @@ -38,6 +39,7 @@ def validate_obj_name_component(obj): # Save the original check object creation __check_object_creation = swift.common.constraints.check_object_creation + # Define our new one which invokes the original def gluster_check_object_creation(req, object_name): """ @@ -61,7 +63,7 @@ def gluster_check_object_creation(req, object_name): reason = validate_obj_name_component(obj) if reason: bdy = 'Invalid object name "%s", component "%s" %s' \ - % (object_name, obj, reason) + % (object_name, obj, reason) ret = HTTPBadRequest(body=bdy, request=req, content_type='text/plain') @@ -74,6 +76,7 @@ swift.common.constraints.check_object_creation = gluster_check_object_creation # Save the original check mount __check_mount = swift.common.constraints.check_mount + # Define our new one which invokes the original def gluster_check_mount(root, drive): # FIXME: Potential performance optimization here to not call the original @@ -84,6 +87,7 @@ def gluster_check_mount(root, drive): return Glusterfs.mount(root, drive) + # Replace the original check mount with ours swift.common.constraints.check_mount = gluster_check_mount diff --git a/gluster/swift/common/exceptions.py b/gluster/swift/common/exceptions.py index d9357db..75a95ec 100644 --- a/gluster/swift/common/exceptions.py +++ b/gluster/swift/common/exceptions.py @@ -13,15 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. + class GlusterfsException(Exception): pass + class FileOrDirNotFoundError(GlusterfsException): pass + class NotDirectoryError(GlusterfsException): pass + class AlreadyExistsAsDir(GlusterfsException): pass - diff --git a/gluster/swift/common/fs_utils.py b/gluster/swift/common/fs_utils.py index 0613a26..de450a5 100644 --- a/gluster/swift/common/fs_utils.py +++ b/gluster/swift/common/fs_utils.py @@ -16,14 +16,16 @@ import logging import os import errno -import os.path as os_path +import os.path as os_path # noqa from eventlet import tpool from gluster.swift.common.exceptions import FileOrDirNotFoundError, \ NotDirectoryError + def do_walk(*args, **kwargs): return os.walk(*args, **kwargs) + def do_write(fd, msg): try: cnt = os.write(fd, msg) @@ -32,6 +34,7 @@ def do_write(fd, msg): raise return cnt + def do_mkdir(path): try: os.mkdir(path) @@ -41,15 +44,18 @@ def do_mkdir(path): raise return True + def do_makedirs(path): try: os.makedirs(path) except OSError as err: if err.errno != errno.EEXIST: - logging.exception("Makedirs failed on %s err: %s", path, err.strerror) + logging.exception("Makedirs failed on %s err: %s", + path, err.strerror) raise return True + def do_listdir(path): try: buf = os.listdir(path) @@ -58,6 +64,7 @@ def do_listdir(path): raise return buf + def do_chown(path, uid, gid): try: os.chown(path, uid, gid) @@ -66,6 +73,7 @@ def do_chown(path, uid, gid): raise return True + def do_stat(path): try: #Check for fd. @@ -78,6 +86,7 @@ def do_stat(path): raise return buf + def do_open(path, mode): if isinstance(mode, int): try: @@ -93,6 +102,7 @@ def do_open(path, mode): raise return fd + def do_close(fd): #fd could be file or int type. try: @@ -105,16 +115,19 @@ def do_close(fd): raise return True -def do_unlink(path, log = True): + +def do_unlink(path, log=True): try: os.unlink(path) except OSError as err: if err.errno != errno.ENOENT: if log: - logging.exception("Unlink failed on %s err: %s", path, err.strerror) + logging.exception("Unlink failed on %s err: %s", + path, err.strerror) raise return True + def do_rmdir(path): try: os.rmdir(path) @@ -127,15 +140,17 @@ def do_rmdir(path): res = True return res + def do_rename(old_path, new_path): try: os.rename(old_path, new_path) except OSError as err: - logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ - err.strerror) + logging.exception("Rename failed on %s to %s err: %s", + old_path, new_path, err.strerror) raise return True + def mkdirs(path): """ Ensures the path is a directory or makes it if not. Errors if the path @@ -146,6 +161,7 @@ def mkdirs(path): if not os.path.isdir(path): do_makedirs(path) + def dir_empty(path): """ Return true if directory/container is empty. @@ -159,6 +175,7 @@ def dir_empty(path): raise FileOrDirNotFoundError() raise NotDirectoryError() + def rmdirs(path): if not os.path.isdir(path): return False @@ -170,6 +187,7 @@ def rmdirs(path): return False return True + def do_fsync(fd): try: tpool.execute(os.fsync, fd) diff --git a/gluster/swift/common/ring.py b/gluster/swift/common/ring.py index 06aab8d..f88af4e 100644 --- a/gluster/swift/common/ring.py +++ b/gluster/swift/common/ring.py @@ -36,13 +36,12 @@ if conf_files and _conf.read(conf_file): if not reseller_prefix.endswith('_'): reseller_prefix = reseller_prefix + '_' + class Ring(ring.Ring): def _get_part_nodes(self, part): seen_ids = set() - nodes = [dev for dev in self._devs \ - if dev['device'] == self.acc_name \ - and not (dev['id'] in seen_ids \ - or seen_ids.add(dev['id']))] + nodes = [dev for dev in self._devs if dev['device'] == self.acc_name + and not (dev['id'] in seen_ids or seen_ids.add(dev['id']))] if not nodes: nodes = [self.false_node] return nodes @@ -86,8 +85,8 @@ class Ring(ring.Ring): hardware description ====== =============================================================== """ - self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 0, \ - 'meta': '', 'device': 'volume_not_in_ring', \ + self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', + 'id': 0, 'meta': '', 'device': 'volume_not_in_ring', 'port': 6012} if account.startswith(reseller_prefix): self.acc_name = account.replace(reseller_prefix, '', 1) @@ -97,7 +96,6 @@ class Ring(ring.Ring): part = 0 return part, self._get_part_nodes(part) - def get_more_nodes(self, part): """ Generator to get extra nodes for a partition for hinted handoff. diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py index f2cd8de..2cbb7f3 100644 --- a/gluster/swift/common/utils.py +++ b/gluster/swift/common/utils.py @@ -21,9 +21,9 @@ import random from hashlib import md5 from eventlet import sleep import cPickle as pickle -from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common.utils import normalize_timestamp, TRUE_VALUES -from gluster.swift.common.fs_utils import * +from swift.common.utils import normalize_timestamp +from gluster.swift.common.fs_utils import do_rename, do_fsync, os_path, \ + do_stat, do_listdir, do_walk from gluster.swift.common import Glusterfs X_CONTENT_TYPE = 'Content-Type' @@ -54,8 +54,11 @@ DEFAULT_GID = -1 PICKLE_PROTOCOL = 2 CHUNK_SIZE = 65536 MEMCACHE_KEY_PREFIX = 'gluster.swift.' -MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.' -MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.' +MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + \ + 'account.details.' +MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + \ + 'container.details.' + def read_metadata(path): """ @@ -70,7 +73,8 @@ def read_metadata(path): key = 0 while metadata is None: try: - metadata_s += xattr.getxattr(path, '%s%s' % (METADATA_KEY, (key or ''))) + metadata_s += xattr.getxattr(path, + '%s%s' % (METADATA_KEY, (key or ''))) except IOError as err: if err.errno == errno.ENODATA: if key > 0: @@ -108,6 +112,7 @@ def read_metadata(path): key += 1 return metadata + def write_metadata(path, metadata): """ Helper function to write pickled metadata for a File/Directory. @@ -120,13 +125,17 @@ def write_metadata(path, metadata): key = 0 while metastr: try: - xattr.setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE]) + xattr.setxattr(path, + '%s%s' % (METADATA_KEY, key or ''), + metastr[:MAX_XATTR_SIZE]) except IOError as err: - logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err)) + logging.exception("setxattr failed on %s key %s err: %s", + path, key, str(err)) raise metastr = metastr[MAX_XATTR_SIZE:] key += 1 + def clean_metadata(path): key = 0 while True: @@ -138,21 +147,25 @@ def clean_metadata(path): raise key += 1 + def check_user_xattr(path): if not os_path.exists(path): return False try: xattr.setxattr(path, 'user.test.key1', 'value1') except IOError as err: - logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err)) + logging.exception("check_user_xattr: set failed on %s err: %s", + path, str(err)) raise try: xattr.removexattr(path, 'user.test.key1') except IOError as err: - logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err)) + logging.exception("check_user_xattr: remove failed on %s err: %s", + path, str(err)) #Remove xattr may fail in case of concurrent remove. return True + def validate_container(metadata): if not metadata: logging.warn('validate_container: No metadata') @@ -163,16 +176,17 @@ def validate_container(metadata): X_PUT_TIMESTAMP not in metadata.keys() or \ X_OBJECTS_COUNT not in metadata.keys() or \ X_BYTES_USED not in metadata.keys(): - #logging.warn('validate_container: Metadata missing entries: %s' % metadata) return False (value, timestamp) = metadata[X_TYPE] if value == CONTAINER: return True - logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,)) + logging.warn('validate_container: metadata type is not CONTAINER (%r)', + value) return False + def validate_account(metadata): if not metadata: logging.warn('validate_account: No metadata') @@ -184,16 +198,17 @@ def validate_account(metadata): X_OBJECTS_COUNT not in metadata.keys() or \ X_BYTES_USED not in metadata.keys() or \ X_CONTAINER_COUNT not in metadata.keys(): - #logging.warn('validate_account: Metadata missing entries: %s' % metadata) return False (value, timestamp) = metadata[X_TYPE] if value == ACCOUNT: return True - logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,)) + logging.warn('validate_account: metadata type is not ACCOUNT (%r)', + value) return False + def validate_object(metadata): if not metadata: logging.warn('validate_object: No metadata') @@ -205,22 +220,24 @@ def validate_object(metadata): X_CONTENT_LENGTH not in metadata.keys() or \ X_TYPE not in metadata.keys() or \ X_OBJECT_TYPE not in metadata.keys(): - #logging.warn('validate_object: Metadata missing entries: %s' % metadata) return False if metadata[X_TYPE] == OBJECT: return True - logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],)) + logging.warn('validate_object: metadata type is not OBJECT (%r)', + metadata[X_TYPE]) return False + def is_marker(metadata): if not metadata: logging.warn('is_marker: No metadata') return False if X_OBJECT_TYPE not in metadata.keys(): - logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata) + logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s', + metadata) return False if metadata[X_OBJECT_TYPE] == MARKER_DIR: @@ -228,6 +245,7 @@ def is_marker(metadata): else: return False + def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, bytes_used=0, obj_list=[]): # strip the prefix off, also stripping the leading and trailing slashes @@ -247,6 +265,7 @@ def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, return object_count, bytes_used + def update_list(path, cont_path, dirs=[], files=[], object_count=0, bytes_used=0, obj_list=[]): if files: @@ -279,15 +298,16 @@ def _get_container_details_from_fs(cont_path): if os_path.isdir(cont_path): for (path, dirs, files) in do_walk(cont_path): - object_count, bytes_used = update_list(path, cont_path, dirs, files, - object_count, bytes_used, - obj_list) + object_count, bytes_used = update_list(path, cont_path, dirs, + files, object_count, + bytes_used, obj_list) dir_list.append((path, do_stat(path).st_mtime)) sleep() return ContainerDetails(bytes_used, object_count, obj_list, dir_list) + def get_container_details(cont_path, memcache=None): """ Return object_list, object_count and bytes_used. @@ -344,6 +364,7 @@ def _get_account_details_from_fs(acc_path, acc_stats): return AccountDetails(acc_stats.st_mtime, container_count, container_list) + def get_account_details(acc_path, memcache=None): """ Return container_list and container_count. @@ -369,6 +390,7 @@ def get_account_details(acc_path, memcache=None): memcache.set(mkey, ad) return ad.container_list, ad.container_count + def _get_etag(path): etag = md5() with open(path, 'rb') as fp: @@ -380,6 +402,7 @@ def _get_etag(path): break return etag.hexdigest() + def get_object_metadata(obj_path): """ Return metadata of object. @@ -398,10 +421,10 @@ def get_object_metadata(obj_path): X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, X_OBJECT_TYPE: DIR if is_dir else FILE, X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, - X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path), - } + X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path)} return metadata + def _add_timestamp(metadata_i): # At this point we have a simple key/value dictionary, turn it into # key/(value,timestamp) pairs. @@ -414,30 +437,38 @@ def _add_timestamp(metadata_i): metadata[key] = value_i return metadata + def get_container_metadata(cont_path, memcache=None): objects = [] object_count = 0 bytes_used = 0 - objects, object_count, bytes_used = get_container_details(cont_path, memcache) + objects, object_count, bytes_used = get_container_details(cont_path, + memcache) metadata = {X_TYPE: CONTAINER, - X_TIMESTAMP: normalize_timestamp(os_path.getctime(cont_path)), - X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(cont_path)), + X_TIMESTAMP: normalize_timestamp( + os_path.getctime(cont_path)), + X_PUT_TIMESTAMP: normalize_timestamp( + os_path.getmtime(cont_path)), X_OBJECTS_COUNT: object_count, X_BYTES_USED: bytes_used} return _add_timestamp(metadata) + def get_account_metadata(acc_path, memcache=None): containers = [] container_count = 0 containers, container_count = get_account_details(acc_path, memcache) metadata = {X_TYPE: ACCOUNT, - X_TIMESTAMP: normalize_timestamp(os_path.getctime(acc_path)), - X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(acc_path)), + X_TIMESTAMP: normalize_timestamp( + os_path.getctime(acc_path)), + X_PUT_TIMESTAMP: normalize_timestamp( + os_path.getmtime(acc_path)), X_OBJECTS_COUNT: 0, X_BYTES_USED: 0, X_CONTAINER_COUNT: container_count} return _add_timestamp(metadata) + def restore_metadata(path, metadata): meta_orig = read_metadata(path) if meta_orig: @@ -449,18 +480,22 @@ def restore_metadata(path, metadata): write_metadata(path, meta_new) return meta_new + def create_object_metadata(obj_path): metadata = get_object_metadata(obj_path) return restore_metadata(obj_path, metadata) + def create_container_metadata(cont_path, memcache=None): metadata = get_container_metadata(cont_path, memcache) return restore_metadata(cont_path, metadata) + def create_account_metadata(acc_path, memcache=None): metadata = get_account_metadata(acc_path, memcache) return restore_metadata(acc_path, metadata) + def write_pickle(obj, dest, tmp=None, pickle_protocol=0): """ Ensure that a pickle file gets written to disk. The file is first written @@ -479,7 +514,8 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0): """ dirname = os.path.dirname(dest) basename = os.path.basename(dest) - tmpname = '.' + basename + '.' + md5(basename + str(random.random())).hexdigest() + tmpname = '.' + basename + '.' + \ + md5(basename + str(random.random())).hexdigest() tmppath = os.path.join(dirname, tmpname) with open(tmppath, 'wb') as fo: pickle.dump(obj, fo, pickle_protocol) @@ -491,6 +527,7 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0): do_fsync(fo) do_rename(tmppath, dest) + # Over-ride Swift's utils.write_pickle with ours import swift.common.utils swift.common.utils.write_pickle = write_pickle diff --git a/gluster/swift/container/server.py b/gluster/swift/container/server.py index c5792aa..ee284c9 100644 --- a/gluster/swift/container/server.py +++ b/gluster/swift/container/server.py @@ -17,7 +17,7 @@ # Simply importing this monkey patches the constraint handling to fit our # needs -import gluster.swift.common.constraints +import gluster.swift.common.constraints # noqa from swift.container import server from gluster.swift.common.DiskDir import DiskDir diff --git a/gluster/swift/obj/server.py b/gluster/swift/obj/server.py index 1c2b6cb..1223036 100644 --- a/gluster/swift/obj/server.py +++ b/gluster/swift/obj/server.py @@ -17,8 +17,8 @@ # Simply importing this monkey patches the constraint handling to fit our # needs -import gluster.swift.common.constraints -import gluster.swift.common.utils +import gluster.swift.common.constraints # noqa +import gluster.swift.common.utils # noqa from swift.obj import server from gluster.swift.common.DiskFile import Gluster_DiskFile diff --git a/gluster/swift/proxy/server.py b/gluster/swift/proxy/server.py index 792a97d..df9b245 100644 --- a/gluster/swift/proxy/server.py +++ b/gluster/swift/proxy/server.py @@ -16,10 +16,11 @@ # Simply importing this monkey patches the constraint handling to fit our # needs -import gluster.swift.common.constraints +import gluster.swift.common.constraints # noqa from swift.proxy import server + def app_factory(global_conf, **local_conf): """paste.deploy app factory for creating WSGI proxy apps.""" conf = global_conf.copy() diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py new file mode 100644 index 0000000..1058367 --- /dev/null +++ b/test/unit/common/test_diskdir.py @@ -0,0 +1,575 @@ +# Copyright (c) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for gluster.swift.common.DiskDir """ + +import os +import errno +import tempfile +import cPickle as pickle +import unittest +import shutil +import tarfile +from nose import SkipTest +from swift.common.utils import normalize_timestamp +from gluster.swift.common import utils +import gluster.swift.common.Glusterfs +from test_utils import _initxattr, _destroyxattr, _setxattr, _getxattr +from test.unit import FakeLogger + +gluster.swift.common.Glusterfs.RUN_DIR = '/tmp/gluster_unit_tests/run' +try: + os.makedirs(gluster.swift.common.Glusterfs.RUN_DIR) +except OSError as e: + if e.errno != errno.EEXIST: + raise + +import gluster.swift.common.DiskDir as dd + + +def timestamp_in_range(ts, base): + low = normalize_timestamp(base - 5) + high = normalize_timestamp(base + 5) + assert low <= ts, "timestamp %s is less than %s" % (ts, low) + assert high >= ts, "timestamp %s is greater than %s" % (ts, high) + + +class TestDiskDirModuleFunctions(unittest.TestCase): + """ Tests for gluster.swift.common.DiskDir module functions """ + + def setUp(self): + raise SkipTest + + def test__read_metadata(self): + def fake_read_metadata(p): + return { 'a': 1, 'b': ('c', 5) } + orig_rm = dd.read_metadata + dd.read_metadata = fake_read_metadata + try: + md = dd._read_metadata("/tmp/foo") + finally: + dd.read_metadata = orig_rm + assert md['a'] == (1, 0) + assert md['b'] == ('c', 5) + + def test_filter_end_marker(self): + in_objs, end_marker = [], '' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == [] + + in_objs, end_marker = [], 'abc' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == [] + + in_objs, end_marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], '' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == [] + + in_objs, end_marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'ABC' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == [] + + in_objs, end_marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'efg' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == ['abc_123', 'abc_456', 'abc_789', 'def_101'] + + # Input not sorted, so we should only expect one name + in_objs, end_marker = ['abc_123', 'def_101', 'abc_456', 'abc_789'], 'abc_789' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == ['abc_123',] + + in_objs, end_marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'abc_789' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == ['abc_123', 'abc_456'] + + in_objs, end_marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'abc_5' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == ['abc_123', 'abc_456'] + + in_objs, end_marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'abc_123' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == [] + + in_objs, end_marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'def_101' + out_objs = dd.filter_end_marker(in_objs, end_marker) + assert list(out_objs) == ['abc_123', 'abc_456', 'abc_789'] + + def test_filter_marker(self): + in_objs, marker = [], '' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == [] + + in_objs, marker = [], 'abc' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == [] + + in_objs, marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], '' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == in_objs + + in_objs, marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'ABC' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == in_objs + + in_objs, marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'efg' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == [] + + # Input not sorted, so we should expect the names as listed + in_objs, marker = ['abc_123', 'def_101', 'abc_456', 'abc_789'], 'abc_456' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == ['def_101', 'abc_789'] + + in_objs, marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'abc_456' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == ['abc_789', 'def_101'] + + in_objs, marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'abc_5' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == ['abc_789', 'def_101'] + + in_objs, marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'abc_123' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == ['abc_456', 'abc_789', 'def_101'] + + in_objs, marker = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'def_101' + out_objs = dd.filter_marker(in_objs, marker) + assert list(out_objs) == [] + + def test_filter_prefix(self): + in_objs, prefix = [], '' + out_objs = dd.filter_prefix(in_objs, prefix) + assert list(out_objs) == [] + + in_objs, prefix = [], 'abc' + out_objs = dd.filter_prefix(in_objs, prefix) + assert list(out_objs) == [] + + in_objs, prefix = ['abc_123', 'abc_456', 'abc_789', 'def_101'], '' + out_objs = dd.filter_prefix(in_objs, prefix) + assert list(out_objs) == in_objs + + in_objs, prefix = ['abc_123', 'abc_456', 'abc_789', 'def_101'], 'abc' + out_objs = dd.filter_prefix(in_objs, prefix) + assert list(out_objs) == ['abc_123', 'abc_456', 'abc_789'] + + in_objs, prefix = ['abc_123', 'def_101', 'abc_456', 'abc_789'], 'abc' + out_objs = dd.filter_prefix(in_objs, prefix) + assert list(out_objs) == ['abc_123',] + + def test_filter_delimiter(self): + in_objs, delimiter, prefix = [], None, '' + try: + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + except AssertionError: + pass + except Exception: + raise SkipTest + self.fail("Failed to raise assertion") + + in_objs, delimiter, prefix = [], '', '' + try: + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + except AssertionError: + pass + except Exception: + self.fail("Failed to raise assertion") + + in_objs, delimiter, prefix = [], str(255), '' + try: + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + except AssertionError: + pass + except Exception: + self.fail("Failed to raise assertion") + + in_objs, delimiter, prefix = [], '_', '' + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + assert list(out_objs) == [] + + in_objs, delimiter, prefix = ['abc_'], '_', '' + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + assert list(out_objs) == in_objs + + in_objs, delimiter, prefix = ['abc_123', 'abc_456'], '_', '' + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + assert list(out_objs) == ['abc_'] + + in_objs, delimiter, prefix = ['abc_123', 'abc_456', 'def_123', 'def_456'], '_', '' + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + assert list(out_objs) == ['abc_', 'def_'] + + in_objs, delimiter, prefix = ['abc_123', 'abc_456', 'abc_789', 'def_101'], '_', 'abc_' + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + l = list(out_objs) + assert l == ['abc_123', 'abc_456', 'abc_789'], repr(l) + + in_objs, delimiter, prefix = ['abc_123_a', 'abc_456', 'abc_789_', 'def_101'], '_', 'abc_' + out_objs = dd.filter_delimiter(in_objs, delimiter, prefix) + assert list(out_objs) == ['abc_123_a', 'abc_789_'] + + def test_filter_limit(self): + try: + l = list(dd.filter_limit([], 0)) + except AssertionError: + pass + else: + self.fail("Accepted a zero limit") + + l = list(dd.filter_limit([], 1)) + assert l == [] + l = list(dd.filter_limit([1,], 1)) + assert l == [1,] + l = list(dd.filter_limit([1,], 10)) + assert l == [1,] + l = list(dd.filter_limit([1,2,3], 1)) + assert l == [1,] + l = list(dd.filter_limit([1,2,3], 2)) + assert l == [1,2] + l = list(dd.filter_limit([1,2,3], 3)) + assert l == [1,2,3] + l = list(dd.filter_limit([1,2,3], 4)) + assert l == [1,2,3] + +class TestDiskCommon(unittest.TestCase): + """ Tests for gluster.swift.common.DiskDir.DiskCommon """ + + def setUp(self): + raise SkipTest + _initxattr() + self.fake_logger = FakeLogger() + self.td = tempfile.mkdtemp() + self.fake_drives = [] + self.fake_accounts = [] + for i in range(0,3): + self.fake_drives.append("drv%d" % i) + os.makedirs(os.path.join(self.td, self.fake_drives[i])) + self.fake_accounts.append(self.fake_drives[i]) + + def tearDown(self): + _destroyxattr() + shutil.rmtree(self.td) + + def test_constructor(self): + dc = dd.DiskCommon(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + assert dc.metadata == {} + assert dc.db_file == dd._db_file + assert dc.pending_timeout == 0 + assert dc.stale_reads_ok == False + assert dc.root == self.td + assert dc.logger == self.fake_logger + assert dc.account == self.fake_accounts[0] + assert dc.datadir == os.path.join(self.td, self.fake_drives[0]) + assert dc._dir_exists is None + + def test__dir_exists_read_metadata_exists(self): + datadir = os.path.join(self.td, self.fake_drives[0]) + fake_md = { "fake": (True,0) } + fake_md_p = pickle.dumps(fake_md, utils.PICKLE_PROTOCOL) + _setxattr(datadir, utils.METADATA_KEY, fake_md_p) + dc = dd.DiskCommon(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + dc._dir_exists_read_metadata() + assert dc.metadata == fake_md, repr(dc.metadata) + assert dc.db_file == dd._db_file + assert dc.pending_timeout == 0 + assert dc.stale_reads_ok == False + assert dc.root == self.td + assert dc.logger == self.fake_logger + assert dc.account == self.fake_accounts[0] + assert dc.datadir == datadir + assert dc._dir_exists is True + + def test__dir_exists_read_metadata_does_not_exist(self): + dc = dd.DiskCommon(self.td, "dne0", "dne0", self.fake_logger) + dc._dir_exists_read_metadata() + assert dc.metadata == {} + assert dc.db_file == dd._db_file + assert dc.pending_timeout == 0 + assert dc.stale_reads_ok == False + assert dc.root == self.td + assert dc.logger == self.fake_logger + assert dc.account == "dne0" + assert dc.datadir == os.path.join(self.td, "dne0") + assert dc._dir_exists is False + + def test_initialize(self): + dc = dd.DiskCommon(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + dc.initialize('12345') + assert dc.metadata == {} + assert dc.db_file == dd._db_file + assert dc.pending_timeout == 0 + assert dc.stale_reads_ok == False + assert dc.root == self.td + assert dc.logger == self.fake_logger + assert dc.account == self.fake_accounts[0] + assert dc.datadir == os.path.join(self.td, self.fake_drives[0]) + assert dc._dir_exists is None + + def test_is_deleted(self): + dc = dd.DiskCommon(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + assert dc.is_deleted() == False + + def test_update_metadata(self): + dc = dd.DiskCommon(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + utils.create_container_metadata(dc.datadir) + dc.metadata = dd._read_metadata(dc.datadir) + md_copy = dc.metadata.copy() + + def _mock_write_metadata(path, md): + self.fail("write_metadata should not have been called") + + orig_wm = dd.write_metadata + dd.write_metadata = _mock_write_metadata + try: + dc.update_metadata({}) + assert dc.metadata == md_copy + dc.update_metadata(md_copy) + assert dc.metadata == md_copy + finally: + dd.write_metadata = orig_wm + + dc.update_metadata({'X-Container-Meta-foo': '42'}) + assert 'X-Container-Meta-foo' in dc.metadata + assert dc.metadata['X-Container-Meta-foo'] == '42' + md = pickle.loads(_getxattr(dc.datadir, utils.METADATA_KEY)) + assert dc.metadata == md, "%r != %r" % (dc.metadata, md) + del dc.metadata['X-Container-Meta-foo'] + assert dc.metadata == md_copy + + +class TestDiskDir(unittest.TestCase): + """ Tests for gluster.swift.common.DiskDir.DiskDir """ + + def setUp(self): + _initxattr() + self.fake_logger = FakeLogger() + self.td = tempfile.mkdtemp() + self.fake_drives = [] + self.fake_accounts = [] + for i in range(0,3): + self.fake_drives.append("drv%d" % i) + os.makedirs(os.path.join(self.td, self.fake_drives[i])) + self.fake_accounts.append(self.fake_drives[i]) + + def tearDown(self): + _destroyxattr() + shutil.rmtree(self.td) + + def test_constructor(self): + raise SkipTest + self.fail("Implement me") + + def test_empty(self): + raise SkipTest + self.fail("Implement me") + + def test_list_objects_iter(self): + raise SkipTest + self.fail("Implement me") + + def test_get_info(self): + raise SkipTest + self.fail("Implement me") + + def test_delete_db(self): + raise SkipTest + self.fail("Implement me") + + +class TestDiskAccount(unittest.TestCase): + """ Tests for gluster.swift.common.DiskDir.DiskAccount """ + + def setUp(self): + _initxattr() + self.fake_logger = FakeLogger() + self.td = tempfile.mkdtemp() + self.fake_drives = [] + self.fake_accounts = [] + self.fake_md = [] + for i in range(0,3): + self.fake_drives.append("drv%d" % i) + os.makedirs(os.path.join(self.td, self.fake_drives[i])) + self.fake_accounts.append(self.fake_drives[i]) + if i == 0: + # First drive does not have any initial account metadata + continue + if i == 1: + # Second drive has account metadata but it is not valid + datadir = os.path.join(self.td, self.fake_drives[i]) + fake_md = { "fake-drv-%d" % i: (True,0) } + self.fake_md.append(fake_md) + fake_md_p = pickle.dumps(fake_md, utils.PICKLE_PROTOCOL) + _setxattr(datadir, utils.METADATA_KEY, fake_md_p) + if i == 2: + # Third drive has valid account metadata + utils.create_account_metadata(datadir) + + def tearDown(self): + _destroyxattr() + shutil.rmtree(self.td) + + def test_constructor_no_metadata(self): + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + raise SkipTest + assert da._container_info is None + assert da._dir_exists is True + ctime = os.path.getctime(da.datadir) + mtime = os.path.getmtime(da.datadir) + exp_md = { + 'X-Bytes-Used': (0, 0), + 'X-Timestamp': (normalize_timestamp(ctime), 0), + 'X-Object-Count': (0, 0), + 'X-Type': ('Account', 0), + 'X-PUT-Timestamp': (normalize_timestamp(mtime), 0), + 'X-Container-Count': (0, 0)} + assert da.metadata == exp_md, repr(da.metadata) + + def test_constructor_metadata_not_valid(self): + da = dd.DiskAccount(self.td, self.fake_drives[1], + self.fake_accounts[1], self.fake_logger) + raise SkipTest + assert da._container_info is None + assert da._dir_exists is True + ctime = os.path.getctime(da.datadir) + mtime = os.path.getmtime(da.datadir) + exp_md = { + 'X-Bytes-Used': (0, 0), + 'X-Timestamp': (normalize_timestamp(ctime), 0), + 'X-Object-Count': (0, 0), + 'X-Type': ('Account', 0), + 'X-PUT-Timestamp': (normalize_timestamp(mtime), 0), + 'X-Container-Count': (0, 0), + 'fake-drv-1': (True, 0)} + assert da.metadata == exp_md, repr(da.metadata) + + def test_constructor_metadata_valid(self): + da = dd.DiskAccount(self.td, self.fake_drives[2], + self.fake_accounts[2], self.fake_logger) + raise SkipTest + assert da._container_info is None + assert da._dir_exists is True + ctime = os.path.getctime(da.datadir) + mtime = os.path.getmtime(da.datadir) + exp_md = { + 'X-Bytes-Used': (0, 0), + 'X-Timestamp': (normalize_timestamp(ctime), 0), + 'X-Object-Count': (0, 0), + 'X-Type': ('Account', 0), + 'X-PUT-Timestamp': (normalize_timestamp(mtime), 0), + 'X-Container-Count': (0, 0)} + assert da.metadata == exp_md, repr(da.metadata) + + def test_list_containers_iter(self): + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + raise SkipTest + self.fail("Implement me") + + get_info_keys = set(['account', 'created_at', 'put_timestamp', + 'delete_timestamp', 'container_count', + 'object_count', 'bytes_used', 'hash', 'id']) + + def test_get_info_empty(self): + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + data = da.get_info() + assert set(data.keys()) == self.get_info_keys + assert data['account'] == self.fake_accounts[0] + assert data['created_at'] == '1' + assert data['put_timestamp'] == '1' + assert data['delete_timestamp'] == '1' + assert data['container_count'] == 0 + assert data['object_count'] == 0 + assert data['bytes_used'] == 0 + assert data['hash'] == '' + assert data['id'] == '' + + def test_get_info(self): + tf = tarfile.open("common/data/account_tree.tar.bz2", "r:bz2") + orig_cwd = os.getcwd() + os.chdir(os.path.join(self.td, self.fake_drives[0])) + try: + tf.extractall() + finally: + os.chdir(orig_cwd) + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + data = da.get_info() + assert set(data.keys()) == self.get_info_keys + assert data['account'] == self.fake_accounts[0] + assert data['created_at'] == '1' + assert data['put_timestamp'] == '1' + assert data['delete_timestamp'] == '1' + assert data['container_count'] == 3 + assert data['object_count'] == 0 + assert data['bytes_used'] == 0 + assert data['hash'] == '' + assert data['id'] == '' + + def test_get_container_timestamp(self): + tf = tarfile.open("common/data/account_tree.tar.bz2", "r:bz2") + orig_cwd = os.getcwd() + datadir = os.path.join(self.td, self.fake_drives[0]) + os.chdir(datadir) + try: + tf.extractall() + finally: + os.chdir(orig_cwd) + md = dd.create_container_metadata(os.path.join(datadir, 'c2')) + assert 'X-PUT-Timestamp' in md, repr(md) + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + raise SkipTest + cts = da.get_container_timestamp('c2') + assert md['X-PUT-Timestamp'][0] == cts, repr(cts) + + def test_update_put_timestamp_not_updated(self): + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + raise SkipTest + da.update_put_timestamp('12345') + assert da.metadata['X-PUT-Timestamp'][0] != '12345', repr(da.metadata) + + def test_update_put_timestamp_updated(self): + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + exp_pts = str(float(da.metadata['X-PUT-Timestamp'][0]) + 100) + da.update_put_timestamp(exp_pts) + raise SkipTest + assert da.metadata['X-PUT-Timestamp'][0] == exp_pts, repr(da.metadata) + + def test_delete_db(self): + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + raise SkipTest + assert da._dir_exists == True + da.delete_db('12345') + assert da._dir_exists == True + + def test_put_container(self): + raise SkipTest + self.fail("Implement me") + + def test_is_status_deleted(self): + da = dd.DiskAccount(self.td, self.fake_drives[0], + self.fake_accounts[0], self.fake_logger) + raise SkipTest + assert da.is_status_deleted() == False diff --git a/test/unit/common/test_diskfile.py b/test/unit/common/test_diskfile.py index 85d539a..5af661e 100644 --- a/test/unit/common/test_diskfile.py +++ b/test/unit/common/test_diskfile.py @@ -57,9 +57,6 @@ class MockException(Exception): def _mock_rmdirs(p): raise MockException("gluster.swift.common.DiskFile.rmdirs() called") -def _mock_do_listdir(p): - raise MockException("gluster.swift.common.DiskFile.do_listdir() called") - def _mock_do_unlink(f): ose = OSError() ose.errno = errno.ENOENT @@ -575,16 +572,13 @@ class TestDiskFile(unittest.TestCase): "z", self.lg) assert gdf.metadata == {} _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs - _saved_do_listdir = gluster.swift.common.DiskFile.do_listdir gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs - gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir try: gdf.unlinkold(None) except MockException as exp: self.fail(str(exp)) finally: gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs - gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir def test_unlinkold_same_timestamp(self): assert not os.path.exists("/tmp/foo") @@ -593,16 +587,13 @@ class TestDiskFile(unittest.TestCase): assert gdf.metadata == {} gdf.metadata['X-Timestamp'] = 1 _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs - _saved_do_listdir = gluster.swift.common.DiskFile.do_listdir gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs - gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir try: gdf.unlinkold(1) except MockException as exp: self.fail(str(exp)) finally: gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs - gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir def test_unlinkold_file(self): td = tempfile.mkdtemp() diff --git a/tools/test-requires b/tools/test-requires index ef8bf14..e2941a0 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -1,3 +1,8 @@ +# Install bounded pep8/pyflakes first, then let flake8 install +pep8==1.4.5 +pyflakes==0.7.2 +flake8==2.0 + coverage nose nosexcover diff --git a/tox.ini b/tox.ini index 6ac4f9d..3e01066 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27 +envlist = py26,py27,pep8 [testenv] setenv = VIRTUAL_ENV={envdir} @@ -18,8 +18,19 @@ commands = nosetests -v --exe --with-xunit --with-coverage --cover-package glust [tox:jenkins] downloadcache = ~/cache/pip +[testenv:pep8] +changedir = {toxinidir} +commands = + flake8 gluster test + [testenv:cover] setenv = NOSE_WITH_COVERAGE=1 [testenv:venv] commands = {posargs} + +[flake8] +ignore = H +builtins = _ +exclude = .venv,.tox,dist,doc,test,*egg +show-source = True