diff --git a/bin/swift-account-audit b/bin/swift-account-audit index 73441c1ba9..bbc2d021e6 100755 --- a/bin/swift-account-audit +++ b/bin/swift-account-audit @@ -55,9 +55,9 @@ class Auditor(object): def __init__(self, swift_dir='/etc/swift', concurrency=50, deep=False, error_file=None): self.pool = GreenPool(concurrency) - self.object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz')) - self.container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz')) - self.account_ring = Ring(os.path.join(swift_dir, 'account.ring.gz')) + self.object_ring = Ring(os.path.join(swift_dir, ring_name='object')) + self.container_ring = Ring(os.path.join(swift_dir, ring_name='container')) + self.account_ring = Ring(os.path.join(swift_dir, ring_name='account')) self.deep = deep self.error_file = error_file # zero out stats diff --git a/bin/swift-dispersion-populate b/bin/swift-dispersion-populate index c5df32f741..b93561710d 100755 --- a/bin/swift-dispersion-populate +++ b/bin/swift-dispersion-populate @@ -104,7 +104,7 @@ if __name__ == '__main__': retries=retries, preauthurl=url, preauthtoken=token) - container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz')) + container_ring = Ring(swift_dir, ring_name='container') parts_left = dict((x, x) for x in xrange(container_ring.partition_count)) item_type = 'containers' created = 0 @@ -130,7 +130,7 @@ if __name__ == '__main__': container = 'dispersion_objects' put_container(connpool, container, None) - object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz')) + object_ring = Ring(swift_dir, ring_name='object')) parts_left = dict((x, x) for x in xrange(object_ring.partition_count)) item_type = 'objects' created = 0 diff --git a/bin/swift-dispersion-report b/bin/swift-dispersion-report index 90458097bc..3c533d7f75 100755 --- a/bin/swift-dispersion-report +++ b/bin/swift-dispersion-report @@ -289,8 +289,8 @@ Usage: %prog [options] [conf_file] retries=retries, preauthurl=url, preauthtoken=token) - container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz')) - object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz')) + container_ring = Ring(swift_dir, ring_name='container') + object_ring = Ring(swift_dir, ring_name='object')) container_result = container_dispersion_report(coropool, connpool, account, container_ring, diff --git a/bin/swift-object-info b/bin/swift-object-info index f1b559be87..98991f87f0 100755 --- a/bin/swift-object-info +++ b/bin/swift-object-info @@ -28,7 +28,7 @@ if __name__ == '__main__': print "Usage: %s OBJECT_FILE" % sys.argv[0] sys.exit(1) try: - ring = Ring('/etc/swift/object.ring.gz') + ring = Ring('/etc/swift/', ring_name='object') except Exception: ring = None datafile = sys.argv[1] diff --git a/bin/swift-recon b/bin/swift-recon index f94654f951..0553dbfe36 100755 --- a/bin/swift-recon +++ b/bin/swift-recon @@ -88,15 +88,16 @@ class SwiftRecon(object): self.pool_size = 30 self.pool = eventlet.GreenPool(self.pool_size) - def get_devices(self, zone_filter, ring_file): + def get_devices(self, zone_filter, swift_dir, ring_name): """ Get a list of hosts in the ring :param zone_filter: Only list zones matching given filter - :param ring_file: Ring file to obtain hosts from + :param swift_dir: Directory of swift config, usually /etc/swift + :param ring_name: Name of the ring, such as 'object' :returns: a set of tuples containing the ip and port of hosts """ - ring_data = Ring(ring_file) + ring_data = Ring(swift_dir, ring_name=ring_name) if zone_filter: ips = set((n['ip'], n['port']) for n in ring_data.devs if n \ if n['zone'] == zone_filter) @@ -430,9 +431,9 @@ class SwiftRecon(object): self.timeout = options.timeout if options.zone: - hosts = self.get_devices(options.zone, obj_ring) + hosts = self.get_devices(options.zone, swift_dir, 'object') else: - hosts = self.get_devices(None, obj_ring) + hosts = self.get_devices(None, swift_dir, 'object') if options.all: self.async_check(hosts) diff --git a/swift/account/reaper.py b/swift/account/reaper.py index d55001be8a..5e080a0cc6 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -58,10 +58,7 @@ class AccountReaper(Daemon): self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') self.interval = int(conf.get('interval', 3600)) - swift_dir = conf.get('swift_dir', '/etc/swift') - self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') - self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') - self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') + self.swift_dir = conf.get('swift_dir', '/etc/swift') self.account_ring = None self.container_ring = None self.object_ring = None @@ -76,25 +73,19 @@ class AccountReaper(Daemon): def get_account_ring(self): """ The account :class:`swift.common.ring.Ring` for the cluster. """ if not self.account_ring: - self.logger.debug( - _('Loading account ring from %s'), self.account_ring_path) - self.account_ring = Ring(self.account_ring_path) + self.account_ring = Ring(self.swift_dir, ring_name='account') return self.account_ring def get_container_ring(self): """ The container :class:`swift.common.ring.Ring` for the cluster. """ if not self.container_ring: - self.logger.debug( - _('Loading container ring from %s'), self.container_ring_path) - self.container_ring = Ring(self.container_ring_path) + self.container_ring = Ring(self.swift_dir, ring_name='container') return self.container_ring def get_object_ring(self): """ The object :class:`swift.common.ring.Ring` for the cluster. """ if not self.object_ring: - self.logger.debug( - _('Loading object ring from %s'), self.object_ring_path) - self.object_ring = Ring(self.object_ring_path) + self.object_ring = Ring(self.swift_dir, ring_name='object') return self.object_ring def run_forever(self, *args, **kwargs): diff --git a/swift/account/replicator.py b/swift/account/replicator.py index aa131c9c8d..d44c5a91d4 100644 --- a/swift/account/replicator.py +++ b/swift/account/replicator.py @@ -19,7 +19,6 @@ from swift.common import db, db_replicator class AccountReplicator(db_replicator.Replicator): server_type = 'account' - ring_file = 'account.ring.gz' brokerclass = db.AccountBroker datadir = account_server.DATADIR default_port = 6002 diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index b27ac43380..eb665e3c74 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -110,7 +110,7 @@ class Replicator(Daemon): concurrency = int(conf.get('concurrency', 8)) self.cpool = GreenPool(size=concurrency) swift_dir = conf.get('swift_dir', '/etc/swift') - self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file)) + self.ring = ring.Ring(swift_dir, ring_name=self.server_type) self.per_diff = int(conf.get('per_diff', 1000)) self.max_diffs = int(conf.get('max_diffs') or 100) self.interval = int(conf.get('interval') or diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 5f875875c7..2c2cfa1c36 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -18,6 +18,8 @@ from gzip import GzipFile from os.path import getmtime from struct import unpack_from from time import time +import os + from swift.common.utils import hash_path, validate_configuration @@ -43,10 +45,14 @@ class Ring(object): :param reload_time: time interval in seconds to check for a ring change """ - def __init__(self, pickle_gz_path, reload_time=15): + def __init__(self, pickle_gz_path, reload_time=15, ring_name=None): # can't use the ring unless HASH_PATH_SUFFIX is set validate_configuration() - self.pickle_gz_path = pickle_gz_path + if ring_name: + self.pickle_gz_path = os.path.join(pickle_gz_path, + ring_name + '.ring.gz') + else: + self.pickle_gz_path = os.path.join(pickle_gz_path) self.reload_time = reload_time self._reload(force=True) diff --git a/swift/container/replicator.py b/swift/container/replicator.py index 4cdc03b884..6b75fc13dc 100644 --- a/swift/container/replicator.py +++ b/swift/container/replicator.py @@ -19,7 +19,6 @@ from swift.common import db, db_replicator class ContainerReplicator(db_replicator.Replicator): server_type = 'container' - ring_file = 'container.ring.gz' brokerclass = db.ContainerBroker datadir = container_server.DATADIR default_port = 6001 diff --git a/swift/container/sync.py b/swift/container/sync.py index 1e99051afd..e52d80b5fb 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -172,11 +172,10 @@ class ContainerSync(Daemon): self.reported = time() swift_dir = conf.get('swift_dir', '/etc/swift') #: swift.common.ring.Ring for locating containers. - self.container_ring = container_ring or \ - Ring(os.path.join(swift_dir, 'container.ring.gz')) + self.container_ring = container_ring or Ring(swift_dir, + ring_name='container') #: swift.common.ring.Ring for locating objects. - self.object_ring = object_ring or \ - Ring(os.path.join(swift_dir, 'object.ring.gz')) + self.object_ring = object_ring or Ring(swift_dir, ring_name='object') self._myips = whataremyips() self._myport = int(conf.get('bind_port', 6001)) diff --git a/swift/container/updater.py b/swift/container/updater.py index aaf722178b..801b582843 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -41,9 +41,8 @@ class ContainerUpdater(Daemon): self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - swift_dir = conf.get('swift_dir', '/etc/swift') + self.swift_dir = conf.get('swift_dir', '/etc/swift') self.interval = int(conf.get('interval', 300)) - self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.account_ring = None self.concurrency = int(conf.get('concurrency', 4)) self.slowdown = float(conf.get('slowdown', 0.01)) @@ -60,9 +59,7 @@ class ContainerUpdater(Daemon): def get_account_ring(self): """Get the account ring. Load it if it hasn't been yet.""" if not self.account_ring: - self.logger.debug( - _('Loading account ring from %s'), self.account_ring_path) - self.account_ring = Ring(self.account_ring_path) + self.account_ring = Ring(self.swift_dir, ring_name='account') return self.account_ring def get_paths(self): diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index f70115c8e7..fd938452e4 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -236,7 +236,7 @@ class ObjectReplicator(Daemon): self.port = int(conf.get('bind_port', 6000)) self.concurrency = int(conf.get('concurrency', 1)) self.stats_interval = int(conf.get('stats_interval', '300')) - self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz')) + self.object_ring = Ring(self.swift_dir, ring_name='object') self.ring_check_interval = int(conf.get('ring_check_interval', 15)) self.next_check = time.time() + self.ring_check_interval self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) diff --git a/swift/obj/updater.py b/swift/obj/updater.py index f1086bc6c7..9d9f6253ea 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -39,9 +39,8 @@ class ObjectUpdater(Daemon): self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - swift_dir = conf.get('swift_dir', '/etc/swift') + self.swift_dir = conf.get('swift_dir', '/etc/swift') self.interval = int(conf.get('interval', 300)) - self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring = None self.concurrency = int(conf.get('concurrency', 1)) self.slowdown = float(conf.get('slowdown', 0.01)) @@ -53,9 +52,7 @@ class ObjectUpdater(Daemon): def get_container_ring(self): """Get the container ring. Load it, if it hasn't been yet.""" if not self.container_ring: - self.logger.debug( - _('Loading container ring from %s'), self.container_ring_path) - self.container_ring = Ring(self.container_ring_path) + self.container_ring = Ring(self.swift_dir, ring_name='container') return self.container_ring def run_forever(self, *args, **kwargs): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 05f207c872..97b8ce1abd 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1718,12 +1718,11 @@ class BaseApplication(object): conf.get('object_post_as_copy', 'true').lower() in TRUE_VALUES self.resellers_conf = ConfigParser() self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) - self.object_ring = object_ring or \ - Ring(os.path.join(swift_dir, 'object.ring.gz')) - self.container_ring = container_ring or \ - Ring(os.path.join(swift_dir, 'container.ring.gz')) - self.account_ring = account_ring or \ - Ring(os.path.join(swift_dir, 'account.ring.gz')) + self.object_ring = object_ring or Ring(swift_dir, ring_name='object') + self.container_ring = container_ring or Ring(swift_dir, + ring_name='container') + self.account_ring = account_ring or Ring(swift_dir, + ring_name='account') self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index 1d459ad919..f71e4b19b1 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -48,7 +48,7 @@ class TestRing(unittest.TestCase): self.testdir = os.path.join(os.path.dirname(__file__), 'ring') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) - self.testgz = os.path.join(self.testdir, 'ring.gz') + self.testgz = os.path.join(self.testdir, 'whatever.ring.gz') self.intended_replica2part2dev_id = [[0, 2, 0, 2], [2, 0, 2, 0]] self.intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0}, None, {'id': 2, 'zone': 2, 'weight': 1.0}] @@ -57,8 +57,8 @@ class TestRing(unittest.TestCase): pickle.dump(ring.RingData(self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift), GzipFile(self.testgz, 'wb')) - self.ring = \ - ring.Ring(self.testgz, reload_time=self.intended_reload_time) + self.ring = ring.Ring(self.testdir, + reload_time=self.intended_reload_time, ring_name='whatever') def tearDown(self): rmtree(self.testdir, ignore_errors=1) @@ -74,7 +74,7 @@ class TestRing(unittest.TestCase): _orig_hash_path_suffix = utils.HASH_PATH_SUFFIX try: utils.HASH_PATH_SUFFIX = '' - self.assertRaises(SystemExit, ring.Ring, self.testgz) + self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever') finally: utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix @@ -85,7 +85,8 @@ class TestRing(unittest.TestCase): def test_reload(self): os.utime(self.testgz, (time() - 300, time() - 300)) - self.ring = ring.Ring(self.testgz, reload_time=0.001) + self.ring = ring.Ring(self.testdir, reload_time=0.001, + ring_name='whatever') orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 3) self.intended_devs.append({'id': 3, 'zone': 3, 'weight': 1.0}) @@ -98,7 +99,8 @@ class TestRing(unittest.TestCase): self.assertNotEquals(self.ring._mtime, orig_mtime) os.utime(self.testgz, (time() - 300, time() - 300)) - self.ring = ring.Ring(self.testgz, reload_time=0.001) + self.ring = ring.Ring(self.testdir, reload_time=0.001, + ring_name='whatever') orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 4) self.intended_devs.append({'id': 4, 'zone': 4, 'weight': 1.0}) @@ -111,8 +113,8 @@ class TestRing(unittest.TestCase): self.assertNotEquals(self.ring._mtime, orig_mtime) os.utime(self.testgz, (time() - 300, time() - 300)) - self.ring = \ - ring.Ring(self.testgz, reload_time=0.001) + self.ring = ring.Ring(self.testdir, reload_time=0.001, + ring_name='whatever') orig_mtime = self.ring._mtime part, nodes = self.ring.get_nodes('a') self.assertEquals(len(self.ring.devs), 5) diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index 30db570868..d7b1e42672 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -36,7 +36,7 @@ def lock_parent_directory(filename): class FakeRing: class Ring: devs = [] - def __init__(self, path): + def __init__(self, path, reload_time=15, ring_name=None): pass def get_part_nodes(self, part): return [] diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index bd54ec8905..2a41c6cb82 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -126,7 +126,7 @@ def _create_test_ring(path): pickle.dump(ring.RingData(intended_replica2part2dev_id, intended_devs, intended_part_shift), GzipFile(testgz, 'wb')) - return ring.Ring(testgz, reload_time=intended_reload_time) + return ring.Ring(path, ring_name='object', reload_time=intended_reload_time) class TestObjectReplicator(unittest.TestCase):