openstack/swift
Code Issues Proposed changes

Container Sync: Simple HTTP Proxy load balancing

Browse Source 
Change-Id: I021b043b927153bacff48cae648d4d8c5bbad765
This commit is contained in:
gholt 2014-01-16 01:58:49 +00:00
parent f538006bbf
commit 69d331d0d6
3 changed files with 36 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
etc/container-server.conf-sample
View File

@ -153,7 +153,9 @@ use = egg:swift#recon
# log_address = /dev/log
#
# If you need to use an HTTP Proxy, set it here; defaults to no proxy.
# sync_proxy = http://127.0.0.1:8888
# You can also set this to a comma separated list of HTTP Proxies and they will
# be randomly used (simple load balancing).
# sync_proxy = http://10.1.1.1:8888,http://10.1.1.2:8888
#
# Will sync each container at most once per interval
# interval = 300

14
swift/container/sync.py
View File

@ -17,7 +17,7 @@ import os
import uuid
from swift import gettext_ as _
from time import ctime, time
from random import random, shuffle
from random import choice, random, shuffle
from struct import unpack_from
from eventlet import sleep, Timeout
@ -133,7 +133,10 @@ class ContainerSync(Daemon):
h.strip()
for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
if h.strip()]
self.proxy = conf.get('sync_proxy')
self.http_proxies = [
a.strip()
for a in conf.get('sync_proxy', '').split(',')
if a.strip()]
#: Number of containers with sync turned on that were successfully
#: synced.
self.container_syncs = 0
@ -352,7 +355,7 @@ class ContainerSync(Daemon):
else:
headers['x-container-sync-key'] = user_key
delete_object(sync_to, name=row['name'], headers=headers,
proxy=self.proxy)
proxy=self.select_http_proxy())
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
@ -415,7 +418,7 @@ class ContainerSync(Daemon):
headers['x-container-sync-key'] = user_key
put_object(sync_to, name=row['name'], headers=headers,
contents=FileLikeIter(body),
proxy=self.proxy)
proxy=self.select_http_proxy())
self.container_puts += 1
self.logger.increment('puts')
self.logger.timing_since('puts.timing', start_time)
@ -448,3 +451,6 @@ class ContainerSync(Daemon):
self.logger.increment('failures')
return False
return True
def select_http_proxy(self):
return choice(self.http_proxies) if self.http_proxies else None

25
test/unit/container/test_sync.py
View File

@ -658,7 +658,7 @@ class TestContainerSync(unittest.TestCase):
sync.delete_object = fake_delete_object
cs = sync.ContainerSync({}, container_ring=FakeRing(),
object_ring=FakeRing())
cs.proxy = 'http://proxy'
cs.http_proxies = ['http://proxy']
# Success
self.assertTrue(cs.container_sync_row(
{'deleted': True,
@ -765,7 +765,7 @@ class TestContainerSync(unittest.TestCase):
cs = sync.ContainerSync({}, container_ring=FakeRing(),
object_ring=FakeRing())
cs.proxy = 'http://proxy'
cs.http_proxies = ['http://proxy']
def fake_direct_get_object(node, part, account, container, obj,
resp_chunk_size=1):
@ -912,6 +912,27 @@ class TestContainerSync(unittest.TestCase):
sync.put_object = orig_put_object
sync.direct_get_object = orig_direct_get_object
def test_select_http_proxy_None(self):
cs = sync.ContainerSync(
{'sync_proxy': ''}, container_ring=FakeRing(),
object_ring=FakeRing())
self.assertEqual(cs.select_http_proxy(), None)
def test_select_http_proxy_one(self):
cs = sync.ContainerSync(
{'sync_proxy': 'http://one'}, container_ring=FakeRing(),
object_ring=FakeRing())
self.assertEqual(cs.select_http_proxy(), 'http://one')
def test_select_http_proxy_multiple(self):
cs = sync.ContainerSync(
{'sync_proxy': 'http://one,http://two,http://three'},
container_ring=FakeRing(),
object_ring=FakeRing())
self.assertEqual(
set(cs.http_proxies),
set(['http://one', 'http://two', 'http://three']))
if __name__ == '__main__':
unittest.main()