openstack/swift
Code Issues Proposed changes

merged with trunk

Browse Source
This commit is contained in:
John Dickinson 2010-10-26 09:17:57 -05:00
commit 95b189264c
14 changed files with 198 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README
View File

@ -10,7 +10,7 @@ To build documentation run `python setup.py build_sphinx`, and then browse to
The best place to get started is the "SAIO - Swift All In One", which will walk
you through setting up a development cluster of Swift in a VM.
For more information, vist us at http://launchpad.net/swift, or come hang out
For more information, visit us at http://launchpad.net/swift, or come hang out
on our IRC channel, #openstack on freenode.
--

1
bin/swift-bench
View File

@ -39,6 +39,7 @@ CONF_DEFAULTS = {
'num_gets': '10000',
'delete': 'yes',
'container_name': uuid.uuid4().hex,
'num_containers': '20',
'use_proxy': 'yes',
'url': '',
'account': '',

8
doc/source/deployment_guide.rst
View File

@ -230,8 +230,6 @@ log_name object-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ============== ==========================================
------------------------------
@ -319,8 +317,6 @@ log_name container-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ================= =======================================
----------------------------
@ -388,10 +384,6 @@ log_name account-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
max_container_count 100 Maximum containers randomly picked for
a given account audit
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
==================== =============== =======================================
[account-reaper]

108
doc/source/development_saio.rst
View File

@ -2,9 +2,9 @@
SAIO - Swift All In One
=======================
------------------------------------
Instructions for setting up a dev VM
------------------------------------
---------------------------------------------
Instructions for setting up a development VM
---------------------------------------------
This documents setting up a virtual machine for doing Swift development. The
virtual machine will emulate running a four node Swift cluster.
@ -15,9 +15,11 @@ virtual machine will emulate running a four node Swift cluster.
- Ubuntu Live/Install: http://cdimage.ubuntu.com/releases/10.04/release/ubuntu-10.04-dvd-amd64.iso (4.1 GB)
- Ubuntu Mirrors: https://launchpad.net/ubuntu/+cdmirrors
* Create guest virtual machine from the Ubuntu image (if you are going to use
a separate partition for swift data, be sure to add another device when
creating the VM)
* Create guest virtual machine from the Ubuntu image.
-----------------------------------------
Installing dependencies and the core code
-----------------------------------------
* As root on guest (you'll have to log in as you, then `sudo su -`):
#. `apt-get install python-software-properties`
@ -28,21 +30,21 @@ virtual machine will emulate running a four node Swift cluster.
python-xattr sqlite3 xfsprogs python-webob python-eventlet
python-greenlet python-pastedeploy`
#. Install anything else you want, like screen, ssh, vim, etc.
#. If you would like to use another partition for storage:
#. Next, choose either see :ref:`partition-section` or :ref:`loopback-section`.
#. `fdisk /dev/sdb` (set up a single partition)
#. `mkfs.xfs -i size=1024 /dev/sdb1`
#. Edit `/etc/fstab` and add
.. _partition-section:
Using a partition for storage
=============================
If you are going to use a separate partition for Swift data, be sure to add another device when
creating the VM, and follow these instructions.
#. `fdisk /dev/sdb` (set up a single partition)
#. `mkfs.xfs -i size=1024 /dev/sdb1`
#. Edit `/etc/fstab` and add
`/dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0`
#. If you would like to use a loopback device instead of another partition:
#. `dd if=/dev/zero of=/srv/swift-disk bs=1024 count=0 seek=1000000`
(modify seek to make a larger or smaller partition)
#. `mkfs.xfs -i size=1024 /srv/swift-disk`
#. Edit `/etc/fstab` and add
`/srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0`
#. `mkdir /mnt/sdb1`
#. `mount /mnt/sdb1`
#. `mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 /mnt/sdb1/test`
@ -56,6 +58,36 @@ virtual machine will emulate running a four node Swift cluster.
mkdir /var/run/swift
chown <your-user-name>:<your-group-name> /var/run/swift
.. _loopback-section:
Using a loopback device for storage
===================================
If you want to use a loopback device instead of another partition, follow these instructions.
#. `dd if=/dev/zero of=/srv/swift-disk bs=1024 count=0 seek=1000000`
(modify seek to make a larger or smaller partition)
#. `mkfs.xfs -i size=1024 /srv/swift-disk`
#. Edit `/etc/fstab` and add
`/srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0`
#. `mkdir /mnt/sdb1`
#. `mount /mnt/sdb1`
#. `mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 /mnt/sdb1/test`
#. `chown <your-user-name>:<your-group-name> /mnt/sdb1/*`
#. `mkdir /srv`
#. `for x in {1..4}; do ln -s /mnt/sdb1/$x /srv/$x; done`
#. `mkdir -p /etc/swift/object-server /etc/swift/container-server /etc/swift/account-server /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 /var/run/swift`
#. `chown -R <your-user-name>:<your-group-name> /etc/swift /srv/[1-4]/ /var/run/swift` -- **Make sure to include the trailing slash after /srv/[1-4]/**
#. Add to `/etc/rc.local` (before the `exit 0`)::
mkdir /var/run/swift
chown <your-user-name>:<your-group-name> /var/run/swift
----------------
Setting up rsync
----------------
#. Create /etc/rsyncd.conf::
uid = <Your user name>
@ -144,7 +176,14 @@ virtual machine will emulate running a four node Swift cluster.
#. `service rsync restart`
* As you on guest:
------------------------------------------------
Getting the code and setting up test environment
------------------------------------------------
Sample configuration files are provided with all defaults in line-by-line comments.
Do these commands as you on guest:
#. `mkdir ~/bin`
#. Create `~/.bazaar/bazaar.conf`::
@ -164,6 +203,13 @@ virtual machine will emulate running a four node Swift cluster.
export PATH=${PATH}:~/bin
#. `. ~/.bashrc`
---------------------
Configuring each node
---------------------
Sample configuration files are provided with all defaults in line-by-line comments.
#. Create `/etc/swift/auth-server.conf`::
[DEFAULT]
@ -331,7 +377,6 @@ virtual machine will emulate running a four node Swift cluster.
[container-auditor]
#. Create `/etc/swift/container-server/3.conf`::
[DEFAULT]
@ -353,7 +398,6 @@ virtual machine will emulate running a four node Swift cluster.
[container-auditor]
#. Create `/etc/swift/container-server/4.conf`::
[DEFAULT]
@ -460,7 +504,11 @@ virtual machine will emulate running a four node Swift cluster.
[object-auditor]
#. Create `~/bin/resetswift`::
------------------------------------
Setting up scripts for running Swift
------------------------------------
#. Create `~/bin/resetswift.` If you are using a loopback device substitute `/dev/sdb1` with `/srv/swift-disk`::
#!/bin/bash
@ -476,11 +524,6 @@ virtual machine will emulate running a four node Swift cluster.
sudo service rsyslog restart
sudo service memcached restart
.. note::
If you are using a loopback device, substitute `/dev/sdb1` above with
`/srv/swift-disk`
#. Create `~/bin/remakerings`::
#!/bin/bash
@ -553,8 +596,13 @@ virtual machine will emulate running a four node Swift cluster.
If you plan to work on documentation (and who doesn't?!):
#. `sudo apt-get install python-sphinx`
#. `python setup.py build_sphinx`
On Ubuntu:
#. `sudo apt-get install python-sphinx` installs Sphinx.
#. `python setup.py build_sphinx` builds the documentation.
On MacOS:
#. `sudo easy_install -U sphinx` installs Sphinx.
#. `python setup.py build_sphinx` builds the documentation.
----------------
Debugging Issues

3
doc/source/ratelimit.rst
View File

@ -27,6 +27,9 @@ clock_accuracy 1000 Represents how accurate the proxy servers'
max_sleep_time_seconds 60 App will immediately return a 498 response
if the necessary sleep time ever exceeds
the given max_sleep_time_seconds.
log_sleep_time_seconds 0 To allow visibility into rate limiting set
this value > 0 and all sleeps greater than
the number will be logged.
account_ratelimit 0 If set, will limit all requests to
/account_name and PUTs to
/account_name/container_name. Number is in

3
etc/proxy-server.conf-sample
View File

@ -60,7 +60,8 @@ use = egg:swift#ratelimit
# clock accuracy.
# clock_accuracy = 1000
# max_sleep_time_seconds = 60
# log_sleep_time_seconds of 0 means disabled
# log_sleep_time_seconds = 0
# account_ratelimit of 0 means disabled
# account_ratelimit = 0

6
swift/auth/server.py
View File

@ -197,7 +197,11 @@ YOU HAVE A FEW OPTIONS:
(?, ?, '.super_admin', '.single_use', '.reseller_admin')''',
(token, time()))
conn.commit()
conn = http_connect(parsed.hostname, parsed.port, 'PUT', parsed.path,
if parsed.port is None:
port = {'http': 80, 'https': 443}.get(parsed.scheme, 80)
else:
port = parsed.port
conn = http_connect(parsed.hostname, port, 'PUT', parsed.path,
{'X-Auth-Token': token}, ssl=(parsed.scheme == 'https'))
resp = conn.getresponse()
resp.read()

27
swift/common/bench.py
View File

@ -58,7 +58,8 @@ class Bench(object):
self.account = conf.account
self.url = conf.url
self.ip, self.port = self.url.split('/')[2].split(':')
self.container_name = conf.container_name
self.containers = ['%s_%d' % (conf.container_name, i)
for i in xrange(int(conf.num_containers))]
self.object_size = int(conf.object_size)
self.object_sources = conf.object_sources
@ -151,16 +152,16 @@ class BenchDELETE(Bench):
if time.time() - self.heartbeat >= 15:
self.heartbeat = time.time()
self._log_status('DEL')
device, partition, name = self.names.pop()
device, partition, name, container_name = self.names.pop()
with self.connection() as conn:
try:
if self.use_proxy:
client.delete_object(self.url, self.token,
self.container_name, name, http_conn=conn)
container_name, name, http_conn=conn)
else:
node = {'ip': self.ip, 'port': self.port, 'device': device}
direct_client.direct_delete_object(node, partition,
self.account, self.container_name, name)
self.account, container_name, name)
except client.ClientException, e:
self.logger.debug(str(e))
self.failures += 1
@ -179,16 +180,16 @@ class BenchGET(Bench):
if time.time() - self.heartbeat >= 15:
self.heartbeat = time.time()
self._log_status('GETS')
device, partition, name = random.choice(self.names)
device, partition, name, container_name = random.choice(self.names)
with self.connection() as conn:
try:
if self.use_proxy:
client.get_object(self.url, self.token,
self.container_name, name, http_conn=conn)
container_name, name, http_conn=conn)
else:
node = {'ip': self.ip, 'port': self.port, 'device': device}
direct_client.direct_get_object(node, partition,
self.account, self.container_name, name)
self.account, container_name, name)
except client.ClientException, e:
self.logger.debug(str(e))
self.failures += 1
@ -204,8 +205,9 @@ class BenchPUT(Bench):
self.msg = 'PUTS'
if self.use_proxy:
with self.connection() as conn:
client.put_container(self.url, self.token,
self.container_name, http_conn=conn)
for container_name in self.containers:
client.put_container(self.url, self.token,
container_name, http_conn=conn)
def _run(self, thread):
if time.time() - self.heartbeat >= 15:
@ -218,19 +220,20 @@ class BenchPUT(Bench):
source = '0' * self.object_size
device = random.choice(self.devices)
partition = str(random.randint(1, 3000))
container_name = random.choice(self.containers)
with self.connection() as conn:
try:
if self.use_proxy:
client.put_object(self.url, self.token,
self.container_name, name, source,
container_name, name, source,
content_length=len(source), http_conn=conn)
else:
node = {'ip': self.ip, 'port': self.port, 'device': device}
direct_client.direct_put_object(node, partition,
self.account, self.container_name, name, source,
self.account, container_name, name, source,
content_length=len(source))
except client.ClientException, e:
self.logger.debug(str(e))
self.failures += 1
self.names.append((device, partition, name))
self.names.append((device, partition, name, container_name))
self.complete += 1

46
swift/common/memcached.py
View File

@ -168,28 +168,60 @@ class MemcacheRing(object):
def incr(self, key, delta=1, timeout=0):
"""
Increments a key which has a numeric value by delta.
If the key can't be found, it's added as delta.
If the key can't be found, it's added as delta or 0 if delta < 0.
If passed a negative number, will use memcached's decr. Returns
the int stored in memcached
Note: The data memcached stores as the result of incr/decr is
an unsigned int. decr's that result in a number below 0 are
stored as 0.
:param key: key
:param delta: amount to add to the value of key (or set as the value
if the key is not found)
if the key is not found) will be cast to an int
:param timeout: ttl in memcache
"""
key = md5hash(key)
command = 'incr'
if delta < 0:
command = 'decr'
delta = str(abs(int(delta)))
for (server, fp, sock) in self._get_conns(key):
try:
sock.sendall('incr %s %s\r\n' % (key, delta))
sock.sendall('%s %s %s\r\n' % (command, key, delta))
line = fp.readline().strip().split()
if line[0].upper() == 'NOT_FOUND':
line[0] = str(delta)
sock.sendall('add %s %d %d %s noreply\r\n%s\r\n' % \
(key, 0, timeout, len(line[0]), line[0]))
ret = int(line[0].strip())
add_val = delta
if command == 'decr':
add_val = '0'
sock.sendall('add %s %d %d %s\r\n%s\r\n' % \
(key, 0, timeout, len(add_val), add_val))
line = fp.readline().strip().split()
if line[0].upper() == 'NOT_STORED':
sock.sendall('%s %s %s\r\n' % (command, key, delta))
line = fp.readline().strip().split()
ret = int(line[0].strip())
else:
ret = int(add_val)
else:
ret = int(line[0].strip())
self._return_conn(server, fp, sock)
return ret
except Exception, e:
self._exception_occurred(server, e)
def decr(self, key, delta=1, timeout=0):
"""
Decrements a key which has a numeric value by delta. Calls incr with
-delta.
:param key: key
:param delta: amount to subtract to the value of key (or set the
value to 0 if the key is not found) will be cast to
an int
:param timeout: ttl in memcache
"""
self.incr(key, delta=-delta, timeout=timeout)
def delete(self, key):
"""
Deletes a key/value pair from memcache.

9
swift/common/middleware/ratelimit.py
View File

@ -40,6 +40,8 @@ class RateLimitMiddleware(object):
self.account_ratelimit = float(conf.get('account_ratelimit', 0))
self.max_sleep_time_seconds = float(conf.get('max_sleep_time_seconds',
60))
self.log_sleep_time_seconds = float(conf.get('log_sleep_time_seconds',
0))
self.clock_accuracy = int(conf.get('clock_accuracy', 1000))
self.ratelimit_whitelist = [acc.strip() for acc in
conf.get('account_whitelist', '').split(',')
@ -148,7 +150,7 @@ class RateLimitMiddleware(object):
max_sleep_m = self.max_sleep_time_seconds * self.clock_accuracy
if max_sleep_m - need_to_sleep_m <= self.clock_accuracy * 0.01:
# treat as no-op decrement time
self.memcache_client.incr(key, delta=-time_per_request_m)
self.memcache_client.decr(key, delta=time_per_request_m)
raise MaxSleepTimeHit("Max Sleep Time Exceeded: %s" %
need_to_sleep_m)
@ -176,6 +178,11 @@ class RateLimitMiddleware(object):
obj_name=obj_name):
try:
need_to_sleep = self._get_sleep_time(key, max_rate)
if self.log_sleep_time_seconds and \
need_to_sleep > self.log_sleep_time_seconds:
self.logger.info("Ratelimit sleep log: %s for %s/%s/%s" % (
need_to_sleep, account_name,
container_name, obj_name))
if need_to_sleep > 0:
eventlet.sleep(need_to_sleep)
except MaxSleepTimeHit, e:

6
swift/common/utils.py
View File

@ -629,10 +629,10 @@ def audit_location_generator(devices, datadir, mount_check=True, logger=None):
:param devices: parent directory of the devices to be audited
:param datadir: a directory located under self.devices. This should be
one of the DATADIR constants defined in the account, container, and
object servers.
one of the DATADIR constants defined in the account,
container, and object servers.
:param mount_check: Flag to check if a mount check should be performed
on devices
on devices
:param logger: a logger object
'''
for device in os.listdir(devices):

3
swift/obj/auditor.py
View File

@ -107,6 +107,9 @@ class ObjectAuditor(Daemon):
partition, account,
container, obj,
keep_data_fp=True)
if df.data_file is None:
# file is deleted, we found the tombstone
return
if os.path.getsize(df.data_file) != \
int(df.metadata['Content-Length']):
raise AuditException('Content-Length of %s does not match '

16
test/unit/common/middleware/test_ratelimit.py
View File

@ -36,9 +36,14 @@ class FakeMemcache(object):
return True
def incr(self, key, delta=1, timeout=0):
self.store[key] = int(self.store.setdefault(key, 0)) + delta
self.store[key] = int(self.store.setdefault(key, 0)) + int(delta)
if self.store[key] < 0:
self.store[key] = 0
return int(self.store[key])
def decr(self, key, delta=1, timeout=0):
return self.incr(key, delta=-delta, timeout=timeout)
@contextmanager
def soft_lock(self, key, timeout=0, retries=5):
yield True
@ -88,9 +93,12 @@ class FakeApp(object):
class FakeLogger(object):
# a thread safe logger
def error(self, msg):
# a thread safe logger
pass
def info(self, msg):
pass
@ -279,7 +287,7 @@ class TestRateLimit(unittest.TestCase):
the_498s = [t for t in all_results if t.startswith('Slow down')]
self.assertEquals(len(the_498s), 2)
time_took = time.time() - begin
self.assert_(1.5 <= round(time_took,1) < 1.7, time_took)
self.assert_(1.5 <= round(time_took, 1) < 1.7, time_took)
def test_ratelimit_max_rate_multiple_acc(self):
num_calls = 4
@ -316,7 +324,7 @@ class TestRateLimit(unittest.TestCase):
thread.join()
time_took = time.time() - begin
# the all 15 threads still take 1.5 secs
self.assert_(1.5 <= round(time_took,1) < 1.7)
self.assert_(1.5 <= round(time_took, 1) < 1.7)
def test_ratelimit_acc_vrs_container(self):
conf_dict = {'clock_accuracy': 1000,

28
test/unit/common/test_memcached.py
View File

@ -98,6 +98,17 @@ class MockMemcached(object):
self.outbuf += str(val[2]) + '\r\n'
else:
self.outbuf += 'NOT_FOUND\r\n'
elif parts[0].lower() == 'decr':
if parts[1] in self.cache:
val = list(self.cache[parts[1]])
if int(val[2]) - int(parts[2]) > 0:
val[2] = str(int(val[2]) - int(parts[2]))
else:
val[2] = '0'
self.cache[parts[1]] = val
self.outbuf += str(val[2]) + '\r\n'
else:
self.outbuf += 'NOT_FOUND\r\n'
def readline(self):
if self.down:
raise Exception('mock is down')
@ -151,6 +162,23 @@ class TestMemcached(unittest.TestCase):
self.assertEquals(memcache_client.get('some_key'), '10')
memcache_client.incr('some_key', delta=1)
self.assertEquals(memcache_client.get('some_key'), '11')
memcache_client.incr('some_key', delta=-5)
self.assertEquals(memcache_client.get('some_key'), '6')
memcache_client.incr('some_key', delta=-15)
self.assertEquals(memcache_client.get('some_key'), '0')
def test_decr(self):
memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'])
mock = MockMemcached()
memcache_client._client_cache['1.2.3.4:11211'] = [(mock, mock)] * 2
memcache_client.decr('some_key', delta=5)
self.assertEquals(memcache_client.get('some_key'), '0')
memcache_client.incr('some_key', delta=15)
self.assertEquals(memcache_client.get('some_key'), '15')
memcache_client.decr('some_key', delta=4)
self.assertEquals(memcache_client.get('some_key'), '11')
memcache_client.decr('some_key', delta=15)
self.assertEquals(memcache_client.get('some_key'), '0')
def test_retry(self):
logging.getLogger().addHandler(NullLoggingHandler())