Browse Source

Merging master into feature/ec branch

Change-Id: Ifed60e050629c674a34ede78feacffd7e6ae243e
changes/66/111866/1
John Dickinson 8 years ago
parent
commit
9ace546b81
  1. 2
      doc/source/deployment_guide.rst
  2. 9
      doc/source/logs.rst
  3. 6
      doc/source/overview_auth.rst
  4. 2
      doc/source/overview_ring.rst
  5. 43
      etc/object-expirer.conf-sample
  6. 2
      etc/proxy-server.conf-sample
  7. 15
      swift/account/backend.py
  8. 2
      swift/cli/recon.py
  9. 9
      swift/common/internal_client.py
  10. 28
      swift/common/middleware/formpost.py
  11. 124
      swift/common/middleware/list_endpoints.py
  12. 4
      swift/common/middleware/proxy_logging.py
  13. 10
      swift/common/middleware/tempurl.py
  14. 15
      swift/common/request_helpers.py
  15. 1
      swift/common/ring/utils.py
  16. 44
      swift/common/utils.py
  17. 10
      swift/common/wsgi.py
  18. 16
      swift/container/backend.py
  19. 4
      swift/obj/diskfile.py
  20. 2
      swift/obj/expirer.py
  21. 2
      swift/obj/mem_server.py
  22. 2
      swift/obj/replicator.py
  23. 26
      swift/obj/server.py
  24. 2
      swift/obj/updater.py
  25. 14
      swift/proxy/controllers/container.py
  26. 13
      swift/proxy/controllers/obj.py
  27. 5
      swift/proxy/server.py
  28. 9
      test/functional/__init__.py
  29. 206
      test/probe/brain.py
  30. 180
      test/probe/test_container_merge_policy_index.py
  31. 186
      test/probe/test_object_metadata_replication.py
  32. 11
      test/unit/__init__.py
  33. 150
      test/unit/common/middleware/test_formpost.py
  34. 191
      test/unit/common/middleware/test_list_endpoints.py
  35. 17
      test/unit/common/middleware/test_tempurl.py
  36. 2
      test/unit/common/ring/test_utils.py
  37. 1
      test/unit/common/test_internal_client.py
  38. 15
      test/unit/common/test_request_helpers.py
  39. 71
      test/unit/common/test_utils.py
  40. 57
      test/unit/common/test_wsgi.py
  41. 19
      test/unit/obj/test_diskfile.py
  42. 5
      test/unit/obj/test_expirer.py
  43. 2
      test/unit/obj/test_replicator.py
  44. 301
      test/unit/obj/test_server.py
  45. 98
      test/unit/proxy/test_server.py
  46. 361
      test/unit/proxy/test_sysmeta.py

2
doc/source/deployment_guide.rst

@ -486,7 +486,7 @@ handoff_delete auto By default handoff partitions will be
the partition if it is successfully
replicated to n nodes. The default
setting should not be changed, except
for extremem situations.
for extreme situations.
node_timeout DEFAULT or 10 Request timeout to external services.
This uses what's set here, or what's set
in the DEFAULT section, or 10 (though

9
doc/source/logs.rst

@ -98,6 +98,7 @@ CQ :ref:`container-quotas`
CS :ref:`container-sync`
TA :ref:`common_tempauth`
DLO :ref:`dynamic-large-objects`
LE :ref:`list_endpoints`
======================= =============================
@ -126,9 +127,11 @@ status_int The response code for the request.
content_length The value of the Content-Length header in the response.
referer The value of the HTTP Referer header.
transaction_id The transaction id of the request.
user_agent The value of the HTTP User-Agent header. Swift's proxy
server sets its user-agent to
``"proxy-server <pid of the proxy>".``
user_agent The value of the HTTP User-Agent header. Swift services
report a user-agent string of the service name followed by
the process ID, such as ``"proxy-server <pid of the
proxy>"`` or ``"object-updater <pid of the object
updater>"``.
request_time The duration of the request.
additional_info Additional useful information.
server_pid The process id of the server

6
doc/source/overview_auth.rst

@ -113,11 +113,11 @@ Swift is able to authenticate against OpenStack keystone via the
:mod:`swift.common.middleware.keystoneauth` middleware.
In order to use the ``keystoneauth`` middleware the ``authtoken``
middleware from python-keystoneclient will need to be configured.
middleware from keystonemiddleware will need to be configured.
The ``authtoken`` middleware performs the authentication token
validation and retrieves actual user authentication information. It
can be found in the python-keystoneclient distribution.
can be found in the keystonemiddleware distribution.
The ``keystoneauth`` middleware performs authorization and mapping the
``keystone`` roles to Swift's ACLs.
@ -149,7 +149,7 @@ and add auth_token and keystoneauth in your
add the configuration for the authtoken middleware::
[filter:authtoken]
paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
paste.filter_factory = keystonemiddleware.auth_token:filter_factory
auth_host = keystonehost
auth_port = 35357
auth_protocol = http

2
doc/source/overview_ring.rst

@ -238,7 +238,7 @@ were determined by "walking" the ring until finding additional devices in other
zones. This was discarded as control was lost as to how many replicas for a
given partition moved at once. Keeping each replica independent allows for
moving only one partition replica within a given time window (except due to
device failures). Using the additional memory was deemed a good tradeoff for
device failures). Using the additional memory was deemed a good trade-off for
moving data around the cluster much less often.
Another ring design was tried where the partition to device assignments weren't

43
etc/object-expirer.conf-sample

@ -52,7 +52,7 @@
# reclaim_age = 604800
[pipeline:main]
pipeline = catch_errors cache proxy-server
pipeline = catch_errors proxy-logging cache proxy-server
[app:proxy-server]
use = egg:swift#proxy
@ -65,3 +65,44 @@ use = egg:swift#memcache
[filter:catch_errors]
use = egg:swift#catch_errors
# See proxy-server.conf-sample for options
[filter:proxy-logging]
use = egg:swift#proxy_logging
# If not set, logging directives from [DEFAULT] without "access_" will be used
# access_log_name = swift
# access_log_facility = LOG_LOCAL0
# access_log_level = INFO
# access_log_address = /dev/log
#
# If set, access_log_udp_host will override access_log_address
# access_log_udp_host =
# access_log_udp_port = 514
#
# You can use log_statsd_* from [DEFAULT] or override them here:
# access_log_statsd_host = localhost
# access_log_statsd_port = 8125
# access_log_statsd_default_sample_rate = 1.0
# access_log_statsd_sample_rate_factor = 1.0
# access_log_statsd_metric_prefix =
# access_log_headers = false
#
# If access_log_headers is True and access_log_headers_only is set only
# these headers are logged. Multiple headers can be defined as comma separated
# list like this: access_log_headers_only = Host, X-Object-Meta-Mtime
# access_log_headers_only =
#
# By default, the X-Auth-Token is logged. To obscure the value,
# set reveal_sensitive_prefix to the number of characters to log.
# For example, if set to 12, only the first 12 characters of the
# token appear in the log. An unauthorized access of the log file
# won't allow unauthorized usage of the token. However, the first
# 12 or so characters is unique enough that you can trace/debug
# token usage. Set to 0 to suppress the token completely (replaced
# by '...' in the log).
# Note: reveal_sensitive_prefix will not affect the value
# logged with access_log_headers=True.
# reveal_sensitive_prefix = 16
#
# What HTTP methods are allowed for StatsD logging (comma-sep); request methods
# not in this list will have "BAD_METHOD" for the <verb> portion of the metric.
# log_statsd_valid_http_methods = GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS

2
etc/proxy-server.conf-sample

@ -266,7 +266,7 @@ user_test_tester3 = testing3
# there you can change it to: authtoken keystoneauth
#
# [filter:authtoken]
# paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
# paste.filter_factory = keystonemiddleware.auth_token:filter_factory
# auth_host = keystonehost
# auth_port = 35357
# auth_protocol = http

15
swift/account/backend.py

@ -453,6 +453,7 @@ class AccountBroker(DatabaseBroker):
"""
def _really_merge_items(conn):
max_rowid = -1
curs = conn.cursor()
for rec in item_list:
record = [rec['name'], rec['put_timestamp'],
rec['delete_timestamp'], rec['object_count'],
@ -466,9 +467,9 @@ class AccountBroker(DatabaseBroker):
'''
if self.get_db_version(conn) >= 1:
query += ' AND deleted IN (0, 1)'
curs = conn.execute(query, (rec['name'],))
curs.row_factory = None
row = curs.fetchone()
curs_row = curs.execute(query, (rec['name'],))
curs_row.row_factory = None
row = curs_row.fetchone()
if row:
row = list(row)
for i in xrange(5):
@ -484,11 +485,11 @@ class AccountBroker(DatabaseBroker):
record[5] = 1
else:
record[5] = 0
conn.execute('''
curs.execute('''
DELETE FROM container WHERE name = ? AND
deleted IN (0, 1)
''', (record[0],))
conn.execute('''
curs.execute('''
INSERT INTO container (name, put_timestamp,
delete_timestamp, object_count, bytes_used,
deleted, storage_policy_index)
@ -498,12 +499,12 @@ class AccountBroker(DatabaseBroker):
max_rowid = max(max_rowid, rec['ROWID'])
if source:
try:
conn.execute('''
curs.execute('''
INSERT INTO incoming_sync (sync_point, remote_id)
VALUES (?, ?)
''', (max_rowid, source))
except sqlite3.IntegrityError:
conn.execute('''
curs.execute('''
UPDATE incoming_sync
SET sync_point=max(?, sync_point)
WHERE remote_id=?

2
swift/cli/recon.py

@ -927,7 +927,7 @@ class SwiftRecon(object):
self.auditor_check(hosts)
self.umount_check(hosts)
self.load_check(hosts)
self.disk_usage(hosts)
self.disk_usage(hosts, options.top, options.human_readable)
self.get_ringmd5(hosts, swift_dir)
self.quarantine_check(hosts)
self.socket_usage(hosts)

9
swift/common/internal_client.py

@ -728,6 +728,7 @@ class SimpleClient(object):
max_backoff=5, retries=5):
self.url = url
self.token = token
self.attempts = 0 # needed in swif-dispersion-populate
self.starting_backoff = starting_backoff
self.max_backoff = max_backoff
self.retries = retries
@ -796,14 +797,14 @@ class SimpleClient(object):
def retry_request(self, method, **kwargs):
retries = kwargs.pop('retries', self.retries)
attempts = 0
self.attempts = 0
backoff = self.starting_backoff
while attempts <= retries:
attempts += 1
while self.attempts <= retries:
self.attempts += 1
try:
return self.base_request(method, **kwargs)
except (socket.error, httplib.HTTPException, urllib2.URLError):
if attempts > retries:
if self.attempts > retries:
raise
sleep(backoff)
backoff = min(backoff * 2, self.max_backoff)

28
swift/common/middleware/formpost.py

@ -31,7 +31,14 @@ The format of the form is::
<input type="submit" />
</form>
The <swift-url> is the URL to the Swift desination, such as::
Optionally, if you want the uploaded files to be temporary you can set
x-delete-at or x-delete-after attributes by adding one of these as a
form input::
<input type="hidden" name="x_delete_at" value="<unix-timestamp>" />
<input type="hidden" name="x_delete_after" value="<seconds>" />
The <swift-url> is the URL of the Swift destination, such as::
https://swift-cluster.example.com/v1/AUTH_account/container/object_prefix
@ -83,10 +90,12 @@ sample code for computing the signature::
max_file_size, max_file_count, expires)
signature = hmac.new(key, hmac_body, sha1).hexdigest()
The key is the value of the X-Account-Meta-Temp-URL-Key header on the
account.
The key is the value of either the X-Account-Meta-Temp-URL-Key or the
X-Account-Meta-Temp-Url-Key-2 header on the account.
Be certain to use the full path, from the /v1/ onward.
Note that x_delete_at and x_delete_after are not used in signature generation
as they are both optional attributes.
The command line tool ``swift-form-signature`` may be used (mostly
just when testing) to compute expires and signature.
@ -441,6 +450,19 @@ class FormPost(object):
subenv['PATH_INFO'].count('/') < 4:
subenv['PATH_INFO'] += '/'
subenv['PATH_INFO'] += attributes['filename'] or 'filename'
if 'x_delete_at' in attributes:
try:
subenv['HTTP_X_DELETE_AT'] = int(attributes['x_delete_at'])
except ValueError:
raise FormInvalid('x_delete_at not an integer: '
'Unix timestamp required.')
if 'x_delete_after' in attributes:
try:
subenv['HTTP_X_DELETE_AFTER'] = int(
attributes['x_delete_after'])
except ValueError:
raise FormInvalid('x_delete_after not an integer: '
'Number of seconds required.')
if 'content-type' in attributes:
subenv['CONTENT_TYPE'] = \
attributes['content-type'] or 'application/octet-stream'

124
swift/common/middleware/list_endpoints.py

@ -20,11 +20,14 @@ This middleware makes it possible to integrate swift with software
that relies on data locality information to avoid network overhead,
such as Hadoop.
Answers requests of the form::
Using the original API, answers requests of the form::
/endpoints/{account}/{container}/{object}
/endpoints/{account}/{container}
/endpoints/{account}
/endpoints/v1/{account}/{container}/{object}
/endpoints/v1/{account}/{container}
/endpoints/v1/{account}
with a JSON-encoded list of endpoints of the form::
@ -38,6 +41,26 @@ correspondingly, e.g.::
http://10.1.1.1:6000/sda1/2/a/c2
http://10.1.1.1:6000/sda1/2/a
Using the v2 API, answers requests of the form::
/endpoints/v2/{account}/{container}/{object}
/endpoints/v2/{account}/{container}
/endpoints/v2/{account}
with a JSON-encoded dictionary containing a key 'endpoints' that maps to a list
of endpoints having the same form as described above, and a key 'headers' that
maps to a dictionary of headers that should be sent with a request made to
the endpoints, e.g.::
{ "endpoints": {"http://10.1.1.1:6010/sda1/2/a/c3/o1",
"http://10.1.1.1:6030/sda3/2/a/c3/o1",
"http://10.1.1.1:6040/sda4/2/a/c3/o1"},
"headers": {"X-Backend-Storage-Policy-Index": "1"}}
In this example, the 'headers' dictionary indicates that requests to the
endpoint URLs should include the header 'X-Backend-Storage-Policy-Index: 1'
because the object's container is using storage policy index 1.
The '/endpoints/' path is customizable ('list_endpoints_path'
configuration parameter).
@ -64,6 +87,8 @@ from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
from swift.common.storage_policy import POLICIES
from swift.proxy.controllers.base import get_container_info
RESPONSE_VERSIONS = (1.0, 2.0)
class ListEndpointsMiddleware(object):
"""
@ -87,6 +112,11 @@ class ListEndpointsMiddleware(object):
self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/')
if not self.endpoints_path.endswith('/'):
self.endpoints_path += '/'
self.default_response_version = 1.0
self.response_map = {
1.0: self.v1_format_response,
2.0: self.v2_format_response,
}
def get_object_ring(self, policy_idx):
"""
@ -97,6 +127,71 @@ class ListEndpointsMiddleware(object):
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def _parse_version(self, raw_version):
err_msg = 'Unsupported version %r' % raw_version
try:
version = float(raw_version.lstrip('v'))
except ValueError:
raise ValueError(err_msg)
if not any(version == v for v in RESPONSE_VERSIONS):
raise ValueError(err_msg)
return version
def _parse_path(self, request):
"""
Parse path parts of request into a tuple of version, account,
container, obj. Unspecified path parts are filled in as None,
except version which is always returned as a float using the
configured default response version if not specified in the
request.
:param request: the swob request
:returns: parsed path parts as a tuple with version filled in as
configured default response version if not specified.
:raises: ValueError if path is invalid, message will say why.
"""
clean_path = request.path[len(self.endpoints_path) - 1:]
# try to peel off version
try:
raw_version, rest = split_path(clean_path, 1, 2, True)
except ValueError:
raise ValueError('No account specified')
try:
version = self._parse_version(raw_version)
except ValueError:
if raw_version.startswith('v') and '_' not in raw_version:
# looks more like a invalid version than an account
raise
# probably no version specified, but if the client really
# said /endpoints/v_3/account they'll probably be sorta
# confused by the useless response and lack of error.
version = self.default_response_version
rest = clean_path
else:
rest = '/' + rest if rest else '/'
try:
account, container, obj = split_path(rest, 1, 3, True)
except ValueError:
raise ValueError('No account specified')
return version, account, container, obj
def v1_format_response(self, req, endpoints, **kwargs):
return Response(json.dumps(endpoints),
content_type='application/json')
def v2_format_response(self, req, endpoints, storage_policy_index,
**kwargs):
resp = {
'endpoints': endpoints,
'headers': {},
}
if storage_policy_index is not None:
resp['headers'][
'X-Backend-Storage-Policy-Index'] = str(storage_policy_index)
return Response(json.dumps(resp),
content_type='application/json')
def __call__(self, env, start_response):
request = Request(env)
if not request.path.startswith(self.endpoints_path):
@ -107,11 +202,9 @@ class ListEndpointsMiddleware(object):
req=request, headers={"Allow": "GET"})(env, start_response)
try:
clean_path = request.path[len(self.endpoints_path) - 1:]
account, container, obj = \
split_path(clean_path, 1, 3, True)
except ValueError:
return HTTPBadRequest('No account specified')(env, start_response)
version, account, container, obj = self._parse_path(request)
except ValueError as err:
return HTTPBadRequest(str(err))(env, start_response)
if account is not None:
account = unquote(account)
@ -120,16 +213,13 @@ class ListEndpointsMiddleware(object):
if obj is not None:
obj = unquote(obj)
storage_policy_index = None
if obj is not None:
# remove 'endpoints' from call to get_container_info
stripped = request.environ
if stripped['PATH_INFO'][:len(self.endpoints_path)] == \
self.endpoints_path:
stripped['PATH_INFO'] = "/v1/" + \
stripped['PATH_INFO'][len(self.endpoints_path):]
container_info = get_container_info(
stripped, self.app, swift_source='LE')
obj_ring = self.get_object_ring(container_info['storage_policy'])
{'PATH_INFO': '/v1/%s/%s' % (account, container)},
self.app, swift_source='LE')
storage_policy_index = container_info['storage_policy']
obj_ring = self.get_object_ring(storage_policy_index)
partition, nodes = obj_ring.get_nodes(
account, container, obj)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
@ -157,8 +247,10 @@ class ListEndpointsMiddleware(object):
obj=quote(obj or ''))
endpoints.append(endpoint)
return Response(json.dumps(endpoints),
content_type='application/json')(env, start_response)
resp = self.response_map[version](
request, endpoints=endpoints,
storage_policy_index=storage_policy_index)
return resp(env, start_response)
def filter_factory(global_conf, **local_conf):

4
swift/common/middleware/proxy_logging.py

@ -253,10 +253,10 @@ class ProxyLoggingMiddleware(object):
break
else:
if not chunk:
start_response_args[0][1].append(('content-length', '0'))
start_response_args[0][1].append(('Content-Length', '0'))
elif isinstance(iterable, list):
start_response_args[0][1].append(
('content-length', str(sum(len(i) for i in iterable))))
('Content-Length', str(sum(len(i) for i in iterable))))
start_response(*start_response_args[0])
req = Request(env)

10
swift/common/middleware/tempurl.py

@ -46,9 +46,9 @@ limited to the expiration time set when the website created the link.
To create such temporary URLs, first an X-Account-Meta-Temp-URL-Key
header must be set on the Swift account. Then, an HMAC-SHA1 (RFC 2104)
signature is generated using the HTTP method to allow (GET or PUT),
the Unix timestamp the access should be allowed until, the full path
to the object, and the key set on the account.
signature is generated using the HTTP method to allow (GET, PUT,
DELETE, etc.), the Unix timestamp the access should be allowed until,
the full path to the object, and the key set on the account.
For example, here is code generating the signature for a GET for 60
seconds on /v1/AUTH_account/container/object::
@ -75,7 +75,7 @@ da39a3ee5e6b4b0d3255bfef95601890afd80709 and expires ends up
Any alteration of the resource path or query arguments would result
in 401 Unauthorized. Similary, a PUT where GET was the allowed method
would 401. HEAD is allowed if GET or PUT is allowed.
would 401. HEAD is allowed if GET, PUT, or POST is allowed.
Using this in combination with browser form post translation
middleware could also allow direct-from-browser uploads to specific
@ -300,6 +300,8 @@ class TempURL(object):
self._get_hmacs(env, temp_url_expires, keys) +
self._get_hmacs(env, temp_url_expires, keys,
request_method='GET') +
self._get_hmacs(env, temp_url_expires, keys,
request_method='POST') +
self._get_hmacs(env, temp_url_expires, keys,
request_method='PUT'))
else:

15
swift/common/request_helpers.py

@ -225,6 +225,21 @@ def remove_items(headers, condition):
return removed
def copy_header_subset(from_r, to_r, condition):
"""
Will copy desired subset of headers from from_r to to_r.
:param from_r: a swob Request or Response
:param to_r: a swob Request or Response
:param condition: a function that will be passed the header key as a
single argument and should return True if the header
is to be copied.
"""
for k, v in from_r.headers.items():
if condition(k):
to_r.headers[k] = v
def close_if_possible(maybe_closable):
close_method = getattr(maybe_closable, 'close', None)
if callable(close_method):

1
swift/common/ring/utils.py

@ -280,7 +280,6 @@ def parse_builder_ring_filename_args(argvish):
ring_file = first_arg
else:
ring_file = builder_file[:-len('.builder')]
if not first_arg.endswith('.ring.gz'):
ring_file += '.ring.gz'
return builder_file, ring_file

44
swift/common/utils.py

@ -632,7 +632,7 @@ class Timestamp(object):
return INTERNAL_FORMAT % (self.timestamp, self.offset)
def __str__(self):
raise TypeError('You must specificy which string format is required')
raise TypeError('You must specify which string format is required')
def __float__(self):
return self.timestamp
@ -1629,26 +1629,32 @@ def lock_file(filename, timeout=10, append=False, unlink=True):
mode = 'a+'
else:
mode = 'r+'
fd = os.open(filename, flags)
file_obj = os.fdopen(fd, mode)
try:
with swift.common.exceptions.LockTimeout(timeout, filename):
while True:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except IOError as err:
if err.errno != errno.EAGAIN:
raise
sleep(0.01)
yield file_obj
finally:
while True:
fd = os.open(filename, flags)
file_obj = os.fdopen(fd, mode)
try:
with swift.common.exceptions.LockTimeout(timeout, filename):
while True:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except IOError as err:
if err.errno != errno.EAGAIN:
raise
sleep(0.01)
try:
if os.stat(filename).st_ino != os.fstat(fd).st_ino:
continue
except OSError as err:
if err.errno == errno.ENOENT:
continue
raise
yield file_obj
if unlink:
os.unlink(filename)
break
finally:
file_obj.close()
except UnboundLocalError:
pass # may have not actually opened the file
if unlink:
os.unlink(filename)
def lock_parent_directory(filename, timeout=10):

10
swift/common/wsgi.py

@ -16,6 +16,7 @@
"""WSGI tools for use with swift."""
import errno
import inspect
import os
import signal
import time
@ -386,7 +387,14 @@ def run_server(conf, logger, sock, global_conf=None):
max_clients = int(conf.get('max_clients', '1024'))
pool = RestrictedGreenPool(size=max_clients)
try:
wsgi.server(sock, app, NullLogger(), custom_pool=pool)
# Disable capitalizing headers in Eventlet if possible. This is
# necessary for the AWS SDK to work with swift3 middleware.
argspec = inspect.getargspec(wsgi.server)
if 'capitalize_response_headers' in argspec.args:
wsgi.server(sock, app, NullLogger(), custom_pool=pool,
capitalize_response_headers=False)
else:
wsgi.server(sock, app, NullLogger(), custom_pool=pool)
except socket.error as err:
if err[0] != errno.EINVAL:
raise

16
swift/container/backend.py

@ -328,7 +328,7 @@ class ContainerBroker(DatabaseBroker):
:param content_type: object content-type
:param etag: object etag
:param deleted: if True, marks the object as deleted and sets the
deteleted_at timestamp to timestamp
deleted_at timestamp to timestamp
:param storage_policy_index: the storage policy index for the object
"""
record = {'name': name, 'created_at': timestamp, 'size': size,
@ -582,7 +582,7 @@ class ContainerBroker(DatabaseBroker):
:param end_marker: end marker query
:param prefix: prefix query
:param delimiter: delimiter for query
:param path: if defined, will set the prefix and delimter based on
:param path: if defined, will set the prefix and delimiter based on
the path
:returns: list of tuples of (name, created_at, size, content_type,
@ -679,7 +679,7 @@ class ContainerBroker(DatabaseBroker):
break
elif end > 0:
marker = name[:end] + chr(ord(delimiter) + 1)
# we want result to be inclusinve of delim+1
# we want result to be inclusive of delim+1
delim_force_gte = True
dir_name = name[:end + 1]
if dir_name != orig_marker:
@ -696,11 +696,13 @@ class ContainerBroker(DatabaseBroker):
Merge items into the object table.
:param item_list: list of dictionaries of {'name', 'created_at',
'size', 'content_type', 'etag', 'deleted'}
'size', 'content_type', 'etag', 'deleted',
'storage_policy_index'}
:param source: if defined, update incoming_sync with the source
"""
def _really_merge_items(conn):
max_rowid = -1
curs = conn.cursor()
for rec in item_list:
rec.setdefault('storage_policy_index', 0) # legacy
query = '''
@ -710,7 +712,7 @@ class ContainerBroker(DatabaseBroker):
'''
if self.get_db_version(conn) >= 1:
query += ' AND deleted IN (0, 1)'
conn.execute(query, (rec['name'], rec['created_at'],
curs.execute(query, (rec['name'], rec['created_at'],
rec['storage_policy_index']))
query = '''
SELECT 1 FROM object WHERE name = ?
@ -718,9 +720,9 @@ class ContainerBroker(DatabaseBroker):
'''
if self.get_db_version(conn) >= 1:
query += ' AND deleted IN (0, 1)'
if not conn.execute(query, (
if not curs.execute(query, (
rec['name'], rec['storage_policy_index'])).fetchall():
conn.execute('''
curs.execute('''
INSERT INTO object (name, created_at, size,
content_type, etag, deleted, storage_policy_index)
VALUES (?, ?, ?, ?, ?, ?, ?)

4
swift/obj/diskfile.py

@ -49,6 +49,7 @@ from eventlet import Timeout
from swift import gettext_ as _
from swift.common.constraints import check_mount
from swift.common.request_helpers import is_sys_meta
from swift.common.utils import mkdirs, Timestamp, \
storage_directory, hash_path, renamer, fallocate, fsync, \
fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle, \
@ -1315,7 +1316,8 @@ class DiskFile(object):
self._metadata = self._failsafe_read_metadata(meta_file, meta_file)
sys_metadata = dict(
[(key, val) for key, val in datafile_metadata.iteritems()
if key.lower() in DATAFILE_SYSTEM_META])
if key.lower() in DATAFILE_SYSTEM_META
or is_sys_meta('object', key)])
self._metadata.update(sys_metadata)
else:
self._metadata = datafile_metadata

2
swift/obj/expirer.py

@ -118,7 +118,7 @@ class ObjectExpirer(Daemon):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (container, obj)).
hashlib.md5('%s/%s' % (str(container), obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue

2
swift/obj/mem_server.py

@ -70,7 +70,7 @@ class ObjectController(server.ObjectController):
:param objdevice: device name that the object is in
:param policy_idx: the associated storage policy index
"""
headers_out['user-agent'] = 'obj-server %s' % os.getpid()
headers_out['user-agent'] = 'object-server %s' % os.getpid()
full_path = '/%s/%s/%s' % (account, container, obj)
if all([host, partition, contdevice]):
try:

2
swift/obj/replicator.py

@ -86,7 +86,7 @@ class ObjectReplicator(Daemon):
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
self.headers = {
'Content-Length': '0',
'user-agent': 'obj-replicator %s' % os.getpid()}
'user-agent': 'object-replicator %s' % os.getpid()}
self.rsync_error_log_line_length = \
int(conf.get('rsync_error_log_line_length', 0))
self.handoffs_first = config_true_value(conf.get('handoffs_first',

26
swift/obj/server.py

@ -38,7 +38,8 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout
from swift.obj import ssync_receiver
from swift.common.http import is_success
from swift.common.request_helpers import get_name_and_placement, is_user_meta
from swift.common.request_helpers import get_name_and_placement, \
is_user_meta, is_sys_or_user_meta
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
@ -169,7 +170,7 @@ class ObjectController(object):
:param objdevice: device name that the object is in
:param policy_index: the associated storage policy index
"""
headers_out['user-agent'] = 'obj-server %s' % os.getpid()
headers_out['user-agent'] = 'object-server %s' % os.getpid()
full_path = '/%s/%s/%s' % (account, container, obj)
if all([host, partition, contdevice]):
try:
@ -342,7 +343,9 @@ class ObjectController(object):
return HTTPNotFound(request=request)
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp >= req_timestamp:
return HTTPConflict(request=request)
return HTTPConflict(
request=request,
headers={'X-Backend-Timestamp': orig_timestamp.internal})
metadata = {'X-Timestamp': req_timestamp.internal}
metadata.update(val for val in request.headers.iteritems()
if is_user_meta('object', val[0]))
@ -402,8 +405,10 @@ class ObjectController(object):
return HTTPPreconditionFailed(request=request)
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp and orig_timestamp >= req_timestamp:
return HTTPConflict(request=request)
if orig_timestamp >= req_timestamp:
return HTTPConflict(
request=request,
headers={'X-Backend-Timestamp': orig_timestamp.internal})
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
upload_expiration = time.time() + self.max_upload_time
etag = md5()
@ -445,7 +450,7 @@ class ObjectController(object):
'Content-Length': str(upload_size),
}
metadata.update(val for val in request.headers.iteritems()
if is_user_meta('object', val[0]))
if is_sys_or_user_meta('object', val[0]))
for header_key in (
request.headers.get('X-Backend-Replication-Headers') or
self.allowed_headers):
@ -503,7 +508,7 @@ class ObjectController(object):
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_user_meta('object', key) or \
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
@ -549,7 +554,7 @@ class ObjectController(object):
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_user_meta('object', key) or \
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
@ -598,6 +603,7 @@ class ObjectController(object):
response_class = HTTPNoContent
else:
response_class = HTTPConflict
response_timestamp = max(orig_timestamp, req_timestamp)
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
try:
req_if_delete_at_val = request.headers['x-if-delete-at']
@ -631,7 +637,9 @@ class ObjectController(object):
'DELETE', account, container, obj, request,
HeaderKeyDict({'x-timestamp': req_timestamp.internal}),
device, policy_idx)
return response_class(request=request)
return response_class(
request=request,
headers={'X-Backend-Timestamp': response_timestamp.internal})
@public
@replication

2
swift/obj/updater.py

@ -258,7 +258,7 @@ class ObjectUpdater(Daemon):
:param headers: headers to send with the update
"""
headers_out = headers.copy()
headers_out['user-agent'] = 'obj-updater %s' % os.getpid()
headers_out['user-agent'] = 'object-updater %s' % os.getpid()
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'],

14
swift/proxy/controllers/container.py

@ -20,7 +20,7 @@ import time
from swift.common.utils import public, csv_append, Timestamp
from swift.common.constraints import check_metadata
from swift.common import constraints
from swift.common.http import HTTP_ACCEPTED
from swift.common.http import HTTP_ACCEPTED, is_success
from swift.proxy.controllers.base import Controller, delay_denial, \
cors_validation, clear_info_cache
from swift.common.storage_policy import POLICIES
@ -144,10 +144,14 @@ class ContainerController(Controller):
if self.app.max_containers_per_account > 0 and \
container_count >= self.app.max_containers_per_account and \
self.account_name not in self.app.max_containers_whitelist:
resp = HTTPForbidden(request=req)
resp.body = 'Reached container limit of %s' % \
self.app.max_containers_per_account
return resp
container_info = \
self.container_info(self.account_name, self.container_name,
req)
if not is_success(container_info.get('status')):
resp = HTTPForbidden(request=req)
resp.body = 'Reached container limit of %s' % \
self.app.max_containers_per_account
return resp
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),

13
swift/proxy/controllers/obj.py

@ -56,7 +56,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
HTTPServerError, HTTPServiceUnavailable, Request, \
HTTPClientDisconnect, HTTPNotImplemented
from swift.common.request_helpers import is_user_meta
from swift.common.request_helpers import is_sys_or_user_meta, is_sys_meta, \
remove_items, copy_header_subset
def copy_headers_into(from_r, to_r):
@ -67,7 +68,7 @@ def copy_headers_into(from_r, to_r):
"""
pass_headers = ['x-delete-at']
for k, v in from_r.headers.items():
if is_user_meta('object', k) or k.lower() in pass_headers:
if is_sys_or_user_meta('object', k) or k.lower() in pass_headers:
to_r.headers[k] = v
@ -624,8 +625,14 @@ class ObjectController(Controller):
if not content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if not config_true_value(
if config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false')):
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies

5
swift/proxy/server.py

@ -490,6 +490,7 @@ class Application(object):
handoff_nodes = node_iter
nodes_left = self.request_node_count(len(primary_nodes))
log_handoffs_threshold = nodes_left - len(primary_nodes)
for node in primary_nodes:
if not self.error_limited(node):
yield node
@ -501,11 +502,11 @@ class Application(object):
for node in handoff_nodes:
if not self.error_limited(node):
handoffs += 1
if self.log_handoffs:
if self.log_handoffs and handoffs > log_handoffs_threshold:
self.logger.increment('handoff_count')
self.logger.warning(
'Handoff requested (%d)' % handoffs)
if handoffs == len(primary_nodes):
if handoffs - log_handoffs_threshold == len(primary_nodes):
self.logger.increment('handoff_all_count')
yield node
if not self.error_limited(node):

9
test/functional/__init__.py

@ -32,7 +32,8 @@ from shutil import rmtree
from tempfile import mkdtemp
from test import get_config
from test.functional.swift_test_client import Connection, ResponseError
from test.functional.swift_test_client import Account, Connection, \
ResponseError
# This has the side effect of mocking out the xattr module so that unit tests
# (and in this case, when in-process functional tests are called for) can run
# on file systems that don't support extended attributes.
@ -513,6 +514,12 @@ def teardown_package():
global orig_collate
locale.setlocale(locale.LC_COLLATE, orig_collate)
# clean up containers and objects left behind after running tests
conn = Connection(config)
conn.authenticate()
account = Account(conn, config.get('account', config['username']))
account.delete_containers()
global in_process
if in_process:
try:

206
test/probe/brain.py

@ -0,0 +1,206 @@
#!/usr/bin/python -u
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import itertools
import uuid
from optparse import OptionParser
from urlparse import urlparse
import random
from swift.common.manager import Manager
from swift.common import utils, ring
from swift.common.storage_policy import POLICIES
from swift.common.http import HTTP_NOT_FOUND
from swiftclient import client, get_auth, ClientException
TIMEOUT = 60
def meta_command(name, bases, attrs):
"""
Look for attrs with a truthy attribute __command__ and add them to an
attribute __commands__ on the type that maps names to decorated methods.
The decorated methods' doc strings also get mapped in __docs__.
Also adds a method run(command_name, *args, **kwargs) that will
execute the method mapped to the name in __commands__.
"""
commands = {}
docs = {}
for attr, value in attrs.items():
if getattr(value, '__command__', False):
commands[attr] = value
# methods have always have a __doc__ attribute, sometimes empty
docs[attr] = (getattr(value, '__doc__', None) or
'perform the %s command' % attr).strip()
attrs['__commands__'] = commands
attrs['__docs__'] = docs
def run(self, command, *args, **kwargs):
return self.__commands__[command](self, *args, **kwargs)
attrs.setdefault('run', run)
return type(name, bases, attrs)
def command(f):
f.__command__ = True
return f
class BrainSplitter(object):
__metaclass__ = meta_command
def __init__(self, url, token, container_name='test', object_name='test',
server_type='container'):
self.url = url
self.token = token
self.account = utils.split_path(urlparse(url).path, 2, 2)[1]
self.container_name = container_name
self.object_name = object_name
server_list = ['%s-server' % server_type] if server_type else ['all']
self.servers = Manager(server_list)
policies = list(POLICIES)
random.shuffle(policies)
self.policies = itertools.cycle(policies)
o = object_name if server_type == 'object' else None
c = container_name if server_type in ('object', 'container') else None
part, nodes = ring.Ring(
'/etc/swift/%s.ring.gz' % server_type).get_nodes(
self.account, c, o)
node_ids = [n['id'] for n in nodes]
if all(n_id in node_ids for n_id in (0, 1)):
self.primary_numbers = (1, 2)
self.handoff_numbers = (3, 4)
else:
self.primary_numbers = (3, 4)
self.handoff_numbers = (1, 2)
@command
def start_primary_half(self):
"""
start servers 1 & 2
"""
tuple(self.servers.start(number=n) for n in self.primary_numbers)
@command
def stop_primary_half(self):
"""
stop servers 1 & 2
"""
tuple(self.servers.stop(number=n) for n in self.primary_numbers)
@command
def start_handoff_half(self):
"""
start servers 3 & 4
"""
tuple(self.servers.start(number=n) for n in self.handoff_numbers)
@command
def stop_handoff_half(self):
"""
stop servers 3 & 4
"""
tuple(self.servers.stop(number=n) for n in self.handoff_numbers)
@command
def put_container(self, policy_index=None):
"""
put container with next storage policy
"""
policy = self.policies.next()
if policy_index is not None:
policy = POLICIES.get_by_index(int(policy_index))
if not policy:
raise ValueError('Unknown policy with index %s' % policy)
headers = {'X-Storage-Policy': policy.name}
client.put_container(self.url, self.token, self.container_name,
headers=headers)
@command
def delete_container(self):
"""
delete container
"""
client.delete_container(self.url, self.token, self.container_name)
@command
def put_object(self, headers=None):
"""
issue put for zero byte test object
"""
client.put_object(self.url, self.token, self.container_name,
self.object_name, headers=headers)
@command
def delete_object(self):
"""
issue delete for test object
"""
try:
client.delete_object(self.url, self.token, self.container_name,
self.object_name)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
parser = OptionParser('%prog [options] '
'<command>[:<args>[,<args>...]] [<command>...]')
parser.usage += '\n\nCommands:\n\t' + \
'\n\t'.join("%s - %s" % (name, doc) for name, doc in
BrainSplitter.__docs__.items())
parser.add_option('-c', '--container', default='container-%s' % uuid.uuid4(),
help='set container name')
<