openstack
/
swift
Code Issues Proposed changes

Added account_autocreate mode where authorized requests to accounts that don't yet exist within Swift will cause those accounts to be automatically created. Also did a small refactor surrounding swift.common.utils.TRUE_VALUES.

This commit is contained in:
gholt 2011-06-10 22:27:05 +00:00 committed by Tarmac
parent db12ee2fc9 818c4faa73
commit 156b6e0c92
10 changed files with 92 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/deployment_guide.rst
View File

@ -547,6 +547,10 @@ error_suppression_limit 10 Error count to consider a
node error limited
allow_account_management false Whether account PUTs and DELETEs
are even callable
account_autocreate false If set to 'true' authorized
accounts that do not yet exist
within the Swift cluster will
be automatically created.
============================ =============== =============================
[tempauth]

3
etc/proxy-server.conf-sample
View File

@ -40,6 +40,9 @@ use = egg:swift#proxy
# If set to 'true' any authorized user may create and delete accounts; if
# 'false' no one, even authorized, can.
# allow_account_management = false
# If set to 'true' authorized accounts that do not yet exist within the Swift
# cluster will be automatically created.
# account_autocreate = false
[filter:tempauth]
use = egg:swift#tempauth

4
swift/common/bench.py
View File

@ -43,7 +43,7 @@ class Bench(object):
self.user = conf.user
self.key = conf.key
self.auth_url = conf.auth
self.use_proxy = conf.use_proxy in TRUE_VALUES
self.use_proxy = conf.use_proxy.lower() in TRUE_VALUES
if self.use_proxy:
url, token = client.get_auth(self.auth_url, self.user, self.key)
self.token = token
@ -125,7 +125,7 @@ class BenchController(object):
self.logger = logger
self.conf = conf
self.names = []
self.delete = conf.delete in TRUE_VALUES
self.delete = conf.delete.lower() in TRUE_VALUES
self.gets = int(conf.num_gets)
def run(self):

3
swift/common/daemon.py
View File

@ -75,7 +75,8 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
log_name=kwargs.get('log_name'))
# once on command line (i.e. daemonize=false) will over-ride config
once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES
once = once or \
conf.get('daemonize', 'true').lower() not in utils.TRUE_VALUES
# pre-configure logger
if 'logger' in kwargs:

2
swift/common/middleware/staticweb.py
View File

@ -270,7 +270,7 @@ class StaticWeb(object):
:param start_response: The original WSGI start_response hook.
:param prefix: Any prefix desired for the container listing.
"""
if self._listings not in TRUE_VALUES:
if self._listings.lower() not in TRUE_VALUES:
resp = HTTPNotFound()(env, self._start_response)
return self._error_response(resp, env, start_response)
tmp_env = self._get_escalated_env(env)

2
swift/common/utils.py
View File

@ -72,7 +72,7 @@ if hash_conf.read('/etc/swift/swift.conf'):
pass
# Used when reading config values
TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes', 'on', 'On', 't', 'y'))
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
def validate_configuration():

41
swift/proxy/server.py
View File

@ -41,8 +41,8 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \
from webob import Request, Response
from swift.common.ring import Ring
from swift.common.utils import get_logger, normalize_timestamp, split_path, \
cache_from_env, ContextPool
from swift.common.utils import cache_from_env, ContextPool, get_logger, \
normalize_timestamp, split_path, TRUE_VALUES
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_metadata, check_object_creation, \
check_utf8, CONTAINER_LISTING_LIMIT, MAX_ACCOUNT_NAME_LENGTH, \
@ -338,7 +338,7 @@ class Controller(object):
node['errors'] = self.app.error_suppression_limit + 1
node['last_error'] = time.time()
def account_info(self, account):
def account_info(self, account, autocreate=False):
"""
Get account information, and also verify that the account exists.
@ -353,7 +353,7 @@ class Controller(object):
result_code = self.app.memcache.get(cache_key)
if result_code == 200:
return partition, nodes
elif result_code == 404:
elif result_code == 404 and not autocreate:
return None, None
result_code = 0
attempts_left = self.app.account_ring.replica_count
@ -386,6 +386,17 @@ class Controller(object):
except (Exception, TimeoutError):
self.exception_occurred(node, _('Account'),
_('Trying to get account info for %s') % path)
if result_code == 404 and autocreate:
if len(account) > MAX_ACCOUNT_NAME_LENGTH:
return None, None
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id}
resp = self.make_requests(Request.blank('/v1' + path),
self.app.account_ring, partition, 'PUT',
path, [headers] * len(nodes))
if resp.status_int // 100 != 2:
raise Exception('Could not autocreate account %r' % path)
result_code = 200
if self.app.memcache and result_code in (200, 404):
if result_code == 200:
cache_timeout = self.app.recheck_account_existence
@ -397,7 +408,7 @@ class Controller(object):
return partition, nodes
return None, None
def container_info(self, account, container):
def container_info(self, account, container, account_autocreate=False):
"""
Get container information and thusly verify container existance.
This will also make a call to account_info to verify that the
@ -423,7 +434,7 @@ class Controller(object):
return partition, nodes, read_acl, write_acl
elif status == 404:
return None, None, None, None
if not self.account_info(account)[1]:
if not self.account_info(account, autocreate=account_autocreate)[1]:
return None, None, None, None
result_code = 0
read_acl = None
@ -854,7 +865,8 @@ class ObjectController(Controller):
if error_response:
return error_response
container_partition, containers, _junk, req.acl = \
self.container_info(self.account_name, self.container_name)
self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
@ -911,7 +923,8 @@ class ObjectController(Controller):
def PUT(self, req):
"""HTTP PUT request handler."""
container_partition, containers, _junk, req.acl = \
self.container_info(self.account_name, self.container_name)
self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
@ -1219,7 +1232,8 @@ class ContainerController(Controller):
resp.body = 'Container name length of %d longer than %d' % \
(len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts = self.account_info(self.account_name)
account_partition, accounts = self.account_info(self.account_name,
autocreate=self.app.account_autocreate)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
@ -1249,7 +1263,8 @@ class ContainerController(Controller):
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
account_partition, accounts = self.account_info(self.account_name)
account_partition, accounts = self.account_info(self.account_name,
autocreate=self.app.account_autocreate)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
@ -1391,7 +1406,7 @@ class BaseApplication(object):
self.put_queue_depth = int(conf.get('put_queue_depth', 10))
self.object_chunk_size = int(conf.get('object_chunk_size', 65536))
self.client_chunk_size = int(conf.get('client_chunk_size', 65536))
self.log_headers = conf.get('log_headers') == 'True'
self.log_headers = conf.get('log_headers', 'no').lower() in TRUE_VALUES
self.error_suppression_interval = \
int(conf.get('error_suppression_interval', 60))
self.error_suppression_limit = \
@ -1401,7 +1416,7 @@ class BaseApplication(object):
self.recheck_account_existence = \
int(conf.get('recheck_account_existence', 60))
self.allow_account_management = \
conf.get('allow_account_management', 'false').lower() == 'true'
conf.get('allow_account_management', 'no').lower() in TRUE_VALUES
self.resellers_conf = ConfigParser()
self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf'))
self.object_ring = object_ring or \
@ -1413,6 +1428,8 @@ class BaseApplication(object):
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles +
[os.path.join(swift_dir, 'mime.types')])
self.account_autocreate = \
conf.get('account_autocreate', 'no').lower() in TRUE_VALUES
def get_controller(self, path):
"""

2
swift/stats/log_uploader.py
View File

@ -69,7 +69,7 @@ class LogUploader(Daemon):
self.internal_proxy = InternalProxy(proxy_server_conf)
self.new_log_cutoff = int(cutoff or
uploader_conf.get('new_log_cutoff', '7200'))
self.unlink_log = uploader_conf.get('unlink_log', 'True').lower() in \
self.unlink_log = uploader_conf.get('unlink_log', 'true').lower() in \
utils.TRUE_VALUES
self.filename_pattern = regex or \
uploader_conf.get('source_filename_pattern',

4
test/unit/common/test_utils.py
View File

@ -768,6 +768,10 @@ log_name = yarr'''
self.assertEquals(utils.human_readable(1237940039285380274899124224),
'1024Yi')
def test_TRUE_VALUES(self):
for v in utils.TRUE_VALUES:
self.assertEquals(v, v.lower())
if __name__ == '__main__':
unittest.main()

48
test/unit/proxy/test_server.py
View File

@ -393,6 +393,48 @@ class TestController(unittest.TestCase):
test(404, 507, 503)
test(503, 503, 503)
def test_account_info_account_autocreate(self):
with save_globals():
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes = \
self.controller.account_info(self.account, autocreate=False)
self.check_account_info_return(partition, nodes, is_none=True)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes = \
self.controller.account_info(self.account)
self.check_account_info_return(partition, nodes, is_none=True)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes = \
self.controller.account_info(self.account, autocreate=True)
self.check_account_info_return(partition, nodes)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 503, 201, 201)
partition, nodes = \
self.controller.account_info(self.account, autocreate=True)
self.check_account_info_return(partition, nodes)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 503, 201, 503)
exc = None
try:
partition, nodes = \
self.controller.account_info(self.account, autocreate=True)
except Exception, err:
exc = err
self.assertEquals(str(exc),
"Could not autocreate account '/some_account'")
def check_container_info_return(self, ret, is_none=False):
if is_none:
partition, nodes, read_acl, write_acl = None, None, None, None
@ -406,7 +448,7 @@ class TestController(unittest.TestCase):
self.assertEqual(write_acl, ret[3])
def test_container_info_invalid_account(self):
def account_info(self, account):
def account_info(self, account, autocreate=False):
return None, None
with save_globals():
@ -417,7 +459,7 @@ class TestController(unittest.TestCase):
# tests if 200 is cached and used
def test_container_info_200(self):
def account_info(self, account):
def account_info(self, account, autocreate=False):
return True, True
with save_globals():
@ -443,7 +485,7 @@ class TestController(unittest.TestCase):
# tests if 404 is cached and used
def test_container_info_404(self):
def account_info(self, account):
def account_info(self, account, autocreate=False):
return True, True
with save_globals():