openstack/swift
Code Issues Proposed changes

merged with trunk

Browse Source
This commit is contained in:
John Dickinson
2011-06-15 14:05:28 -05:00
parent 5490c514fe e15b0732e5
commit 65d79d18b5
16 changed files with 132 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
CHANGELOG
View File

@@ -1,3 +1,19 @@
swift (1.4.1)
* st renamed to swift
* swauth was separated froms swift. It is now its own project and can be
found at https://github.com/gholt/swauth.
* tempauth middleware added as an extremely limited auth system for dev
work.
* Account and container listings now properly labeled UTF-8 (previously the
label was "utf8").
* Accounts are auto-created if an auth token is valid when the
account_autocreate proxy config parameter is set to true.
swift (1.4.0)
* swift-bench now cleans up containers it creates.

0
bin/st → bin/swift
View File

2
doc/source/development_saio.rst
View File

@@ -625,7 +625,7 @@ Setting up scripts for running Swift
#. `recreateaccounts`
#. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0``
#. Check that you can GET account: ``curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>``
#. Check that `st` works: `st -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat`
#. Check that `swift` works: `swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat`
#. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf`
#. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete
everything in the configured accounts.)

30
doc/source/howto_installmultinode.rst
View File

@@ -372,34 +372,34 @@ You run these commands from the Proxy node.
curl -k -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>
#. Check that ``st`` works (at this point, expect zero containers, zero objects, and zero bytes)::
#. Check that ``swift`` works (at this point, expect zero containers, zero objects, and zero bytes)::
st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat
swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat
#. Use ``st`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles'::
#. Use ``swift`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles'::
st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz
st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz
swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz
swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz
#. Use ``st`` to download all files from the 'myfiles' container::
#. Use ``swift`` to download all files from the 'myfiles' container::
st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles
swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles
#. Use ``st`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!::
#. Use ``swift`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!::
st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder
swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder
#. Use ``st`` to list your containers::
#. Use ``swift`` to list your containers::
st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list
swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list
#. Use ``st`` to list the contents of your 'builders' container::
#. Use ``swift`` to list the contents of your 'builders' container::
st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders
swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders
#. Use ``st`` to download all files from the 'builders' container::
#. Use ``swift`` to download all files from the 'builders' container::
st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders
swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders
.. _add-proxy-server:

20
doc/source/overview_large_objects.rst
View File

@@ -14,24 +14,24 @@ concatenated as a single object. This also offers much greater upload speed
with the possibility of parallel uploads of the segments.
----------------------------------
Using ``st`` for Segmented Objects
Using ``swift`` for Segmented Objects
----------------------------------
The quickest way to try out this feature is use the included ``st`` Swift Tool.
The quickest way to try out this feature is use the included ``swift`` Swift Tool.
You can use the ``-S`` option to specify the segment size to use when splitting
a large file. For example::
st upload test_container -S 1073741824 large_file
swift upload test_container -S 1073741824 large_file
This would split the large_file into 1G segments and begin uploading those
segments in parallel. Once all the segments have been uploaded, ``st`` will
segments in parallel. Once all the segments have been uploaded, ``swift`` will
then create the manifest file so the segments can be downloaded as one.
So now, the following ``st`` command would download the entire large object::
So now, the following ``swift`` command would download the entire large object::
st download test_container large_file
swift download test_container large_file
``st`` uses a strict convention for its segmented object support. In the above
``swift`` uses a strict convention for its segmented object support. In the above
example it will upload all the segments into a second container named
test_container_segments. These segments will have names like
large_file/1290206778.25/21474836480/00000000,
@@ -43,7 +43,7 @@ the segment name format of <name>/<timestamp>/<size>/<segment> is so that an
upload of a new file with the same name won't overwrite the contents of the
first until the last moment when the manifest file is updated.
``st`` will manage these segment files for you, deleting old segments on
``swift`` will manage these segment files for you, deleting old segments on
deletes and overwrites, etc. You can override this behavior with the
``--leave-segments`` option if desired; this is useful if you want to have
multiple versions of the same large object available.
@@ -53,14 +53,14 @@ Direct API
----------
You can also work with the segments and manifests directly with HTTP requests
instead of having ``st`` do that for you. You can just upload the segments like
instead of having ``swift`` do that for you. You can just upload the segments like
you would any other object and the manifest is just a zero-byte file with an
extra ``X-Object-Manifest`` header.
All the object segments need to be in the same container, have a common object
name prefix, and their names sort in the order they should be concatenated.
They don't have to be in the same container as the manifest file will be, which
is useful to keep container listings clean as explained above with ``st``.
is useful to keep container listings clean as explained above with ``swift``.
The manifest file is simply a zero-byte file with the extra
``X-Object-Manifest: <container>/<prefix>`` header, where ``<container>`` is

2
setup.py
View File

@@ -76,7 +76,7 @@ setup(
],
install_requires=[], # removed for better compat
scripts=[
'bin/st', 'bin/swift-account-auditor',
'bin/swift', 'bin/swift-account-auditor',
'bin/swift-account-audit', 'bin/swift-account-reaper',
'bin/swift-account-replicator', 'bin/swift-account-server',
'bin/swift-container-auditor',

2
swift/__init__.py
View File

@@ -14,7 +14,7 @@ class Version(object):
return '%s-dev' % (self.canonical_version,)
_version = Version('1.4.1', False)
_version = Version('1.4.2', False)
__version__ = _version.pretty_version
__canonical_version__ = _version.canonical_version

2
swift/account/server.py
View File

@@ -244,7 +244,7 @@ class AccountController(object):
account_list = '\n'.join(r[0] for r in account_list) + '\n'
ret = Response(body=account_list, request=req, headers=resp_headers)
ret.content_type = out_content_type
ret.charset = 'utf8'
ret.charset = 'utf-8'
return ret
def REPLICATE(self, req):

12
swift/common/middleware/staticweb.py
View File

@@ -74,36 +74,36 @@ the .../listing.css style sheet. If you "view source" in your browser on a
listing page, you will see the well defined document structure that can be
styled.
Example usage of this middleware via ``st``:
Example usage of this middleware via ``swift``:
Make the container publicly readable::
st post -r '.r:*' container
swift post -r '.r:*' container
You should be able to get objects directly, but no index.html resolution or
listings.
Set an index file directive::
st post -m 'web-index:index.html' container
swift post -m 'web-index:index.html' container
You should be able to hit paths that have an index.html without needing to
type the index.html part.
Turn on listings::
st post -m 'web-listings: true' container
swift post -m 'web-listings: true' container
Now you should see object listings for paths and pseudo paths that have no
index.html.
Enable a custom listings style sheet::
st post -m 'web-listings-css:listings.css' container
swift post -m 'web-listings-css:listings.css' container
Set an error file::
st post -m 'web-error:error.html' container
swift post -m 'web-error:error.html' container
Now 401's should load 401error.html, 404's should load 404error.html, etc.
"""

2
swift/container/server.py
View File

@@ -334,7 +334,7 @@ class ContainerController(object):
container_list = '\n'.join(r[0] for r in container_list) + '\n'
ret = Response(body=container_list, request=req, headers=resp_headers)
ret.content_type = out_content_type
ret.charset = 'utf8'
ret.charset = 'utf-8'
return ret
def REPLICATE(self, req):

36
swift/proxy/server.py
View File

@@ -1319,8 +1319,26 @@ class AccountController(Controller):
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
return self.GETorHEAD_base(req, _('Account'), partition, nodes,
resp = self.GETorHEAD_base(req, _('Account'), partition, nodes,
req.path_info.rstrip('/'), self.app.account_ring.replica_count)
if resp.status_int == 404 and self.app.account_autocreate:
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id}
resp = self.make_requests(
Request.blank('/v1/' + self.account_name),
self.app.account_ring, partition, 'PUT',
'/' + self.account_name, [headers] * len(nodes))
if resp.status_int // 100 != 2:
raise Exception('Could not autocreate account %r' %
self.account_name)
resp = self.GETorHEAD_base(req, _('Account'), partition, nodes,
req.path_info.rstrip('/'), self.app.account_ring.replica_count)
return resp
@public
def PUT(self, req):
@@ -1360,9 +1378,23 @@ class AccountController(Controller):
if value[0].lower().startswith('x-account-meta-'))
if self.app.memcache:
self.app.memcache.delete('account%s' % req.path_info.rstrip('/'))
return self.make_requests(req, self.app.account_ring,
resp = self.make_requests(req, self.app.account_ring,
account_partition, 'POST', req.path_info,
[headers] * len(accounts))
if resp.status_int == 404 and self.app.account_autocreate:
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
resp = self.make_requests(
Request.blank('/v1/' + self.account_name),
self.app.account_ring, account_partition, 'PUT',
'/' + self.account_name, [headers] * len(accounts))
if resp.status_int // 100 != 2:
raise Exception('Could not autocreate account %r' %
self.account_name)
return resp
@public
def DELETE(self, req):

8
test/functional/tests.py
View File

@@ -227,10 +227,10 @@ class TestAccount(Base):
headers = dict(self.env.conn.response.getheaders())
if format == 'json':
self.assertEquals(headers['content-type'],
'application/json; charset=utf8')
'application/json; charset=utf-8')
elif format == 'xml':
self.assertEquals(headers['content-type'],
'application/xml; charset=utf8')
'application/xml; charset=utf-8')
def testListingLimit(self):
limit = 10000
@@ -1355,10 +1355,10 @@ class TestFile(Base):
headers = dict(self.env.conn.response.getheaders())
if format == 'json':
self.assertEquals(headers['content-type'],
'application/json; charset=utf8')
'application/json; charset=utf-8')
elif format == 'xml':
self.assertEquals(headers['content-type'],
'application/xml; charset=utf8')
'application/xml; charset=utf-8')
lm_diff = max([f['last_modified'] for f in files]) - \
min([f['last_modified'] for f in files])

3
test/unit/account/test_server.py
View File

@@ -388,6 +388,7 @@ class TestAccountController(unittest.TestCase):
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.body.strip().split('\n'), ['c1', 'c2'])
self.assertEquals(resp.content_type, 'text/plain')
self.assertEquals(resp.charset, 'utf-8')
def test_GET_with_containers_json(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@@ -436,6 +437,7 @@ class TestAccountController(unittest.TestCase):
[{'count': 1, 'bytes': 2, 'name': 'c1'},
{'count': 3, 'bytes': 4, 'name': 'c2'}])
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(resp.charset, 'utf-8')
def test_GET_with_containers_xml(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@@ -529,6 +531,7 @@ class TestAccountController(unittest.TestCase):
self.assertEquals(node.firstChild.nodeValue, '3')
node = [n for n in container if n.nodeName == 'bytes'][0]
self.assertEquals(node.firstChild.nodeValue, '4')
self.assertEquals(resp.charset, 'utf-8')
def test_GET_limit_marker_plain(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',

12
test/unit/common/middleware/test_staticweb.py
View File

@@ -187,7 +187,7 @@ class FakeApp(object):
headers.update({'X-Container-Object-Count': '11',
'X-Container-Bytes-Used': '73741',
'X-Container-Read': '.r:*',
'Content-Type': 'application/json; charset=utf8'})
'Content-Type': 'application/json; charset=utf-8'})
body = '''
[{"name":"subdir/1.txt",
"hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20,
@@ -204,14 +204,14 @@ class FakeApp(object):
headers.update({'X-Container-Object-Count': '11',
'X-Container-Bytes-Used': '73741',
'X-Container-Read': '.r:*',
'Content-Type': 'application/json; charset=utf8'})
'Content-Type': 'application/json; charset=utf-8'})
body = '[]'
elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \
'limit=1&format=json&delimiter=/&limit=1&prefix=subdirz/':
headers.update({'X-Container-Object-Count': '11',
'X-Container-Bytes-Used': '73741',
'X-Container-Read': '.r:*',
'Content-Type': 'application/json; charset=utf8'})
'Content-Type': 'application/json; charset=utf-8'})
body = '''
[{"name":"subdirz/1.txt",
"hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20,
@@ -224,7 +224,7 @@ class FakeApp(object):
'X-Container-Bytes-Used': '73741',
'X-Container-Read': '.r:*',
'X-Container-Web-Listings': 't',
'Content-Type': 'application/json; charset=utf8'})
'Content-Type': 'application/json; charset=utf-8'})
body = '''
[{"name":"subdir/1.txt",
"hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20,
@@ -236,7 +236,7 @@ class FakeApp(object):
elif 'format=json' in env['QUERY_STRING']:
headers.update({'X-Container-Object-Count': '11',
'X-Container-Bytes-Used': '73741',
'Content-Type': 'application/json; charset=utf8'})
'Content-Type': 'application/json; charset=utf-8'})
body = '''
[{"name":"401error.html",
"hash":"893f8d80692a4d3875b45be8f152ad18", "bytes":110,
@@ -283,7 +283,7 @@ class FakeApp(object):
else:
headers.update({'X-Container-Object-Count': '11',
'X-Container-Bytes-Used': '73741',
'Content-Type': 'text/plain; charset=utf8'})
'Content-Type': 'text/plain; charset=utf-8'})
body = '\n'.join(['401error.html', '404error.html', 'index.html',
'listing.css', 'one.txt', 'subdir/1.txt',
'subdir/2.txt', 'subdir/omgomg.txt', 'subdir2',

3
test/unit/container/test_server.py
View File

@@ -514,6 +514,7 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(eval(resp.body), json_body)
self.assertEquals(resp.charset, 'utf-8')
for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9',
'*/*;q=0.9,application/json;q=1.0', 'application/*'):
@@ -552,6 +553,7 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'text/plain')
self.assertEquals(resp.body, plain_body)
self.assertEquals(resp.charset, 'utf-8')
for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9',
'*/*;q=0.9,application/xml;q=0.8', '*/*',
@@ -609,6 +611,7 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'application/xml')
self.assertEquals(resp.body, xml_body)
self.assertEquals(resp.charset, 'utf-8')
for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9',
'*/*;q=0.9,application/xml;q=1.0', 'application/xml,text/xml'):

30
test/unit/proxy/test_server.py
View File

@@ -3171,6 +3171,16 @@ class TestAccountController(unittest.TestCase):
self.app.memcache = FakeMemcacheReturnsNone()
self.assert_status_map(controller.GET, (404, 404, 404), 404)
def test_GET_autocreate(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
self.app.memcache = FakeMemcacheReturnsNone()
self.assert_status_map(controller.GET,
(404, 404, 404, 201, 201, 201, 204), 404)
controller.app.account_autocreate = True
self.assert_status_map(controller.GET,
(404, 404, 404, 201, 201, 201, 204), 204)
def test_HEAD(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
@@ -3189,6 +3199,26 @@ class TestAccountController(unittest.TestCase):
self.assert_status_map(controller.HEAD, (404, 503, 503), 503)
self.assert_status_map(controller.HEAD, (404, 204, 503), 204)
def test_HEAD_autocreate(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
self.app.memcache = FakeMemcacheReturnsNone()
self.assert_status_map(controller.HEAD,
(404, 404, 404, 201, 201, 201, 204), 404)
controller.app.account_autocreate = True
self.assert_status_map(controller.HEAD,
(404, 404, 404, 201, 201, 201, 204), 204)
def test_POST_autocreate(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
self.app.memcache = FakeMemcacheReturnsNone()
self.assert_status_map(controller.POST,
(404, 404, 404, 201, 201, 201), 404)
controller.app.account_autocreate = True
self.assert_status_map(controller.POST,
(404, 404, 404, 201, 201, 201), 201)
def test_connection_refused(self):
self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values():