diff --git a/CHANGELOG b/CHANGELOG index ff74a19d57..c2e56f2edd 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,19 @@ +swift (1.4.1) + + * st renamed to swift + + * swauth was separated froms swift. It is now its own project and can be + found at https://github.com/gholt/swauth. + + * tempauth middleware added as an extremely limited auth system for dev + work. + + * Account and container listings now properly labeled UTF-8 (previously the + label was "utf8"). + + * Accounts are auto-created if an auth token is valid when the + account_autocreate proxy config parameter is set to true. + swift (1.4.0) * swift-bench now cleans up containers it creates. diff --git a/bin/st b/bin/swift similarity index 100% rename from bin/st rename to bin/swift diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index bbce6fdeb0..a6dc0060b6 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -625,7 +625,7 @@ Setting up scripts for running Swift #. `recreateaccounts` #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0`` #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: ' `` - #. Check that `st` works: `st -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat` + #. Check that `swift` works: `swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat` #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf` #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete everything in the configured accounts.) diff --git a/doc/source/howto_installmultinode.rst b/doc/source/howto_installmultinode.rst index 6f10e30757..abe7647db8 100644 --- a/doc/source/howto_installmultinode.rst +++ b/doc/source/howto_installmultinode.rst @@ -372,34 +372,34 @@ You run these commands from the Proxy node. curl -k -v -H 'X-Auth-Token: ' -#. Check that ``st`` works (at this point, expect zero containers, zero objects, and zero bytes):: +#. Check that ``swift`` works (at this point, expect zero containers, zero objects, and zero bytes):: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat -#. Use ``st`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles':: +#. Use ``swift`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles':: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz -#. Use ``st`` to download all files from the 'myfiles' container:: +#. Use ``swift`` to download all files from the 'myfiles' container:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles -#. Use ``st`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!:: +#. Use ``swift`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder -#. Use ``st`` to list your containers:: +#. Use ``swift`` to list your containers:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list -#. Use ``st`` to list the contents of your 'builders' container:: +#. Use ``swift`` to list the contents of your 'builders' container:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders -#. Use ``st`` to download all files from the 'builders' container:: +#. Use ``swift`` to download all files from the 'builders' container:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders .. _add-proxy-server: diff --git a/doc/source/overview_large_objects.rst b/doc/source/overview_large_objects.rst index 01f4990732..923afb872a 100644 --- a/doc/source/overview_large_objects.rst +++ b/doc/source/overview_large_objects.rst @@ -14,24 +14,24 @@ concatenated as a single object. This also offers much greater upload speed with the possibility of parallel uploads of the segments. ---------------------------------- -Using ``st`` for Segmented Objects +Using ``swift`` for Segmented Objects ---------------------------------- -The quickest way to try out this feature is use the included ``st`` Swift Tool. +The quickest way to try out this feature is use the included ``swift`` Swift Tool. You can use the ``-S`` option to specify the segment size to use when splitting a large file. For example:: - st upload test_container -S 1073741824 large_file + swift upload test_container -S 1073741824 large_file This would split the large_file into 1G segments and begin uploading those -segments in parallel. Once all the segments have been uploaded, ``st`` will +segments in parallel. Once all the segments have been uploaded, ``swift`` will then create the manifest file so the segments can be downloaded as one. -So now, the following ``st`` command would download the entire large object:: +So now, the following ``swift`` command would download the entire large object:: - st download test_container large_file + swift download test_container large_file -``st`` uses a strict convention for its segmented object support. In the above +``swift`` uses a strict convention for its segmented object support. In the above example it will upload all the segments into a second container named test_container_segments. These segments will have names like large_file/1290206778.25/21474836480/00000000, @@ -43,7 +43,7 @@ the segment name format of /// is so that an upload of a new file with the same name won't overwrite the contents of the first until the last moment when the manifest file is updated. -``st`` will manage these segment files for you, deleting old segments on +``swift`` will manage these segment files for you, deleting old segments on deletes and overwrites, etc. You can override this behavior with the ``--leave-segments`` option if desired; this is useful if you want to have multiple versions of the same large object available. @@ -53,14 +53,14 @@ Direct API ---------- You can also work with the segments and manifests directly with HTTP requests -instead of having ``st`` do that for you. You can just upload the segments like +instead of having ``swift`` do that for you. You can just upload the segments like you would any other object and the manifest is just a zero-byte file with an extra ``X-Object-Manifest`` header. All the object segments need to be in the same container, have a common object name prefix, and their names sort in the order they should be concatenated. They don't have to be in the same container as the manifest file will be, which -is useful to keep container listings clean as explained above with ``st``. +is useful to keep container listings clean as explained above with ``swift``. The manifest file is simply a zero-byte file with the extra ``X-Object-Manifest: /`` header, where ```` is diff --git a/setup.py b/setup.py index 641451acb4..1cf325f13e 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( ], install_requires=[], # removed for better compat scripts=[ - 'bin/st', 'bin/swift-account-auditor', + 'bin/swift', 'bin/swift-account-auditor', 'bin/swift-account-audit', 'bin/swift-account-reaper', 'bin/swift-account-replicator', 'bin/swift-account-server', 'bin/swift-container-auditor', diff --git a/swift/__init__.py b/swift/__init__.py index c26a554fbd..36791d4794 100644 --- a/swift/__init__.py +++ b/swift/__init__.py @@ -14,7 +14,7 @@ class Version(object): return '%s-dev' % (self.canonical_version,) -_version = Version('1.4.1', False) +_version = Version('1.4.2', False) __version__ = _version.pretty_version __canonical_version__ = _version.canonical_version diff --git a/swift/account/server.py b/swift/account/server.py index d51f9f27a6..fd839a4909 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -244,7 +244,7 @@ class AccountController(object): account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = Response(body=account_list, request=req, headers=resp_headers) ret.content_type = out_content_type - ret.charset = 'utf8' + ret.charset = 'utf-8' return ret def REPLICATE(self, req): diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 81225a90ea..f04d4a7d55 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -74,36 +74,36 @@ the .../listing.css style sheet. If you "view source" in your browser on a listing page, you will see the well defined document structure that can be styled. -Example usage of this middleware via ``st``: +Example usage of this middleware via ``swift``: Make the container publicly readable:: - st post -r '.r:*' container + swift post -r '.r:*' container You should be able to get objects directly, but no index.html resolution or listings. Set an index file directive:: - st post -m 'web-index:index.html' container + swift post -m 'web-index:index.html' container You should be able to hit paths that have an index.html without needing to type the index.html part. Turn on listings:: - st post -m 'web-listings: true' container + swift post -m 'web-listings: true' container Now you should see object listings for paths and pseudo paths that have no index.html. Enable a custom listings style sheet:: - st post -m 'web-listings-css:listings.css' container + swift post -m 'web-listings-css:listings.css' container Set an error file:: - st post -m 'web-error:error.html' container + swift post -m 'web-error:error.html' container Now 401's should load 401error.html, 404's should load 404error.html, etc. """ diff --git a/swift/container/server.py b/swift/container/server.py index dfdc884a6d..bc3856d18e 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -334,7 +334,7 @@ class ContainerController(object): container_list = '\n'.join(r[0] for r in container_list) + '\n' ret = Response(body=container_list, request=req, headers=resp_headers) ret.content_type = out_content_type - ret.charset = 'utf8' + ret.charset = 'utf-8' return ret def REPLICATE(self, req): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 8a451200cf..581eef21c3 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1319,8 +1319,26 @@ class AccountController(Controller): def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" partition, nodes = self.app.account_ring.get_nodes(self.account_name) - return self.GETorHEAD_base(req, _('Account'), partition, nodes, + resp = self.GETorHEAD_base(req, _('Account'), partition, nodes, req.path_info.rstrip('/'), self.app.account_ring.replica_count) + if resp.status_int == 404 and self.app.account_autocreate: + if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: + resp = HTTPBadRequest(request=req) + resp.body = 'Account name length of %d longer than %d' % \ + (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) + return resp + headers = {'X-Timestamp': normalize_timestamp(time.time()), + 'X-Trans-Id': self.trans_id} + resp = self.make_requests( + Request.blank('/v1/' + self.account_name), + self.app.account_ring, partition, 'PUT', + '/' + self.account_name, [headers] * len(nodes)) + if resp.status_int // 100 != 2: + raise Exception('Could not autocreate account %r' % + self.account_name) + resp = self.GETorHEAD_base(req, _('Account'), partition, nodes, + req.path_info.rstrip('/'), self.app.account_ring.replica_count) + return resp @public def PUT(self, req): @@ -1360,9 +1378,23 @@ class AccountController(Controller): if value[0].lower().startswith('x-account-meta-')) if self.app.memcache: self.app.memcache.delete('account%s' % req.path_info.rstrip('/')) - return self.make_requests(req, self.app.account_ring, + resp = self.make_requests(req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) + if resp.status_int == 404 and self.app.account_autocreate: + if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: + resp = HTTPBadRequest(request=req) + resp.body = 'Account name length of %d longer than %d' % \ + (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) + return resp + resp = self.make_requests( + Request.blank('/v1/' + self.account_name), + self.app.account_ring, account_partition, 'PUT', + '/' + self.account_name, [headers] * len(accounts)) + if resp.status_int // 100 != 2: + raise Exception('Could not autocreate account %r' % + self.account_name) + return resp @public def DELETE(self, req): diff --git a/test/functional/tests.py b/test/functional/tests.py index 59dcf38960..8c513490d2 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -227,10 +227,10 @@ class TestAccount(Base): headers = dict(self.env.conn.response.getheaders()) if format == 'json': self.assertEquals(headers['content-type'], - 'application/json; charset=utf8') + 'application/json; charset=utf-8') elif format == 'xml': self.assertEquals(headers['content-type'], - 'application/xml; charset=utf8') + 'application/xml; charset=utf-8') def testListingLimit(self): limit = 10000 @@ -1355,10 +1355,10 @@ class TestFile(Base): headers = dict(self.env.conn.response.getheaders()) if format == 'json': self.assertEquals(headers['content-type'], - 'application/json; charset=utf8') + 'application/json; charset=utf-8') elif format == 'xml': self.assertEquals(headers['content-type'], - 'application/xml; charset=utf8') + 'application/xml; charset=utf-8') lm_diff = max([f['last_modified'] for f in files]) - \ min([f['last_modified'] for f in files]) diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 16800ca165..238b7f3d18 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -388,6 +388,7 @@ class TestAccountController(unittest.TestCase): self.assertEquals(resp.status_int, 200) self.assertEquals(resp.body.strip().split('\n'), ['c1', 'c2']) self.assertEquals(resp.content_type, 'text/plain') + self.assertEquals(resp.charset, 'utf-8') def test_GET_with_containers_json(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', @@ -436,6 +437,7 @@ class TestAccountController(unittest.TestCase): [{'count': 1, 'bytes': 2, 'name': 'c1'}, {'count': 3, 'bytes': 4, 'name': 'c2'}]) self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(resp.charset, 'utf-8') def test_GET_with_containers_xml(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', @@ -529,6 +531,7 @@ class TestAccountController(unittest.TestCase): self.assertEquals(node.firstChild.nodeValue, '3') node = [n for n in container if n.nodeName == 'bytes'][0] self.assertEquals(node.firstChild.nodeValue, '4') + self.assertEquals(resp.charset, 'utf-8') def test_GET_limit_marker_plain(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', diff --git a/test/unit/common/middleware/test_staticweb.py b/test/unit/common/middleware/test_staticweb.py index 7b5385da71..55ff3959fa 100644 --- a/test/unit/common/middleware/test_staticweb.py +++ b/test/unit/common/middleware/test_staticweb.py @@ -187,7 +187,7 @@ class FakeApp(object): headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdir/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, @@ -204,14 +204,14 @@ class FakeApp(object): headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = '[]' elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \ 'limit=1&format=json&delimiter=/&limit=1&prefix=subdirz/': headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdirz/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, @@ -224,7 +224,7 @@ class FakeApp(object): 'X-Container-Bytes-Used': '73741', 'X-Container-Read': '.r:*', 'X-Container-Web-Listings': 't', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdir/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, @@ -236,7 +236,7 @@ class FakeApp(object): elif 'format=json' in env['QUERY_STRING']: headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"401error.html", "hash":"893f8d80692a4d3875b45be8f152ad18", "bytes":110, @@ -283,7 +283,7 @@ class FakeApp(object): else: headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', - 'Content-Type': 'text/plain; charset=utf8'}) + 'Content-Type': 'text/plain; charset=utf-8'}) body = '\n'.join(['401error.html', '404error.html', 'index.html', 'listing.css', 'one.txt', 'subdir/1.txt', 'subdir/2.txt', 'subdir/omgomg.txt', 'subdir2', diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 5127fe093c..117a0dccd5 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -514,6 +514,7 @@ class TestContainerController(unittest.TestCase): resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(eval(resp.body), json_body) + self.assertEquals(resp.charset, 'utf-8') for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9', '*/*;q=0.9,application/json;q=1.0', 'application/*'): @@ -552,6 +553,7 @@ class TestContainerController(unittest.TestCase): resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'text/plain') self.assertEquals(resp.body, plain_body) + self.assertEquals(resp.charset, 'utf-8') for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9', '*/*;q=0.9,application/xml;q=0.8', '*/*', @@ -609,6 +611,7 @@ class TestContainerController(unittest.TestCase): resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'application/xml') self.assertEquals(resp.body, xml_body) + self.assertEquals(resp.charset, 'utf-8') for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9', '*/*;q=0.9,application/xml;q=1.0', 'application/xml,text/xml'): diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index eb09ad34dc..b525ea9037 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -3171,6 +3171,16 @@ class TestAccountController(unittest.TestCase): self.app.memcache = FakeMemcacheReturnsNone() self.assert_status_map(controller.GET, (404, 404, 404), 404) + def test_GET_autocreate(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.app.memcache = FakeMemcacheReturnsNone() + self.assert_status_map(controller.GET, + (404, 404, 404, 201, 201, 201, 204), 404) + controller.app.account_autocreate = True + self.assert_status_map(controller.GET, + (404, 404, 404, 201, 201, 201, 204), 204) + def test_HEAD(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'account') @@ -3189,6 +3199,26 @@ class TestAccountController(unittest.TestCase): self.assert_status_map(controller.HEAD, (404, 503, 503), 503) self.assert_status_map(controller.HEAD, (404, 204, 503), 204) + def test_HEAD_autocreate(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.app.memcache = FakeMemcacheReturnsNone() + self.assert_status_map(controller.HEAD, + (404, 404, 404, 201, 201, 201, 204), 404) + controller.app.account_autocreate = True + self.assert_status_map(controller.HEAD, + (404, 404, 404, 201, 201, 201, 204), 204) + + def test_POST_autocreate(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.app.memcache = FakeMemcacheReturnsNone() + self.assert_status_map(controller.POST, + (404, 404, 404, 201, 201, 201), 404) + controller.app.account_autocreate = True + self.assert_status_map(controller.POST, + (404, 404, 404, 201, 201, 201), 201) + def test_connection_refused(self): self.app.account_ring.get_nodes('account') for dev in self.app.account_ring.devs.values():