commit 001407b969bc12d48bd7f10960f84f519bb19111 Author: Chuck Thier Date: Mon Jul 12 17:03:45 2010 -0500 Initial commit of Swift code diff --git a/.bzrignore b/.bzrignore new file mode 100644 index 0000000000..7936e61fc5 --- /dev/null +++ b/.bzrignore @@ -0,0 +1,2 @@ +*.py[co] +*.sw? diff --git a/.functests b/.functests new file mode 100755 index 0000000000..17122d75ce --- /dev/null +++ b/.functests @@ -0,0 +1,3 @@ +#!/bin/bash + +python test/functional/tests.py diff --git a/.probetests b/.probetests new file mode 100755 index 0000000000..7514693868 --- /dev/null +++ b/.probetests @@ -0,0 +1,3 @@ +#!/bin/bash + +nosetests test/probe --exe diff --git a/.unittests b/.unittests new file mode 100755 index 0000000000..c115578a87 --- /dev/null +++ b/.unittests @@ -0,0 +1,4 @@ +#!/bin/bash + +nosetests test/unit --exe --with-coverage --cover-package swift --cover-erase +rm -f .coverage diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000000..43c5aa35ea --- /dev/null +++ b/AUTHORS @@ -0,0 +1,20 @@ +Maintainer +---------- +OpenStack, LLC. +IRC: #openstack + +Original Authors +---------------- +Michael Barton +John Dickinson +Greg Holt +Greg Lange +Jay Payne +Will Reese +Chuck Thier + +Contributors +------------ +Chmouel Boudjnah +Ed Leafe +Conrad Weidenkeller diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..75b52484ea --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README b/README new file mode 100644 index 0000000000..ac8681ae07 --- /dev/null +++ b/README @@ -0,0 +1,17 @@ +Swift +----- + +A distributed object store that was originally developed as the basis for +Rackspace's Cloud Files. + +To build documentation run `make html` in the /doc folder, and then browse to +/doc/build/html/index.html. + +The best place to get started is the "SAIO - Swift All In One", which will walk +you through setting up a development cluster of Swift in a VM. + +For more information, vist us at http://launchpad.net/swift, or come hang out +on our IRC channel, #openstack on freenode. + +-- +Swift Development Team diff --git a/bin/st.py b/bin/st.py new file mode 100755 index 0000000000..ede5c985de --- /dev/null +++ b/bin/st.py @@ -0,0 +1,1276 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + # Try to use installed swift.common.client... + from swift.common.client import get_auth, ClientException, Connection +except: + # But if not installed, use an included copy. + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # Inclusion of swift.common.client + + """ + Cloud Files client library used internally + """ + import socket + from cStringIO import StringIO + from httplib import HTTPConnection, HTTPException, HTTPSConnection + from re import compile, DOTALL + from tokenize import generate_tokens, STRING, NAME, OP + from urllib import quote as _quote, unquote + from urlparse import urlparse, urlunparse + + try: + from eventlet import sleep + except: + from time import sleep + + + def quote(value, safe='/'): + """ + Patched version of urllib.quote that encodes utf8 strings before quoting + """ + if isinstance(value, unicode): + value = value.encode('utf8') + return _quote(value, safe) + + + # look for a real json parser first + try: + # simplejson is popular and pretty good + from simplejson import loads as json_loads + except ImportError: + try: + # 2.6 will have a json module in the stdlib + from json import loads as json_loads + except ImportError: + # fall back on local parser otherwise + comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL) + + def json_loads(string): + ''' + Fairly competent json parser exploiting the python tokenizer and + eval(). -- From python-cloudfiles + + _loads(serialized_json) -> object + ''' + try: + res = [] + consts = {'true': True, 'false': False, 'null': None} + string = '(' + comments.sub('', string) + ')' + for type, val, _, _, _ in \ + generate_tokens(StringIO(string).readline): + if (type == OP and val not in '[]{}:,()-') or \ + (type == NAME and val not in consts): + raise AttributeError() + elif type == STRING: + res.append('u') + res.append(val.replace('\\/', '/')) + else: + res.append(val) + return eval(''.join(res), {}, consts) + except: + raise AttributeError() + + + class ClientException(Exception): + + def __init__(self, msg, http_scheme='', http_host='', http_port='', + http_path='', http_query='', http_status=0, http_reason='', + http_device=''): + Exception.__init__(self, msg) + self.msg = msg + self.http_scheme = http_scheme + self.http_host = http_host + self.http_port = http_port + self.http_path = http_path + self.http_query = http_query + self.http_status = http_status + self.http_reason = http_reason + self.http_device = http_device + + def __str__(self): + a = self.msg + b = '' + if self.http_scheme: + b += '%s://' % self.http_scheme + if self.http_host: + b += self.http_host + if self.http_port: + b += ':%s' % self.http_port + if self.http_path: + b += self.http_path + if self.http_query: + b += '?%s' % self.http_query + if self.http_status: + if b: + b = '%s %s' % (b, self.http_status) + else: + b = str(self.http_status) + if self.http_reason: + if b: + b = '%s %s' % (b, self.http_reason) + else: + b = '- %s' % self.http_reason + if self.http_device: + if b: + b = '%s: device %s' % (b, self.http_device) + else: + b = 'device %s' % self.http_device + return b and '%s: %s' % (a, b) or a + + + def http_connection(url): + """ + Make an HTTPConnection or HTTPSConnection + + :param url: url to connect to + :returns: tuple of (parsed url, connection object) + :raises ClientException: Unable to handle protocol scheme + """ + parsed = urlparse(url) + if parsed.scheme == 'http': + conn = HTTPConnection(parsed.netloc) + elif parsed.scheme == 'https': + conn = HTTPSConnection(parsed.netloc) + else: + raise ClientException('Cannot handle protocol scheme %s for url %s' % + (parsed.scheme, repr(url))) + return parsed, conn + + + def get_auth(url, user, key, snet=False): + """ + Get authentication credentials + + :param url: authentication URL + :param user: user to auth as + :param key: key or passowrd for auth + :param snet: use SERVICENET internal network default is False + :returns: tuple of (storage URL, storage token, auth token) + :raises ClientException: HTTP GET request to auth URL failed + """ + parsed, conn = http_connection(url) + conn.request('GET', parsed.path, '', + {'X-Auth-User': user, 'X-Auth-Key': key}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Auth GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_status=resp.status, + http_reason=resp.reason) + url = resp.getheader('x-storage-url') + if snet: + parsed = list(urlparse(url)) + # Second item in the list is the netloc + parsed[1] = 'snet-' + parsed[1] + url = urlunparse(parsed) + return url, resp.getheader('x-storage-token', + resp.getheader('x-auth-token')) + + + def get_account(url, token, marker=None, limit=None, prefix=None, + http_conn=None, full_listing=False): + """ + Get a listing of containers for the account. + + :param url: storage URL + :param token: auth token + :param marker: marker query + :param limit: limit query + :param prefix: prefix query + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param full_listing: if True, return a full listing, else returns a max + of 10000 listings + :returns: a list of accounts + :raises ClientException: HTTP GET request failed + """ + if not http_conn: + http_conn = http_connection(url) + if full_listing: + rv = [] + listing = get_account(url, token, marker, limit, prefix, http_conn) + while listing: + rv.extend(listing) + marker = listing[-1]['name'] + listing = get_account(url, token, marker, limit, prefix, http_conn) + return rv + parsed, conn = http_conn + qs = 'format=json' + if marker: + qs += '&marker=%s' % quote(marker) + if limit: + qs += '&limit=%d' % limit + if prefix: + qs += '&prefix=%s' % quote(prefix) + conn.request('GET', '%s?%s' % (parsed.path, qs), '', + {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Account GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_query=qs, http_status=resp.status, + http_reason=resp.reason) + if resp.status == 204: + resp.read() + return [] + return json_loads(resp.read()) + + + def head_account(url, token, http_conn=None): + """ + Get account stats. + + :param url: storage URL + :param token: auth token + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a tuple of (container count, object count, bytes used) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_status=resp.status, + http_reason=resp.reason) + return int(resp.getheader('x-account-container-count', 0)), \ + int(resp.getheader('x-account-object-count', 0)), \ + int(resp.getheader('x-account-bytes-used', 0)) + + + def get_container(url, token, container, marker=None, limit=None, + prefix=None, delimiter=None, http_conn=None, + full_listing=False): + """ + Get a listing of objects for the container. + + :param url: storage URL + :param token: auth token + :param container: container name to get a listing for + :param marker: marker query + :param limit: limit query + :param prefix: prefix query + :param delimeter: string to delimit the queries on + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param full_listing: if True, return a full listing, else returns a max + of 10000 listings + :returns: a list of objects + :raises ClientException: HTTP GET request failed + """ + if not http_conn: + http_conn = http_connection(url) + if full_listing: + rv = [] + listing = get_container(url, token, container, marker, limit, prefix, + delimiter, http_conn) + while listing: + rv.extend(listing) + if not delimiter: + marker = listing[-1]['name'] + else: + marker = listing[-1].get('name', listing[-1].get('subdir')) + listing = get_container(url, token, container, marker, limit, + prefix, delimiter, http_conn) + return rv + parsed, conn = http_conn + path = '%s/%s' % (parsed.path, quote(container)) + qs = 'format=json' + if marker: + qs += '&marker=%s' % quote(marker) + if limit: + qs += '&limit=%d' % limit + if prefix: + qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) + conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Container GET failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_query=qs, + http_status=resp.status, http_reason=resp.reason) + if resp.status == 204: + resp.read() + return [] + return json_loads(resp.read()) + + + def head_container(url, token, container, http_conn=None): + """ + Get container stats. + + :param url: storage URL + :param token: auth token + :param container: container name to get stats for + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a tuple of (object count, bytes used) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + conn.request('HEAD', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container HEAD failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + return int(resp.getheader('x-container-object-count', 0)), \ + int(resp.getheader('x-container-bytes-used', 0)) + + + def put_container(url, token, container, http_conn=None): + """ + Create a container + + :param url: storage URL + :param token: auth token + :param container: container name to create + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP PUT request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + conn.request('PUT', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container PUT failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + + def delete_container(url, token, container, http_conn=None): + """ + Delete a container + + :param url: storage URL + :param token: auth token + :param container: container name to delete + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP DELETE request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + conn.request('DELETE', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container DELETE failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + + def get_object(url, token, container, name, http_conn=None, + resp_chunk_size=None): + """ + Get an object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to get + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param resp_chunk_size: if defined, chunk size of data to read + :returns: a list of objects + :raises ClientException: HTTP GET request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('GET', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Object GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + metadata = {} + for key, value in resp.getheaders(): + if key.lower().startswith('x-object-meta-'): + metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) + if resp_chunk_size: + + def _object_body(): + buf = resp.read(resp_chunk_size) + while buf: + yield buf + buf = resp.read(resp_chunk_size) + object_body = _object_body() + else: + object_body = resp.read() + return resp.getheader('content-type'), \ + int(resp.getheader('content-length', 0)), \ + resp.getheader('last-modified'), \ + resp.getheader('etag').strip('"'), \ + metadata, \ + object_body + + + def head_object(url, token, container, name, http_conn=None): + """ + Get object info + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to get info for + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a tuple of (content type, content length, last modfied, etag, + dictionary of metadata) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('HEAD', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + metadata = {} + for key, value in resp.getheaders(): + if key.lower().startswith('x-object-meta-'): + metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) + return resp.getheader('content-type'), \ + int(resp.getheader('content-length', 0)), \ + resp.getheader('last-modified'), \ + resp.getheader('etag').strip('"'), \ + metadata + + + def put_object(url, token, container, name, contents, metadata={}, + content_length=None, etag=None, chunk_size=65536, + content_type=None, http_conn=None): + """ + Put an object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to put + :param contents: file like object to read object data from + :param metadata: dictionary of object metadata + :param content_length: value to send as content-length header + :param etag: etag of contents + :param chunk_size: chunk size of data to write + :param content_type: value to send as content-type header + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: etag from server response + :raises ClientException: HTTP PUT request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + headers = {'X-Auth-Token': token} + for key, value in metadata.iteritems(): + headers['X-Object-Meta-%s' % quote(key)] = quote(value) + if etag: + headers['ETag'] = etag.strip('"') + if content_length is not None: + headers['Content-Length'] = str(content_length) + if content_type is not None: + headers['Content-Type'] = content_type + if not contents: + headers['Content-Length'] = '0' + if hasattr(contents, 'read'): + conn.putrequest('PUT', path) + for header, value in headers.iteritems(): + conn.putheader(header, value) + if not content_length: + conn.putheader('Transfer-Encoding', 'chunked') + conn.endheaders() + chunk = contents.read(chunk_size) + while chunk: + if not content_length: + conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) + else: + conn.send(chunk) + chunk = contents.read(chunk_size) + if not content_length: + conn.send('0\r\n\r\n') + else: + conn.request('PUT', path, contents, headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object PUT failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + return resp.getheader('etag').strip('"') + + + def post_object(url, token, container, name, metadata, http_conn=None): + """ + Change object metadata + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to change + :param metadata: dictionary of object metadata + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + headers = {'X-Auth-Token': token} + for key, value in metadata.iteritems(): + headers['X-Object-Meta-%s' % quote(key)] = quote(value) + conn.request('POST', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object POST failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + + + def delete_object(url, token, container, name, http_conn=None): + """ + Delete object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to delete + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP DELETE request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('DELETE', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object DELETE failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + + class Connection(object): + """Convenience class to make requests that will also retry the request""" + + def __init__(self, authurl, user, key, retries=5, preauthurl=None, + preauthtoken=None, snet=False): + """ + :param authurl: authenitcation URL + :param user: user name to authenticate as + :param key: key/password to authenticate with + :param retries: Number of times to retry the request before failing + :param preauthurl: storage URL (if you have already authenticated) + :param preauthtoken: authentication token (if you have already + authenticated) + :param snet: use SERVICENET internal network default is False + """ + self.authurl = authurl + self.user = user + self.key = key + self.retries = retries + self.http_conn = None + self.url = preauthurl + self.token = preauthtoken + self.attempts = 0 + self.snet = snet + + def _retry(self, func, *args, **kwargs): + kwargs['http_conn'] = self.http_conn + self.attempts = 0 + backoff = 1 + while self.attempts <= self.retries: + self.attempts += 1 + try: + if not self.url or not self.token: + self.url, self.token = \ + get_auth(self.authurl, self.user, self.key, snet=self.snet) + self.http_conn = None + if not self.http_conn: + self.http_conn = http_connection(self.url) + kwargs['http_conn'] = self.http_conn + rv = func(self.url, self.token, *args, **kwargs) + return rv + except (socket.error, HTTPException): + if self.attempts > self.retries: + raise + self.http_conn = None + except ClientException, err: + if self.attempts > self.retries: + raise + if err.http_status == 401: + self.url = self.token = None + if self.attempts > 1: + raise + elif 500 <= err.http_status <= 599: + pass + else: + raise + sleep(backoff) + backoff *= 2 + + def head_account(self): + """Wrapper for head_account""" + return self._retry(head_account) + + def get_account(self, marker=None, limit=None, prefix=None, + full_listing=False): + """Wrapper for get_account""" + # TODO: With full_listing=True this will restart the entire listing + # with each retry. Need to make a better version that just retries + # where it left off. + return self._retry(get_account, marker=marker, limit=limit, + prefix=prefix, full_listing=full_listing) + + def head_container(self, container): + """Wrapper for head_container""" + return self._retry(head_container, container) + + def get_container(self, container, marker=None, limit=None, prefix=None, + delimiter=None, full_listing=False): + """Wrapper for get_container""" + # TODO: With full_listing=True this will restart the entire listing + # with each retry. Need to make a better version that just retries + # where it left off. + return self._retry(get_container, container, marker=marker, + limit=limit, prefix=prefix, delimiter=delimiter, + full_listing=full_listing) + + def put_container(self, container): + """Wrapper for put_container""" + return self._retry(put_container, container) + + def delete_container(self, container): + """Wrapper for delete_container""" + return self._retry(delete_container, container) + + def head_object(self, container, obj): + """Wrapper for head_object""" + return self._retry(head_object, container, obj) + + def get_object(self, container, obj, resp_chunk_size=None): + """Wrapper for get_object""" + return self._retry(get_object, container, obj, + resp_chunk_size=resp_chunk_size) + + def put_object(self, container, obj, contents, metadata={}, + content_length=None, etag=None, chunk_size=65536, + content_type=None): + """Wrapper for put_object""" + return self._retry(put_object, container, obj, contents, + metadata=metadata, content_length=content_length, etag=etag, + chunk_size=chunk_size, content_type=content_type) + + def post_object(self, container, obj, metadata): + """Wrapper for post_object""" + return self._retry(post_object, container, obj, metadata) + + def delete_object(self, container, obj): + """Wrapper for delete_object""" + return self._retry(delete_object, container, obj) + + # End inclusion of swift.common.client + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + + +from errno import EEXIST, ENOENT +from hashlib import md5 +from optparse import OptionParser +from os import environ, listdir, makedirs, utime +from os.path import basename, dirname, getmtime, getsize, isdir, join +from Queue import Empty, Queue +from sys import argv, exit, stderr +from threading import enumerate as threading_enumerate, Thread +from time import sleep + + +def mkdirs(path): + try: + makedirs(path) + except OSError, err: + if err.errno != EEXIST: + raise + + +class QueueFunctionThread(Thread): + + def __init__(self, queue, func, *args, **kwargs): + """ Calls func for each item in queue; func is called with a queued + item as the first arg followed by *args and **kwargs. Use the abort + attribute to have the thread empty the queue (without processing) + and exit. """ + Thread.__init__(self) + self.abort = False + self.queue = queue + self.func = func + self.args = args + self.kwargs = kwargs + + def run(self): + while True: + try: + item = self.queue.get_nowait() + if not self.abort: + self.func(item, *self.args, **self.kwargs) + self.queue.task_done() + except Empty: + if self.abort: + break + sleep(0.01) + + +st_delete_help = ''' +delete --all OR delete container [object] [object] ... + Deletes everything in the account (with --all), or everything in a + container, or a list of objects depending on the args given.'''.strip('\n') +def st_delete(options, args): + if (not args and not options.yes_all) or (args and options.yes_all): + options.error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_delete_help)) + return + object_queue = Queue(10000) + def _delete_object((container, obj), conn): + try: + conn.delete_object(container, obj) + if options.verbose: + path = options.yes_all and join(container, obj) or obj + if path[:1] in ('/', '\\'): + path = path[1:] + options.print_queue.put(path) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Object %s not found' % + repr('%s/%s' % (container, obj))) + container_queue = Queue(10000) + def _delete_container(container, conn): + try: + marker = '' + while True: + objects = [o['name'] for o in + conn.get_container(container, marker=marker)] + if not objects: + break + for obj in objects: + object_queue.put((container, obj)) + marker = objects[-1] + while not object_queue.empty(): + sleep(0.01) + attempts = 1 + while True: + try: + conn.delete_container(container) + break + except ClientException, err: + if err.http_status != 409: + raise + if attempts > 10: + raise + attempts += 1 + sleep(1) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Container %s not found' % repr(container)) + url, token = get_auth(options.auth, options.user, options.key, snet=options.snet) + create_connection = lambda: Connection(options.auth, options.user, + options.key, preauthurl=url, + preauthtoken=token, snet=options.snet) + object_threads = [QueueFunctionThread(object_queue, _delete_object, + create_connection()) for _ in xrange(10)] + for thread in object_threads: + thread.start() + container_threads = [QueueFunctionThread(container_queue, + _delete_container, create_connection()) for _ in xrange(10)] + for thread in container_threads: + thread.start() + if not args: + conn = create_connection() + try: + marker = '' + while True: + containers = \ + [c['name'] for c in conn.get_account(marker=marker)] + if not containers: + break + for container in containers: + container_queue.put(container) + marker = containers[-1] + while not container_queue.empty(): + sleep(0.01) + while not object_queue.empty(): + sleep(0.01) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Account not found') + elif len(args) == 1: + conn = create_connection() + _delete_container(args[0], conn) + else: + for obj in args[1:]: + object_queue.put((args[0], obj)) + while not container_queue.empty(): + sleep(0.01) + for thread in container_threads: + thread.abort = True + while thread.isAlive(): + thread.join(0.01) + while not object_queue.empty(): + sleep(0.01) + for thread in object_threads: + thread.abort = True + while thread.isAlive(): + thread.join(0.01) + + +st_download_help = ''' +download --all OR download container [object] [object] ... + Downloads everything in the account (with --all), or everything in a + container, or a list of objects depending on the args given.'''.strip('\n') +def st_download(options, args): + if (not args and not options.yes_all) or (args and options.yes_all): + options.error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_download_help)) + return + object_queue = Queue(10000) + def _download_object((container, obj), conn): + try: + content_type, content_length, _, etag, metadata, body = \ + conn.get_object(container, obj, resp_chunk_size=65536) + path = options.yes_all and join(container, obj) or obj + if path[:1] in ('/', '\\'): + path = path[1:] + if content_type.split(';', 1)[0] == 'text/directory': + if not isdir(path): + mkdirs(path) + read_length = 0 + md5sum = md5() + for chunk in body: + read_length += len(chunk) + md5sum.update(chunk) + else: + dirpath = dirname(path) + if dirpath and not isdir(dirpath): + mkdirs(dirpath) + fp = open(path, 'wb') + read_length = 0 + md5sum = md5() + for chunk in body : + fp.write(chunk) + read_length += len(chunk) + md5sum.update(chunk) + fp.close() + if md5sum.hexdigest() != etag: + options.error_queue.put('%s: md5sum != etag, %s != %s' % + (path, md5sum.hexdigest(), etag)) + if read_length != content_length: + options.error_queue.put( + '%s: read_length != content_length, %d != %d' % + (path, read_length, content_length)) + if 'mtime' in metadata: + mtime = float(metadata['mtime']) + utime(path, (mtime, mtime)) + if options.verbose: + options.print_queue.put(path) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Object %s not found' % + repr('%s/%s' % (container, obj))) + container_queue = Queue(10000) + def _download_container(container, conn): + try: + marker = '' + while True: + objects = [o['name'] for o in + conn.get_container(container, marker=marker)] + if not objects: + break + for obj in objects: + object_queue.put((container, obj)) + marker = objects[-1] + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Container %s not found' % repr(container)) + url, token = get_auth(options.auth, options.user, options.key, snet=options.snet) + create_connection = lambda: Connection(options.auth, options.user, + options.key, preauthurl=url, + preauthtoken=token, snet=options.snet) + object_threads = [QueueFunctionThread(object_queue, _download_object, + create_connection()) for _ in xrange(10)] + for thread in object_threads: + thread.start() + container_threads = [QueueFunctionThread(container_queue, + _download_container, create_connection()) for _ in xrange(10)] + for thread in container_threads: + thread.start() + if not args: + conn = create_connection() + try: + marker = '' + while True: + containers = [c['name'] + for c in conn.get_account(marker=marker)] + if not containers: + break + for container in containers: + container_queue.put(container) + marker = containers[-1] + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Account not found') + elif len(args) == 1: + _download_container(args[0], create_connection()) + else: + for obj in args[1:]: + object_queue.put((args[0], obj)) + while not container_queue.empty(): + sleep(0.01) + for thread in container_threads: + thread.abort = True + while thread.isAlive(): + thread.join(0.01) + while not object_queue.empty(): + sleep(0.01) + for thread in object_threads: + thread.abort = True + while thread.isAlive(): + thread.join(0.01) + + +st_list_help = ''' +list [options] [container] + Lists the containers for the account or the objects for a container. -p or + --prefix is an option that will only list items beginning with that prefix. + -d or --delimiter is option (for container listings only) that will roll up + items with the given delimiter (see Cloud Files general documentation for + what this means). +'''.strip('\n') +def st_list(options, args): + if len(args) > 1: + options.error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_list_help)) + return + conn = Connection(options.auth, options.user, options.key, snet=options.snet) + try: + marker = '' + while True: + if not args: + items = conn.get_account(marker=marker, prefix=options.prefix) + else: + items = conn.get_container(args[0], marker=marker, + prefix=options.prefix, delimiter=options.delimiter) + if not items: + break + for item in items: + options.print_queue.put(item.get('name', item.get('subdir'))) + marker = items[-1].get('name', items[-1].get('subdir')) + except ClientException, err: + if err.http_status != 404: + raise + if not args: + options.error_queue.put('Account not found') + else: + options.error_queue.put('Container %s not found' % repr(args[0])) + + +st_stat_help = ''' +stat [container] [object] + Displays information for the account, container, or object depending on the + args given (if any).'''.strip('\n') +def st_stat(options, args): + conn = Connection(options.auth, options.user, options.key) + if not args: + try: + container_count, object_count, bytes_used = conn.head_account() + options.print_queue.put(''' + Account: %s +Containers: %d + Objects: %d + Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count, + object_count, bytes_used)) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Account not found') + elif len(args) == 1: + try: + object_count, bytes_used = conn.head_container(args[0]) + options.print_queue.put(''' + Account: %s +Container: %s + Objects: %d + Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], + object_count, bytes_used)) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Container %s not found' % repr(args[0])) + elif len(args) == 2: + try: + content_type, content_length, last_modified, etag, metadata = \ + conn.head_object(args[0], args[1]) + options.print_queue.put(''' + Account: %s + Container: %s + Object: %s + Content Type: %s +Content Length: %d + Last Modified: %s + ETag: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], + args[1], content_type, content_length, + last_modified, etag)) + for key, value in metadata.items(): + options.print_queue.put('%14s: %s' % ('Meta %s' % key, value)) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Object %s not found' % + repr('%s/%s' % (args[0], args[1]))) + else: + options.error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_stat_help)) + + +st_upload_help = ''' +upload [options] container file_or_directory [file_or_directory] [...] + Uploads to the given container the files and directories specified by the + remaining args. -c or --changed is an option that will only upload files + that have changed since the last upload.'''.strip('\n') +def st_upload(options, args): + if len(args) < 2: + options.error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_upload_help)) + return + file_queue = Queue(10000) + def _upload_file((path, dir_marker), conn): + try: + obj = path + if obj.startswith('./') or obj.startswith('.\\'): + obj = obj[2:] + metadata = {'mtime': str(getmtime(path))} + if dir_marker: + if options.changed: + try: + ct, cl, lm, et, md = conn.head_object(args[0], obj) + if ct.split(';', 1)[0] == 'text/directory' and \ + cl == 0 and \ + et == 'd41d8cd98f00b204e9800998ecf8427e' and \ + md.get('mtime') == metadata['mtime']: + return + except ClientException, err: + if err.http_status != 404: + raise + conn.put_object(args[0], obj, '', content_length=0, + content_type='text/directory', + metadata=metadata) + else: + if options.changed: + try: + ct, cl, lm, et, md = conn.head_object(args[0], obj) + if cl == getsize(path) and \ + md.get('mtime') == metadata['mtime']: + return + except ClientException, err: + if err.http_status != 404: + raise + conn.put_object(args[0], obj, open(path, 'rb'), + content_length=getsize(path), + metadata=metadata) + if options.verbose: + options.print_queue.put(obj) + except OSError, err: + if err.errno != ENOENT: + raise + options.error_queue.put('Local file %s not found' % repr(path)) + def _upload_dir(path): + names = listdir(path) + if not names: + file_queue.put((path, True)) # dir_marker = True + else: + for name in listdir(path): + subpath = join(path, name) + if isdir(subpath): + _upload_dir(subpath) + else: + file_queue.put((subpath, False)) # dir_marker = False + url, token = get_auth(options.auth, options.user, options.key, snet=options.snet) + create_connection = lambda: Connection(options.auth, options.user, + options.key, preauthurl=url, + preauthtoken=token, snet=options.snet) + file_threads = [QueueFunctionThread(file_queue, _upload_file, + create_connection()) for _ in xrange(10)] + for thread in file_threads: + thread.start() + conn = create_connection() + try: + conn.put_container(args[0]) + for arg in args[1:]: + if isdir(arg): + _upload_dir(arg) + else: + file_queue.put((arg, False)) # dir_marker = False + while not file_queue.empty(): + sleep(0.01) + for thread in file_threads: + thread.abort = True + while thread.isAlive(): + thread.join(0.01) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Account not found') + + +if __name__ == '__main__': + parser = OptionParser(version='%prog 1.0', usage=''' +Usage: %%prog [options] [args] + +Commands: + %(st_stat_help)s + %(st_list_help)s + %(st_upload_help)s + %(st_download_help)s + %(st_delete_help)s + +Example: + %%prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K key stat +'''.strip('\n') % globals()) + parser.add_option('-s', '--snet', action='store_true', dest='snet', + default=False, help='Use SERVICENET internal network') + parser.add_option('-q', '--quiet', action='store_false', dest='verbose', + default=True, help='Suppress status output') + parser.add_option('-a', '--all', action='store_true', dest='yes_all', + default=False, help='Indicate that you really want the ' + 'whole account for commands that require --all in such ' + 'a case') + parser.add_option('-c', '--changed', action='store_true', dest='changed', + default=False, help='For the upload command: will ' + 'only upload files that have changed since the last ' + 'upload') + parser.add_option('-p', '--prefix', dest='prefix', + help='For the list command: will only list items ' + 'beginning with the prefix') + parser.add_option('-d', '--delimiter', dest='delimiter', + help='For the list command on containers: will roll up ' + 'items with the given delimiter (see Cloud Files ' + 'general documentation for what this means).') + parser.add_option('-A', '--auth', dest='auth', + help='URL for obtaining an auth token') + parser.add_option('-U', '--user', dest='user', + help='User name for obtaining an auth token') + parser.add_option('-K', '--key', dest='key', + help='Key for obtaining an auth token') + args = argv[1:] + if not args: + args.append('-h') + (options, args) = parser.parse_args(args) + + required_help = ''' +Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or +overridden with -A, -U, or -K.'''.strip('\n') + for attr in ('auth', 'user', 'key'): + if not getattr(options, attr, None): + setattr(options, attr, environ.get('ST_%s' % attr.upper())) + if not getattr(options, attr, None): + exit(required_help) + + commands = ('delete', 'download', 'list', 'stat', 'upload') + if not args or args[0] not in commands: + parser.print_usage() + if args: + exit('no such command: %s' % args[0]) + exit() + + options.print_queue = Queue(10000) + def _print(item): + if isinstance(item, unicode): + item = item.encode('utf8') + print item + print_thread = QueueFunctionThread(options.print_queue, _print) + print_thread.start() + + options.error_queue = Queue(10000) + def _error(item): + if isinstance(item, unicode): + item = item.encode('utf8') + print >>stderr, item + error_thread = QueueFunctionThread(options.error_queue, _error) + error_thread.start() + + try: + globals()['st_%s' % args[0]](options, args[1:]) + while not options.print_queue.empty(): + sleep(0.01) + print_thread.abort = True + while print_thread.isAlive(): + print_thread.join(0.01) + while not options.error_queue.empty(): + sleep(0.01) + error_thread.abort = True + while error_thread.isAlive(): + error_thread.join(0.01) + except: + for thread in threading_enumerate(): + thread.abort = True + raise diff --git a/bin/swift-account-audit.py b/bin/swift-account-audit.py new file mode 100755 index 0000000000..fe611562d7 --- /dev/null +++ b/bin/swift-account-audit.py @@ -0,0 +1,351 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from urllib import quote +from hashlib import md5 +import getopt +from itertools import chain + +import simplejson +from eventlet.greenpool import GreenPool +from eventlet.event import Event + +from swift.common.ring import Ring +from swift.common.utils import split_path +from swift.common.bufferedhttp import http_connect + + +usage = """ +Usage! + +%(cmd)s [options] [url 1] [url 2] ... + -c [concurrency] Set the concurrency, default 50 + -r [ring dir] Ring locations, default /etc/swift + -e [filename] File for writing a list of inconsistent urls + -d Also download files and verify md5 + +You can also feed a list of urls to the script through stdin. + +Examples! + + %(cmd)s SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076 + %(cmd)s SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076/container/object + %(cmd)s -e errors.txt SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076/container + %(cmd)s < errors.txt + %(cmd)s -c 25 -d < errors.txt +""" % {'cmd': sys.argv[0]} + + +class Auditor(object): + def __init__(self, swift_dir='/etc/swift', concurrency=50, deep=False, + error_file=None): + self.pool = GreenPool(concurrency) + self.object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz')) + self.container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz')) + self.account_ring = Ring(os.path.join(swift_dir, 'account.ring.gz')) + self.deep = deep + self.error_file = error_file + # zero out stats + self.accounts_checked = self.account_exceptions = \ + self.account_not_found = self.account_container_mismatch = \ + self.account_object_mismatch = self.objects_checked = \ + self.object_exceptions = self.object_not_found = \ + self.object_checksum_mismatch = self.containers_checked = \ + self.container_exceptions = self.container_count_mismatch = \ + self.container_not_found = self.container_obj_mismatch = 0 + self.list_cache = {} + self.in_progress = {} + + def audit_object(self, account, container, name): + path = '/%s/%s/%s' % (quote(account), quote(container), quote(name)) + part, nodes = self.object_ring.get_nodes(account, container, name) + container_listing = self.audit_container(account, container) + consistent = True + if name not in container_listing: + print " Object %s missing in container listing!" % path + consistent = False + hash = None + else: + hash = container_listing[name]['hash'] + etags = [] + for node in nodes: + try: + if self.deep: + conn = http_connect(node['ip'], node['port'], + node['device'], part, 'GET', path, {}) + resp = conn.getresponse() + calc_hash = md5() + chunk = True + while chunk: + chunk = resp.read(8192) + calc_hash.update(chunk) + calc_hash = calc_hash.hexdigest() + if resp.status // 100 != 2: + self.object_not_found += 1 + consistent = False + print ' Bad status GETting object "%s" on %s/%s' \ + % (path, node['ip'], node['device']) + continue + if resp.getheader('ETag').strip('"') != calc_hash: + self.object_checksum_mismatch += 1 + consistent = False + print ' MD5 doesnt match etag for "%s" on %s/%s' \ + % (path, node['ip'], node['device']) + etags.append(resp.getheader('ETag')) + else: + conn = http_connect(node['ip'], node['port'], + node['device'], part, 'HEAD', path, {}) + resp = conn.getresponse() + if resp.status // 100 != 2: + self.object_not_found += 1 + consistent = False + print ' Bad status HEADing object "%s" on %s/%s' \ + % (path, node['ip'], node['device']) + continue + etags.append(resp.getheader('ETag')) + except Exception: + self.object_exceptions += 1 + consistent = False + print ' Exception fetching object "%s" on %s/%s' \ + % (path, node['ip'], node['device']) + continue + if not etags: + consistent = False + print " Failed fo fetch object %s at all!" % path + elif hash: + for etag in etags: + if resp.getheader('ETag').strip('"') != hash: + consistent = False + self.object_checksum_mismatch += 1 + print ' ETag mismatch for "%s" on %s/%s' \ + % (path, node['ip'], node['device']) + if not consistent and self.error_file: + print >>open(self.error_file, 'a'), path + self.objects_checked += 1 + + def audit_container(self, account, name, recurse=False): + if (account, name) in self.in_progress: + self.in_progress[(account, name)].wait() + if (account, name) in self.list_cache: + return self.list_cache[(account, name)] + self.in_progress[(account, name)] = Event() + print 'Auditing container "%s"...' % name + path = '/%s/%s' % (quote(account), quote(name)) + account_listing = self.audit_account(account) + consistent = True + if name not in account_listing: + consistent = False + print " Container %s not in account listing!" % path + part, nodes = self.container_ring.get_nodes(account, name) + rec_d = {} + responses = {} + for node in nodes: + marker = '' + results = True + while results: + node_id = node['id'] + try: + conn = http_connect(node['ip'], node['port'], node['device'], + part, 'GET', path, {}, + 'format=json&marker=%s' % quote(marker)) + resp = conn.getresponse() + if resp.status // 100 != 2: + self.container_not_found += 1 + consistent = False + print ' Bad status GETting container "%s" on %s/%s' % \ + (path, node['ip'], node['device']) + break + if node['id'] not in responses: + responses[node['id']] = dict(resp.getheaders()) + results = simplejson.loads(resp.read()) + except Exception: + self.container_exceptions += 1 + consistent = False + print ' Exception GETting container "%s" on %s/%s' % \ + (path, node['ip'], node['device']) + break + if results: + marker = results[-1]['name'] + for obj in results: + obj_name = obj['name'] + if obj_name not in rec_d: + rec_d[obj_name] = obj + if obj['last_modified'] != rec_d[obj_name]['last_modified']: + self.container_obj_mismatch += 1 + consistent = False + print " Different versions of %s/%s in container dbs." % \ + (quote(name), quote(obj['name'])) + if obj['last_modified'] > rec_d[obj_name]['last_modified']: + rec_d[obj_name] = obj + obj_counts = [int(header['x-container-object-count']) + for header in responses.values()] + if not obj_counts: + consistent = False + print " Failed to fetch container %s at all!" % path + else: + if len(set(obj_counts)) != 1: + self.container_count_mismatch += 1 + consistent = False + print " Container databases don't agree on number of objects." + print " Max: %s, Min: %s" % (max(obj_counts), min(obj_counts)) + self.containers_checked += 1 + self.list_cache[(account, name)] = rec_d + self.in_progress[(account, name)].send(True) + del self.in_progress[(account, name)] + if recurse: + for obj in rec_d.keys(): + self.pool.spawn_n(self.audit_object, account, name, obj) + if not consistent and self.error_file: + print >>open(self.error_file, 'a'), path + return rec_d + + def audit_account(self, account, recurse=False): + if account in self.in_progress: + self.in_progress[account].wait() + if account in self.list_cache: + return self.list_cache[account] + self.in_progress[account] = Event() + print "Auditing account %s..." % account + consistent = True + path = '/%s' % account + part, nodes = self.account_ring.get_nodes(account) + responses = {} + for node in nodes: + marker = '' + results = True + while results: + node_id = node['id'] + try: + conn = http_connect(node['ip'], node['port'], + node['device'], part, 'GET', path, {}, + 'format=json&marker=%s' % quote(marker)) + resp = conn.getresponse() + if resp.status // 100 != 2: + self.account_not_found += 1 + consistent = False + print " Bad status GETting account %(ip)s:%(device)s" \ + % node + break + results = simplejson.loads(resp.read()) + except Exception: + self.account_exceptions += 1 + consistent = False + print " Exception GETting account %(ip)s:%(device)s" % node + break + if node_id not in responses: + responses[node_id] = [dict(resp.getheaders()), []] + responses[node_id][1].extend(results) + if results: + marker = results[-1]['name'] + headers = [resp[0] for resp in responses.values()] + cont_counts = [int(header['x-account-container-count']) + for header in headers] + if len(set(cont_counts)) != 1: + self.account_container_mismatch += 1 + consistent = False + print " Account databases don't agree on number of containers." + print " Max: %s, Min: %s" % (max(cont_counts), min(cont_counts)) + obj_counts = [int(header['x-account-object-count']) + for header in headers] + if len(set(obj_counts)) != 1: + self.account_object_mismatch += 1 + consistent = False + print " Account databases don't agree on number of objects." + print " Max: %s, Min: %s" % (max(obj_counts), min(obj_counts)) + containers = set() + for resp in responses.values(): + containers.update(container['name'] for container in resp[1]) + self.list_cache[account] = containers + self.in_progress[account].send(True) + del self.in_progress[account] + self.accounts_checked += 1 + if recurse: + for container in containers: + self.pool.spawn_n(self.audit_container, account, container, True) + if not consistent and self.error_file: + print >>open(self.error_file, 'a'), path + return containers + + def audit(self, account, container=None, obj=None): + if obj and container: + self.pool.spawn_n(self.audit_object, account, container, obj) + elif container: + self.pool.spawn_n(self.audit_container, account, container, True) + else: + self.pool.spawn_n(self.audit_account, account, True) + + def wait(self): + self.pool.waitall() + + def print_stats(self): + print + print " Accounts checked: %d" % self.accounts_checked + if self.account_not_found: + print " Missing Replicas: %d" % self.account_not_found + if self.account_exceptions: + print " Exceptions: %d" % self.account_exceptions + if self.account_container_mismatch: + print " Cntainer mismatch: %d" % self.account_container_mismatch + if self.account_object_mismatch: + print " Object mismatch: %d" % self.account_object_mismatch + print + print "Containers checked: %d" % self.containers_checked + if self.container_not_found: + print " Missing Replicas: %d" % self.container_not_found + if self.container_exceptions: + print " Exceptions: %d" % self.container_exceptions + if self.container_count_mismatch: + print " Count mismatch: %d" % self.container_count_mismatch + if self.container_obj_mismatch: + print " Obj mismatch: %d" % self.container_obj_mismatch + print + print " Objects checked: %d" % self.objects_checked + if self.object_not_found: + print " Missing Replicas: %d" % self.object_not_found + if self.object_exceptions: + print " Exceptions: %d" % self.object_exceptions + if self.object_checksum_mismatch: + print " MD5 Mismatch: %d" % self.object_checksum_mismatch + + +if __name__ == '__main__': + try: + optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d') + except getopt.GetoptError, err: + print str(err) + print usage + sys.exit(2) + if not args and os.isatty(sys.stdin.fileno()): + print usage + sys.exit() + opts = dict(optlist) + options = { + 'concurrency': int(opts.get('-c', 50)), + 'error_file': opts.get('-e', None), + 'swift_dir': opts.get('-r', '/etc/swift'), + 'deep': '-d' in opts, + } + auditor = Auditor(**options) + if not os.isatty(sys.stdin.fileno()): + args = chain(args, sys.stdin) + for path in args: + path = '/' + path.rstrip('\r\n').lstrip('/') + auditor.audit(*split_path(path, 1, 3, True)) + auditor.wait() + auditor.print_stats() + diff --git a/bin/swift-account-auditor.py b/bin/swift-account-auditor.py new file mode 100755 index 0000000000..2bbb73c23f --- /dev/null +++ b/bin/swift-account-auditor.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import signal +import sys +from ConfigParser import ConfigParser + +from swift.account.auditor import AccountAuditor +from swift.common import utils + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: account-auditor CONFIG_FILE [once]" + sys.exit() + + once = len(sys.argv) > 2 and sys.argv[2] == 'once' + + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + + server_conf = dict(c.items('account-server')) + if c.has_section('account-auditor'): + auditor_conf = dict(c.items('account-auditor')) + else: + print "Unable to find account-auditor config section in %s." % \ + sys.argv[1] + sys.exit(1) + + logger = utils.get_logger(auditor_conf, 'account-auditor') + # log uncaught exceptions + sys.excepthook = lambda *exc_info: \ + logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + sys.stdout = sys.stderr = utils.LoggerFileObject(logger) + + utils.drop_privileges(server_conf.get('user', 'swift')) + + try: + os.setsid() + except OSError: + pass + + def kill_children(*args): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + os.killpg(0, signal.SIGTERM) + sys.exit() + + signal.signal(signal.SIGTERM, kill_children) + + auditor = AccountAuditor(server_conf, auditor_conf) + if once: + auditor.audit_once() + else: + auditor.audit_forever() diff --git a/bin/swift-account-reaper.py b/bin/swift-account-reaper.py new file mode 100755 index 0000000000..d62e43add8 --- /dev/null +++ b/bin/swift-account-reaper.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import signal +import sys +from ConfigParser import ConfigParser + +from swift.account.reaper import AccountReaper +from swift.common import utils + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: account-reaper CONFIG_FILE [once]" + sys.exit() + + once = len(sys.argv) > 2 and sys.argv[2] == 'once' + + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + + server_conf = dict(c.items('account-server')) + if c.has_section('account-reaper'): + reaper_conf = dict(c.items('account-reaper')) + else: + print "Unable to find account-reaper config section in %s." % \ + sys.argv[1] + sys.exit(1) + + logger = utils.get_logger(reaper_conf, 'account-reaper') + # log uncaught exceptions + sys.excepthook = lambda *exc_info: \ + logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + sys.stdout = sys.stderr = utils.LoggerFileObject(logger) + + utils.drop_privileges(server_conf.get('user', 'swift')) + + try: + os.setsid() + except OSError: + pass + + def kill_children(*args): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + os.killpg(0, signal.SIGTERM) + sys.exit() + + signal.signal(signal.SIGTERM, kill_children) + + reaper = AccountReaper(server_conf, reaper_conf) + if once: + reaper.reap_once() + else: + reaper.reap_forever() diff --git a/bin/swift-account-replicator.py b/bin/swift-account-replicator.py new file mode 100755 index 0000000000..3e47eaa4f1 --- /dev/null +++ b/bin/swift-account-replicator.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from ConfigParser import ConfigParser +import getopt + +from swift.account import server as account_server +from swift.common import db, db_replicator, utils + +class AccountReplicator(db_replicator.Replicator): + server_type = 'account' + ring_file = 'account.ring.gz' + brokerclass = db.AccountBroker + datadir = account_server.DATADIR + default_port = 6002 + +if __name__ == '__main__': + optlist, args = getopt.getopt(sys.argv[1:], '', ['once']) + + if not args: + print "Usage: account-replicator <--once> CONFIG_FILE [once]" + sys.exit() + + c = ConfigParser() + if not c.read(args[0]): + print "Unable to read config file." + sys.exit(1) + once = len(args) > 1 and args[1] == 'once' + + server_conf = dict(c.items('account-server')) + if c.has_section('account-replicator'): + replicator_conf = dict(c.items('account-replicator')) + else: + print "Unable to find account-replicator config section in %s." % \ + args[0] + sys.exit(1) + + utils.drop_privileges(server_conf.get('user', 'swift')) + if once or '--once' in [opt[0] for opt in optlist]: + AccountReplicator(server_conf, replicator_conf).replicate_once() + else: + AccountReplicator(server_conf, replicator_conf).replicate_forever() + diff --git a/bin/swift-account-server.py b/bin/swift-account-server.py new file mode 100755 index 0000000000..d5e8021049 --- /dev/null +++ b/bin/swift-account-server.py @@ -0,0 +1,30 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +import sys + +from swift.common.wsgi import run_wsgi +from swift.account.server import AccountController + +if __name__ == '__main__': + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + conf = dict(c.items('account-server')) + run_wsgi(AccountController, conf, default_port=6002) + diff --git a/bin/swift-auth-create-account.py b/bin/swift-auth-create-account.py new file mode 100755 index 0000000000..ccaf93cf5f --- /dev/null +++ b/bin/swift-auth-create-account.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +from sys import argv, exit + +from swift.common.bufferedhttp import http_connect_raw as http_connect + + +if __name__ == '__main__': + f = '/etc/swift/auth-server.conf' + if len(argv) == 5: + f = argv[4] + elif len(argv) != 4: + exit('Syntax: %s [conf_file]' % + argv[0]) + new_account = argv[1] + new_user = argv[2] + new_password = argv[3] + c = ConfigParser() + if not c.read(f): + exit('Unable to read conf file: %s' % f) + conf = dict(c.items('auth-server')) + host = conf.get('bind_ip', '127.0.0.1') + port = int(conf.get('bind_port', 11000)) + path = '/account/%s/%s' % (new_account, new_user) + conn = http_connect(host, port, 'PUT', path, {'x-auth-key':new_password}) + resp = conn.getresponse() + if resp.status == 204: + print resp.getheader('x-storage-url') + else: + print 'Account creation failed. (%d)' % resp.status diff --git a/bin/swift-auth-recreate-accounts.py b/bin/swift-auth-recreate-accounts.py new file mode 100755 index 0000000000..51212b4f4f --- /dev/null +++ b/bin/swift-auth-recreate-accounts.py @@ -0,0 +1,40 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +from sys import argv, exit + +from swift.common.bufferedhttp import http_connect_raw as http_connect + +if __name__ == '__main__': + f = '/etc/swift/auth-server.conf' + if len(argv) == 2: + f = argv[1] + elif len(argv) != 1: + exit('Syntax: %s [conf_file]' % argv[0]) + c = ConfigParser() + if not c.read(f): + exit('Unable to read conf file: %s' % f) + conf = dict(c.items('auth-server')) + host = conf.get('bind_ip', '127.0.0.1') + port = int(conf.get('bind_port', 11000)) + path = '/recreate_accounts' + conn = http_connect(host, port, 'POST', path) + resp = conn.getresponse() + if resp.status == 200: + print resp.read() + else: + print 'Recreating accounts failed. (%d)' % resp.status diff --git a/bin/swift-auth-server.py b/bin/swift-auth-server.py new file mode 100755 index 0000000000..775530e078 --- /dev/null +++ b/bin/swift-auth-server.py @@ -0,0 +1,30 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +import sys + +from swift.common.wsgi import run_wsgi +from swift.auth.server import AuthController + +if __name__ == '__main__': + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + conf = dict(c.items('auth-server')) + run_wsgi(AuthController, conf, default_port=11000) + diff --git a/bin/swift-container-auditor.py b/bin/swift-container-auditor.py new file mode 100755 index 0000000000..1ff1682c52 --- /dev/null +++ b/bin/swift-container-auditor.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import signal +import sys +from ConfigParser import ConfigParser + +from swift.container.auditor import ContainerAuditor +from swift.common import utils + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: container-auditor CONFIG_FILE [once]" + sys.exit() + + once = len(sys.argv) > 2 and sys.argv[2] == 'once' + + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + + server_conf = dict(c.items('container-server')) + if c.has_section('container-auditor'): + auditor_conf = dict(c.items('container-auditor')) + else: + print "Unable to find container-auditor config section in %s." % \ + sys.argv[1] + sys.exit(1) + + logger = utils.get_logger(auditor_conf, 'container-auditor') + # log uncaught exceptions + sys.excepthook = lambda *exc_info: \ + logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + sys.stdout = sys.stderr = utils.LoggerFileObject(logger) + + utils.drop_privileges(server_conf.get('user', 'swift')) + + try: + os.setsid() + except OSError: + pass + + def kill_children(*args): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + os.killpg(0, signal.SIGTERM) + sys.exit() + + signal.signal(signal.SIGTERM, kill_children) + + auditor = ContainerAuditor(server_conf, auditor_conf) + if once: + auditor.audit_once() + else: + auditor.audit_forever() diff --git a/bin/swift-container-replicator.py b/bin/swift-container-replicator.py new file mode 100755 index 0000000000..db2ffc8b20 --- /dev/null +++ b/bin/swift-container-replicator.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from ConfigParser import ConfigParser +import getopt + +from swift.container import server as container_server +from swift.common import db, db_replicator, utils + +class ContainerReplicator(db_replicator.Replicator): + server_type = 'container' + ring_file = 'container.ring.gz' + brokerclass = db.ContainerBroker + datadir = container_server.DATADIR + default_port = 6001 + +if __name__ == '__main__': + optlist, args = getopt.getopt(sys.argv[1:], '', ['once']) + + if not args: + print "Usage: container-replicator <--once> CONFIG_FILE [once]" + sys.exit() + + c = ConfigParser() + if not c.read(args[0]): + print "Unable to read config file." + sys.exit(1) + once = len(args) > 1 and args[1] == 'once' + + server_conf = dict(c.items('container-server')) + if c.has_section('container-replicator'): + replicator_conf = dict(c.items('container-replicator')) + else: + print "Unable to find container-replicator config section in %s." % \ + args[0] + sys.exit(1) + + utils.drop_privileges(server_conf.get('user', 'swift')) + if once or '--once' in [opt[0] for opt in optlist]: + ContainerReplicator(server_conf, replicator_conf).replicate_once() + else: + ContainerReplicator(server_conf, replicator_conf).replicate_forever() + diff --git a/bin/swift-container-server.py b/bin/swift-container-server.py new file mode 100755 index 0000000000..e0ad813f46 --- /dev/null +++ b/bin/swift-container-server.py @@ -0,0 +1,30 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +import sys + +from swift.common.wsgi import run_wsgi +from swift.container.server import ContainerController + +if __name__ == '__main__': + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + conf = dict(c.items('container-server')) + run_wsgi(ContainerController, conf, default_port=6001) + diff --git a/bin/swift-container-updater.py b/bin/swift-container-updater.py new file mode 100755 index 0000000000..c37ba3680a --- /dev/null +++ b/bin/swift-container-updater.py @@ -0,0 +1,63 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import signal +import sys +from ConfigParser import ConfigParser + +from swift.container.updater import ContainerUpdater +from swift.common import utils + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: container-updater CONFIG_FILE [once]" + sys.exit() + + once = len(sys.argv) > 2 and sys.argv[2] == 'once' + + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + + server_conf = dict(c.items('container-server')) + if c.has_section('container-updater'): + updater_conf = dict(c.items('container-updater')) + else: + print "Unable to find container-updater config section in %s." % \ + sys.argv[1] + sys.exit(1) + + utils.drop_privileges(server_conf.get('user', 'swift')) + + try: + os.setsid() + except OSError: + pass + + def kill_children(*args): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + os.killpg(0, signal.SIGTERM) + sys.exit() + + signal.signal(signal.SIGTERM, kill_children) + + updater = ContainerUpdater(server_conf, updater_conf) + if once: + updater.update_once_single_threaded() + else: + updater.update_forever() diff --git a/bin/swift-drive-audit.py b/bin/swift-drive-audit.py new file mode 100755 index 0000000000..cde28c1ed7 --- /dev/null +++ b/bin/swift-drive-audit.py @@ -0,0 +1,125 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import re +import subprocess +import sys +from ConfigParser import ConfigParser + +from swift.common.utils import get_logger + +# To search for more types of errors, add the regex to the list below +error_re = [ + 'error.*(sd[a-z])', + '(sd[a-z]).*error', +] + +def get_devices(device_dir, logger): + devices = [] + for line in open('/proc/mounts').readlines(): + data = line.strip().split() + block_device = data[0] + mount_point = data[1] + if mount_point.startswith(device_dir): + device = {} + device['mount_point'] = mount_point + device['block_device'] = block_device + try: + device_num = os.stat(block_device).st_rdev + except OSError, e: + # If we can't stat the device, then something weird is going on + logger.error("Error: Could not stat %s!" % + block_device) + continue + device['major'] = str(os.major(device_num)) + device['minor'] = str(os.minor(device_num)) + devices.append(device) + for line in open('/proc/partitions').readlines()[2:]: + major,minor,blocks,kernel_device = line.strip().split() + device = [d for d in devices + if d['major'] == major and d['minor'] == minor] + if device: + device[0]['kernel_device'] = kernel_device + return devices + +def get_errors(minutes): + errors = {} + start_time = datetime.datetime.now() - datetime.timedelta(minutes=minutes) + for line in open('/var/log/kern.log'): + if '[ 0.000000]' in line: + # Ignore anything before the last boot + errors = {} + continue + log_time_string = '%s %s' % (start_time.year,' '.join(line.split()[:3])) + log_time = datetime.datetime.strptime( + log_time_string,'%Y %b %d %H:%M:%S') + if log_time > start_time: + for err in error_re: + for device in re.findall(err,line): + errors[device] = errors.get(device,0) + 1 + return errors + +def comment_fstab(mount_point): + with open('/etc/fstab', 'r') as fstab: + with open('/etc/fstab.new', 'w') as new_fstab: + for line in fstab: + parts = line.split() + if len(parts) > 2 and line.split()[1] == mount_point: + new_fstab.write('#' + line) + else: + new_fstab.write(line) + os.rename('/etc/fstab.new', '/etc/fstab') + +if __name__ == '__main__': + c = ConfigParser() + try: + conf_path = sys.argv[1] + except: + print "Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1] + sys.exit(1) + if not c.read(conf_path): + print "Unable to read config file %s" % conf_path + sys.exit(1) + conf = dict(c.items('drive-audit')) + device_dir = conf.get('device_dir', '/srv/node') + minutes = int(conf.get('minutes', 60)) + error_limit = int(conf.get('error_limit', 1)) + logger = get_logger(conf, 'drive-audit') + devices = get_devices(device_dir, logger) + logger.debug("Devices found: %s" % str(devices)) + if not devices: + logger.error("Error: No devices found!") + errors = get_errors(minutes) + logger.debug("Errors found: %s" % str(errors)) + unmounts = 0 + for kernel_device,count in errors.items(): + if count >= error_limit: + device = [d for d in devices + if d['kernel_device'].startswith(kernel_device)] + if device: + mount_point = device[0]['mount_point'] + if mount_point.startswith('/srv/node'): + logger.info("Unmounting %s with %d errors" % + (mount_point, count)) + subprocess.call(['umount','-fl',mount_point]) + logger.info("Commenting out %s from /etc/fstab" % + (mount_point)) + comment_fstab(mount_point) + unmounts += 1 + if unmounts == 0: + logger.info("No drives were unmounted") diff --git a/bin/swift-get-nodes.py b/bin/swift-get-nodes.py new file mode 100755 index 0000000000..0a0c5be8f5 --- /dev/null +++ b/bin/swift-get-nodes.py @@ -0,0 +1,87 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import urllib + +from swift.common.ring import Ring +from swift.common.utils import hash_path + + +if len(sys.argv) < 3 or len(sys.argv) > 5: + print 'Usage: %s [] []' % sys.argv[0] + print 'Shows the nodes responsible for the item specified.' + print 'Example:' + print ' $ %s /etc/swift/account.ring.gz MyAccount' % sys.argv[0] + print ' Partition 5743883' + print ' Hash 96ae332a60b58910784e4417a03e1ad0' + print ' 10.1.1.7:8000 sdd1' + print ' 10.1.9.2:8000 sdb1' + print ' 10.1.5.5:8000 sdf1' + sys.exit(1) + +ringloc = None +account = None +container = None +obj = None + +if len(sys.argv) > 4: ring,account,container,obj = sys.argv[1:5] +elif len(sys.argv) > 3: ring,account,container = sys.argv[1:4] +elif len(sys.argv) > 2: ring,account = sys.argv[1:3] + +print '\nAccount \t%s' % account +print 'Container\t%s' % container +print 'Object \t%s\n' % obj + +if obj: + hash_str = hash_path(account,container,obj) + part, nodes = Ring(ring).get_nodes(account,container,obj) + for node in nodes: + print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'], node['device']) + print '\nPartition\t%s' % part + print 'Hash \t%s\n' % hash_str + for node in nodes: + acct_cont_obj = "%s/%s/%s" % (account, container, obj) + print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' % (node['ip'],node['port'],node['device'],part,urllib.quote(acct_cont_obj)) + print "\n" + for node in nodes: + print 'ssh %s "ls -lah /srv/node/%s/objects/%s/%s/%s/"' % (node['ip'],node['device'],part,hash_str[-3:],hash_str) +elif container: + hash_str = hash_path(account,container) + part, nodes = Ring(ring).get_nodes(account,container) + for node in nodes: + print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'], node['device']) + print '\nPartition %s' % part + print 'Hash %s\n' % hash_str + for node in nodes: + acct_cont = "%s/%s" % (account,container) + print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' % (node['ip'],node['port'],node['device'],part,urllib.quote(acct_cont)) + print "\n" + for node in nodes: + print 'ssh %s "ls -lah /srv/node/%s/containers/%s/%s/%s/%s.db"' % (node['ip'],node['device'],part,hash_str[-3:],hash_str,hash_str) +elif account: + hash_str = hash_path(account) + part, nodes = Ring(ring).get_nodes(account) + for node in nodes: + print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'], node['device']) + print '\nPartition %s' % part + print 'Hash %s\n' % hash_str + for node in nodes: + print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' % (node['ip'],node['port'],node['device'],part, urllib.quote(account)) + print "\n" + for node in nodes: + print 'ssh %s "ls -lah /srv/node/%s/accounts/%s/%s/%s/%s.db"' % (node['ip'],node['device'],part,hash_str[-3:],hash_str,hash_str) + print "\n\n" diff --git a/bin/swift-init.py b/bin/swift-init.py new file mode 100755 index 0000000000..e403764b4f --- /dev/null +++ b/bin/swift-init.py @@ -0,0 +1,181 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import errno +import glob +import os +import resource +import signal +import sys +import time + +ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', + 'container-replicator', 'container-server', 'container-updater', + 'object-auditor', 'object-server', 'object-replicator', 'object-updater', + 'proxy-server', 'account-replicator', 'auth-server', 'account-reaper'] +GRACEFUL_SHUTDOWN_SERVERS = ['account-server', 'container-server', + 'object-server', 'proxy-server', 'auth-server'] +MAX_DESCRIPTORS = 32768 +MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB + +_, server, command = sys.argv +if server == 'all': + servers = ALL_SERVERS +else: + if '-' not in server: + server = '%s-server' % server + servers = [server] +command = command.lower() + +def pid_files(server): + if os.path.exists('/var/run/swift/%s.pid' % server): + pid_files = ['/var/run/swift/%s.pid' % server] + else: + pid_files = glob.glob('/var/run/swift/%s/*.pid' % server) + for pid_file in pid_files: + pid = int(open(pid_file).read().strip()) + yield pid_file, pid + +def do_start(server, once=False): + server_type = '-'.join(server.split('-')[:-1]) + + for pid_file, pid in pid_files(server): + if os.path.exists('/proc/%s' % pid): + print "%s appears to already be running: %s" % (server, pid_file) + return + else: + print "Removing stale pid file %s" % pid_file + os.unlink(pid_file) + + try: + resource.setrlimit(resource.RLIMIT_NOFILE, + (MAX_DESCRIPTORS, MAX_DESCRIPTORS)) + resource.setrlimit(resource.RLIMIT_DATA, + (MAX_MEMORY, MAX_MEMORY)) + except ValueError: + print "Unable to increase file descriptor limit. Running as non-root?" + os.environ['PYTHON_EGG_CACHE'] = '/tmp' + + def launch(ini_file, pid_file): + pid = os.fork() + if pid == 0: + os.setsid() + with open(os.devnull, 'r+b') as nullfile: + for desc in (0, 1, 2): # close stdio + try: + os.dup2(nullfile.fileno(), desc) + except OSError: + pass + try: + if once: + os.execl('/usr/bin/swift-%s' % server, server, + ini_file, 'once') + else: + os.execl('/usr/bin/swift-%s' % server, server, ini_file) + except OSError: + print 'unable to launch %s' % server + sys.exit(0) + else: + fp = open(pid_file, 'w') + fp.write('%d\n' % pid) + fp.close() + try: + os.mkdir('/var/run/swift') + except OSError, err: + if err.errno == errno.EACCES: + sys.exit('Unable to create /var/run/swift. Running as non-root?') + elif err.errno != errno.EEXIST: + raise + if os.path.exists('/etc/swift/%s-server.conf' % server_type): + if once: + print 'Running %s once' % server + else: + print 'Starting %s' % server + launch('/etc/swift/%s-server.conf' % server_type, + '/var/run/swift/%s.pid' % server) + else: + try: + os.mkdir('/var/run/swift/%s' % server) + except OSError, err: + if err.errno == errno.EACCES: + sys.exit( + 'Unable to create /var/run/swift. Running as non-root?') + elif err.errno != errno.EEXIST: + raise + if once: + print 'Running %ss once' % server + else: + print 'Starting %ss' % server + for num, ini_file in enumerate(glob.glob('/etc/swift/%s-server/*.conf' % server_type)): + launch(ini_file, '/var/run/swift/%s/%d.pid' % (server, num)) + +def do_stop(server, graceful=False): + if graceful and server in GRACEFUL_SHUTDOWN_SERVERS: + sig = signal.SIGHUP + else: + sig = signal.SIGTERM + + did_anything = False + pfiles = pid_files(server) + for pid_file, pid in pfiles: + did_anything = True + try: + print 'Stopping %s pid: %s signal: %s' % (server, pid, sig) + os.kill(pid, sig) + except OSError: + print "Process %d not running" % pid + try: + os.unlink(pid_file) + except OSError: + pass + for pid_file, pid in pfiles: + for _ in xrange(150): # 15 seconds + if not os.path.exists('/proc/%s' % pid): + break + time.sleep(0.1) + else: + print 'Waited 15 seconds for pid %s (%s) to die; giving up' % \ + (pid, pid_file) + if not did_anything: + print 'No %s running' % server + +if command == 'start': + for server in servers: + do_start(server) + +if command == 'stop': + for server in servers: + do_stop(server) + +if command == 'shutdown': + for server in servers: + do_stop(server, graceful=True) + +if command == 'restart': + for server in servers: + do_stop(server) + for server in servers: + do_start(server) + +if command == 'reload': + for server in servers: + do_stop(server, graceful=True) + do_start(server) + +if command == 'once': + for server in servers: + do_start(server, once=True) diff --git a/bin/swift-object-auditor.py b/bin/swift-object-auditor.py new file mode 100755 index 0000000000..5d0e54e661 --- /dev/null +++ b/bin/swift-object-auditor.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import signal +import sys +from ConfigParser import ConfigParser + +from swift.obj.auditor import ObjectAuditor +from swift.common import utils + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: object-auditor CONFIG_FILE [once]" + sys.exit() + + once = len(sys.argv) > 2 and sys.argv[2] == 'once' + + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + + server_conf = dict(c.items('object-server')) + if c.has_section('object-auditor'): + auditor_conf = dict(c.items('object-auditor')) + else: + print "Unable to find object-auditor config section in %s." % \ + sys.argv[1] + sys.exit(1) + + logger = utils.get_logger(auditor_conf, 'object-auditor') + # log uncaught exceptions + sys.excepthook = lambda *exc_info: \ + logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + sys.stdout = sys.stderr = utils.LoggerFileObject(logger) + + utils.drop_privileges(server_conf.get('user', 'swift')) + + try: + os.setsid() + except OSError: + pass + + def kill_children(*args): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + os.killpg(0, signal.SIGTERM) + sys.exit() + + signal.signal(signal.SIGTERM, kill_children) + + auditor = ObjectAuditor(server_conf, auditor_conf) + if once: + auditor.audit_once() + else: + auditor.audit_forever() diff --git a/bin/swift-object-info.py b/bin/swift-object-info.py new file mode 100755 index 0000000000..57f2522071 --- /dev/null +++ b/bin/swift-object-info.py @@ -0,0 +1,92 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import cPickle as pickle +from datetime import datetime +from hashlib import md5 + +from swift.common.ring import Ring +from swift.obj.server import read_metadata +from swift.common.utils import hash_path + +if __name__ == '__main__': + if len(sys.argv) <= 1: + print "Usage: %s OBJECT_FILE" % sys.argv[0] + sys.exit(1) + try: + ring = Ring('/etc/swift/object.ring.gz') + except: + ring = None + datafile = sys.argv[1] + fp = open(datafile, 'rb') + metadata = read_metadata(fp) + path = metadata.pop('name','') + content_type = metadata.pop('Content-Type','') + ts = metadata.pop('X-Timestamp','') + etag = metadata.pop('ETag','') + length = metadata.pop('Content-Length','') + if path: + print 'Path: %s' % path + account, container, obj = path.split('/',3)[1:] + print ' Account: %s' % account + print ' Container: %s' % container + print ' Object: %s' % obj + obj_hash = hash_path(account, container, obj) + print ' Object hash: %s' % obj_hash + if ring is not None: + print 'Ring locations:' + part, nodes = ring.get_nodes(account, container, obj) + for node in nodes: + print (' %s:%s - /srv/node/%s/objects/%s/%s/%s/%s.data' % + (node['ip'], node['port'], node['device'], part, + obj_hash[-3:], obj_hash, ts)) + else: + print 'Path: Not found in metadata' + if content_type: + print 'Content-Type: %s' % content_type + else: + print 'Content-Type: Not found in metadata' + if ts: + print 'Timestamp: %s (%s)' % (datetime.fromtimestamp(float(ts)), ts) + else: + print 'Timestamp: Not found in metadata' + h = md5() + file_len = 0 + while True: + data = fp.read(64*1024) + if not data: + break + h.update(data) + file_len += len(data) + h = h.hexdigest() + if etag: + if h == etag: + print 'ETag: %s (valid)' % etag + else: + print "Etag: %s doesn't match file hash of %s!" % (etag, h) + else: + print 'ETag: Not found in metadata' + if length: + if file_len == int(length): + print 'Content-Length: %s (valid)' % length + else: + print "Content-Length: %s doesn't match file length of %s" % ( + length, file_len) + else: + print 'Content-Length: Not found in metadata' + print 'User Metadata: %s' % metadata + fp.close() diff --git a/bin/swift-object-replicator.py b/bin/swift-object-replicator.py new file mode 100755 index 0000000000..0de0b5d64a --- /dev/null +++ b/bin/swift-object-replicator.py @@ -0,0 +1,93 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from ConfigParser import ConfigParser +import logging +import time + +from eventlet import sleep, hubs +hubs.use_hub('poll') + +from swift.obj.replicator import ObjectReplicator +from swift.common.utils import get_logger, drop_privileges, LoggerFileObject + +TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes')) + +def read_configs(conf_file): + c = ConfigParser() + if not c.read(conf_file): + print "Unable to read config file: %s" % conf_file + sys.exit(1) + conf = dict(c.items('object-server')) + repl_conf = dict(c.items('object-replicator')) + if not repl_conf: + sys.exit() + conf['replication_concurrency'] = repl_conf.get('concurrency',1) + conf['vm_test_mode'] = repl_conf.get('vm_test_mode', 'no') + conf['daemonize'] = repl_conf.get('daemonize', 'yes') + conf['run_pause'] = repl_conf.get('run_pause', '30') + conf['log_facility'] = repl_conf.get('log_facility', 'LOG_LOCAL1') + conf['log_level'] = repl_conf.get('log_level', 'INFO') + conf['timeout'] = repl_conf.get('timeout', '5') + conf['stats_interval'] = repl_conf.get('stats_interval', '3600') + conf['reclaim_age'] = int(repl_conf.get('reclaim_age', 86400)) + + return conf + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: object-replicator CONFIG_FILE [once]" + sys.exit() + try: + conf = read_configs(sys.argv[1]) + except: + print "Problem reading the config. Aborting object replication." + sys.exit() + once = len(sys.argv) > 2 and sys.argv[2] == 'once' + logger = get_logger(conf, 'object-replicator') + # log uncaught exceptions + sys.excepthook = lambda *exc_info: \ + logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + sys.stdout = sys.stderr = LoggerFileObject(logger) + drop_privileges(conf.get('user', 'swift')) + if not once and conf.get('daemonize', 'true') in TRUE_VALUES: + logger.info("Starting object replicator in daemon mode.") + # Run the replicator continually + while True: + start = time.time() + logger.info("Starting object replication pass.") + # Run the replicator + replicator = ObjectReplicator(conf, logger) + replicator.run() + total = (time.time() - start)/60 + # Reload the config + logger.info("Object replication complete. (%.02f minutes)" % total) + conf = read_configs(sys.argv[1]) + if conf.get('daemonize', 'true') not in TRUE_VALUES: + # Stop running + logger.info("Daemon mode turned off in config, stopping.") + break + logger.debug('Replication sleeping for %s seconds.' % + conf['run_pause']) + sleep(int(conf['run_pause'])) + else: + start = time.time() + logger.info("Running object replicator in script mode.") + replicator = ObjectReplicator(conf, logger) + replicator.run() + total = (time.time() - start)/60 + logger.info("Object replication complete. (%.02f minutes)" % total) diff --git a/bin/swift-object-server.py b/bin/swift-object-server.py new file mode 100755 index 0000000000..c0603b96a7 --- /dev/null +++ b/bin/swift-object-server.py @@ -0,0 +1,30 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +import sys + +from swift.common.wsgi import run_wsgi +from swift.obj.server import ObjectController + +if __name__ == '__main__': + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + conf = dict(c.items('object-server')) + run_wsgi(ObjectController, conf, default_port=6000) + diff --git a/bin/swift-object-updater.py b/bin/swift-object-updater.py new file mode 100755 index 0000000000..595840a079 --- /dev/null +++ b/bin/swift-object-updater.py @@ -0,0 +1,64 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import signal +import sys +from ConfigParser import ConfigParser + +from swift.obj.updater import ObjectUpdater +from swift.common import utils + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "Usage: object-updater CONFIG_FILE [once]" + sys.exit() + + once = len(sys.argv) > 2 and sys.argv[2] == 'once' + + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + + server_conf = dict(c.items('object-server')) + if c.has_section('object-updater'): + updater_conf = dict(c.items('object-updater')) + else: + print "Unable to find object-updater config section in %s." % \ + sys.argv[1] + sys.exit(1) + + utils.drop_privileges(server_conf.get('user', 'swift')) + + try: + os.setsid() + except OSError: + pass + + def kill_children(*args): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + os.killpg(0, signal.SIGTERM) + sys.exit() + + signal.signal(signal.SIGTERM, kill_children) + + updater = ObjectUpdater(server_conf, updater_conf) + if once: + updater.update_once_single_threaded() + else: + updater.update_forever() + diff --git a/bin/swift-proxy-server.py b/bin/swift-proxy-server.py new file mode 100755 index 0000000000..d1d19677ef --- /dev/null +++ b/bin/swift-proxy-server.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +import os +import sys + +from swift.common.wsgi import run_wsgi +from swift.common.auth import DevAuthMiddleware +from swift.common.memcached import MemcacheRing +from swift.common.utils import get_logger +from swift.proxy.server import Application + +if __name__ == '__main__': + c = ConfigParser() + if not c.read(sys.argv[1]): + print "Unable to read config file." + sys.exit(1) + conf = dict(c.items('proxy-server')) + swift_dir = conf.get('swift_dir', '/etc/swift') + c = ConfigParser() + c.read(os.path.join(swift_dir, 'auth-server.conf')) + auth_conf = dict(c.items('auth-server')) + + memcache = MemcacheRing([s.strip() for s in + conf.get('memcache_servers', '127.0.0.1:11211').split(',') + if s.strip()]) + logger = get_logger(conf, 'proxy') + app = Application(conf, memcache, logger) + # Wrap the app with auth + app = DevAuthMiddleware(app, auth_conf, memcache, logger) + run_wsgi(app, conf, logger=logger, default_port=80) diff --git a/bin/swift-ring-builder.py b/bin/swift-ring-builder.py new file mode 100755 index 0000000000..6c12fb582e --- /dev/null +++ b/bin/swift-ring-builder.py @@ -0,0 +1,558 @@ +#!/usr/bin/python -uO +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cPickle as pickle +from errno import EEXIST +from gzip import GzipFile +from os import mkdir +from os.path import basename, dirname, exists, join as pathjoin +from sys import argv, exit +from time import time + +from swift.common.ring import RingBuilder + + +MAJOR_VERSION = 1 +MINOR_VERSION = 1 +EXIT_RING_CHANGED = 0 +EXIT_RING_UNCHANGED = 1 +EXIT_ERROR = 2 + + +def search_devs(builder, search_value): + # dz-:/_ + orig_search_value = search_value + match = [] + if search_value.startswith('d'): + i = 1 + while i < len(search_value) and search_value[i].isdigit(): + i += 1 + match.append(('id', int(search_value[1:i]))) + search_value = search_value[i:] + if search_value.startswith('z'): + i = 1 + while i < len(search_value) and search_value[i].isdigit(): + i += 1 + match.append(('zone', int(search_value[1:i]))) + search_value = search_value[i:] + if search_value.startswith('-'): + search_value = search_value[1:] + if len(search_value) and search_value[0].isdigit(): + i = 1 + while i < len(search_value) and search_value[i] in '0123456789.': + i += 1 + match.append(('ip', search_value[:i])) + search_value = search_value[i:] + if search_value.startswith(':'): + i = 1 + while i < len(search_value) and search_value[i].isdigit(): + i += 1 + match.append(('port', int(search_value[1:i]))) + search_value = search_value[i:] + if search_value.startswith('/'): + i = 1 + while i < len(search_value) and search_value[i] != '_': + i += 1 + match.append(('device', search_value[1:i])) + search_value = search_value[i:] + if search_value.startswith('_'): + match.append(('meta', search_value[1:])) + search_value = '' + if search_value: + raise ValueError('Invalid : %s' % repr(orig_search_value)) + devs = [] + for dev in builder.devs: + if not dev: + continue + matched = True + for key, value in match: + if key == 'meta': + if value not in dev.get(key): + matched = False + elif dev.get(key) != value: + matched = False + if matched: + devs.append(dev) + return devs + + +SEARCH_VALUE_HELP = ''' + The can be of the form: + dz-:/_ + Any part is optional, but you must include at least one part. + Examples: + d74 Matches the device id 74 + z1 Matches devices in zone 1 + z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4 + 1.2.3.4 Matches devices in any zone with the ip 1.2.3.4 + z1:5678 Matches devices in zone 1 using port 5678 + :5678 Matches devices that use port 5678 + /sdb1 Matches devices with the device name sdb1 + _shiny Matches devices with shiny in the meta data + _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data + Most specific example: + d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" + Nerd explanation: + All items require their single character prefix except the ip, in which + case the - is optional unless the device id or zone is also included. +'''.strip() + +CREATE_HELP = ''' +ring_builder create + Creates with 2^ partitions and . + is number of hours to restrict moving a partition more + than once. +'''.strip() + +SEARCH_HELP = ''' +ring_builder search + Shows information about matching devices. + + %(SEARCH_VALUE_HELP)s +'''.strip() % globals() + +ADD_HELP = ''' +ring_builder add z-:/_ + Adds a device to the ring with the given information. No partitions will be + assigned to the new device until after running 'rebalance'. This is so you + can make multiple device changes and rebalance them all just once. +'''.strip() + +SET_WEIGHT_HELP = ''' +ring_builder set_weight + Resets the device's weight. No partitions will be reassigned to or from the + device until after running 'rebalance'. This is so you can make multiple + device changes and rebalance them all just once. + + %(SEARCH_VALUE_HELP)s +'''.strip() % globals() + +SET_INFO_HELP = ''' +ring_builder set_info + :/_ + Resets the device's information. This information isn't used to assign + partitions, so you can use 'write_ring' afterward to rewrite the current + ring with the newer device information. Any of the parts are optional + in the final :/_ parameter; just give what you + want to change. For instance set_info d74 _"snet: 5.6.7.8" would just + update the meta data for device id 74. + + %(SEARCH_VALUE_HELP)s +'''.strip() % globals() + +REMOVE_HELP = ''' +ring_builder remove + Removes the device(s) from the ring. This should normally just be used for + a device that has failed. For a device you wish to decommission, it's best + to set its weight to 0, wait for it to drain all its data, then use this + remove command. This will not take effect until after running 'rebalance'. + This is so you can make multiple device changes and rebalance them all just + once. + + %(SEARCH_VALUE_HELP)s +'''.strip() % globals() + +SET_MIN_PART_HOURS_HELP = ''' +ring_builder set_min_part_hours + Changes the to the given . This should be set to + however long a full replication/update cycle takes. We're working on a way + to determine this more easily than scanning logs. +'''.strip() + + +if __name__ == '__main__': + if len(argv) < 2: + print ''' +ring_builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s + +%(CREATE_HELP)s + +ring_builder + Shows information about the ring and the devices within. + +%(SEARCH_HELP)s + +%(ADD_HELP)s + +%(SET_WEIGHT_HELP)s + +%(SET_INFO_HELP)s + +%(REMOVE_HELP)s + +ring_builder rebalance + Attempts to rebalance the ring by reassigning partitions that haven't been + recently reassigned. + +ring_builder validate + Just runs the validation routines on the ring. + +ring_builder write_ring + Just rewrites the distributable ring file. This is done automatically after + a successful rebalance, so really this is only useful after one or more + 'set_info' calls when no rebalance is needed but you want to send out the + new device information. + +%(SET_MIN_PART_HOURS_HELP)s + +Quick list: create search add set_weight set_info remove rebalance write_ring + set_min_part_hours +Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error +'''.strip() % globals() + exit(EXIT_RING_UNCHANGED) + + if exists(argv[1]): + builder = pickle.load(open(argv[1], 'rb')) + for dev in builder.devs: + if dev and 'meta' not in dev: + dev['meta'] = '' + elif len(argv) < 3 or argv[2] != 'create': + print 'Ring Builder file does not exist: %s' % argv[1] + exit(EXIT_ERROR) + elif argv[2] == 'create': + if len(argv) < 6: + print CREATE_HELP + exit(EXIT_RING_UNCHANGED) + builder = RingBuilder(int(argv[3]), int(argv[4]), int(argv[5])) + backup_dir = pathjoin(dirname(argv[1]), 'backups') + try: + mkdir(backup_dir) + except OSError, err: + if err.errno != EEXIST: + raise + pickle.dump(builder, open(pathjoin(backup_dir, + '%d.' % time() + basename(argv[1])), 'wb'), protocol=2) + pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + exit(EXIT_RING_CHANGED) + + backup_dir = pathjoin(dirname(argv[1]), 'backups') + try: + mkdir(backup_dir) + except OSError, err: + if err.errno != EEXIST: + raise + + ring_file = argv[1] + if ring_file.endswith('.builder'): + ring_file = ring_file[:-len('.builder')] + ring_file += '.ring.gz' + + if len(argv) == 2: + print '%s, build version %d' % (argv[1], builder.version) + zones = 0 + balance = 0 + if builder.devs: + zones = len(set(d['zone'] for d in builder.devs if d is not None)) + balance = builder.get_balance() + print '%d partitions, %d replicas, %d zones, %d devices, %.02f ' \ + 'balance' % (builder.parts, builder.replicas, zones, + len([d for d in builder.devs if d]), balance) + print 'The minimum number of hours before a partition can be ' \ + 'reassigned is %s' % builder.min_part_hours + if builder.devs: + print 'Devices: id zone ip address port name ' \ + 'weight partitions balance meta' + weighted_parts = builder.parts * builder.replicas / \ + sum(d['weight'] for d in builder.devs if d is not None) + for dev in builder.devs: + if dev is None: + continue + if not dev['weight']: + if dev['parts']: + balance = 999.99 + else: + balance = 0 + else: + balance = 100.0 * dev['parts'] / \ + (dev['weight'] * weighted_parts) - 100.0 + print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \ + (dev['id'], dev['zone'], dev['ip'], dev['port'], + dev['device'], dev['weight'], dev['parts'], balance, + dev['meta']) + exit(EXIT_RING_UNCHANGED) + + if argv[2] == 'search': + if len(argv) < 4: + print SEARCH_HELP + exit(EXIT_RING_UNCHANGED) + devs = search_devs(builder, argv[3]) + if not devs: + print 'No matching devices found' + exit(EXIT_ERROR) + print 'Devices: id zone ip address port name ' \ + 'weight partitions balance meta' + weighted_parts = builder.parts * builder.replicas / \ + sum(d['weight'] for d in builder.devs if d is not None) + for dev in devs: + if not dev['weight']: + if dev['parts']: + balance = 999.99 + else: + balance = 0 + else: + balance = 100.0 * dev['parts'] / \ + (dev['weight'] * weighted_parts) - 100.0 + print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \ + (dev['id'], dev['zone'], dev['ip'], dev['port'], + dev['device'], dev['weight'], dev['parts'], balance, + dev['meta']) + exit(EXIT_RING_UNCHANGED) + + elif argv[2] == 'add': + # add z-:/_ + if len(argv) < 5: + print ADD_HELP + exit(EXIT_RING_UNCHANGED) + + if not argv[3].startswith('z'): + print 'Invalid add value: %s' % argv[3] + exit(EXIT_ERROR) + i = 1 + while i < len(argv[3]) and argv[3][i].isdigit(): + i += 1 + zone = int(argv[3][1:i]) + rest = argv[3][i:] + + if not rest.startswith('-'): + print 'Invalid add value: %s' % argv[3] + exit(EXIT_ERROR) + i = 1 + while i < len(rest) and rest[i] in '0123456789.': + i += 1 + ip = rest[1:i] + rest = rest[i:] + + if not rest.startswith(':'): + print 'Invalid add value: %s' % argv[3] + exit(EXIT_ERROR) + i = 1 + while i < len(rest) and rest[i].isdigit(): + i += 1 + port = int(rest[1:i]) + rest = rest[i:] + + if not rest.startswith('/'): + print 'Invalid add value: %s' % argv[3] + exit(EXIT_ERROR) + i = 1 + while i < len(rest) and rest[i] != '_': + i += 1 + device_name = rest[1:i] + rest = rest[i:] + + meta = '' + if rest.startswith('_'): + meta = rest[1:] + + weight = float(argv[4]) + + for dev in builder.devs: + if dev is None: + continue + if dev['ip'] == ip and dev['port'] == port and \ + dev['device'] == device_name: + print 'Device %d already uses %s:%d/%s.' % \ + (dev['id'], dev['ip'], dev['port'], dev['device']) + exit(EXIT_ERROR) + + next_dev_id = 0 + if builder.devs: + next_dev_id = max(d['id'] for d in builder.devs if d) + 1 + builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip, + 'port': port, 'device': device_name, 'weight': weight, + 'meta': meta}) + print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ + (zone, ip, port, device_name, meta, weight, next_dev_id) + pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + exit(EXIT_RING_UNCHANGED) + + elif argv[2] == 'set_weight': + if len(argv) != 5: + print SET_WEIGHT_HELP + exit(EXIT_RING_UNCHANGED) + devs = search_devs(builder, argv[3]) + weight = float(argv[4]) + if not devs: + print 'No matching devices found' + exit(EXIT_ERROR) + if len(devs) > 1: + print 'Matched more than one device:' + for dev in devs: + print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ + '"%(meta)s"' % dev + if raw_input('Are you sure you want to update the weight for ' + 'these %s devices? (y/N) ' % len(devs)) != 'y': + print 'Aborting device modifications' + exit(EXIT_ERROR) + for dev in devs: + builder.set_dev_weight(dev['id'], weight) + print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ + 'weight set to %(weight)s' % dev + pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + exit(EXIT_RING_UNCHANGED) + + elif argv[2] == 'set_info': + if len(argv) != 5: + print SET_INFO_HELP + exit(EXIT_RING_UNCHANGED) + devs = search_devs(builder, argv[3]) + change_value = argv[4] + change = [] + if len(change_value) and change_value[0].isdigit(): + i = 1 + while i < len(change_value) and change_value[i] in '0123456789.': + i += 1 + change.append(('ip', change_value[:i])) + change_value = change_value[i:] + if change_value.startswith(':'): + i = 1 + while i < len(change_value) and change_value[i].isdigit(): + i += 1 + change.append(('port', int(change_value[1:i]))) + change_value = change_value[i:] + if change_value.startswith('/'): + i = 1 + while i < len(change_value) and change_value[i] != '_': + i += 1 + change.append(('device', change_value[1:i])) + change_value = change_value[i:] + if change_value.startswith('_'): + change.append(('meta', change_value[1:])) + change_value = '' + if change_value or not change: + raise ValueError('Invalid set info change value: %s' % + repr(argv[4])) + if not devs: + print 'No matching devices found' + exit(EXIT_ERROR) + if len(devs) > 1: + print 'Matched more than one device:' + for dev in devs: + print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ + '"%(meta)s"' % dev + if raw_input('Are you sure you want to update the info for ' + 'these %s devices? (y/N) ' % len(devs)) != 'y': + print 'Aborting device modifications' + exit(EXIT_ERROR) + for dev in devs: + orig_dev_string = \ + 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev + test_dev = dict(dev) + for key, value in change: + test_dev[key] = value + for check_dev in builder.devs: + if not check_dev or check_dev['id'] == test_dev['id']: + continue + if check_dev['ip'] == test_dev['ip'] and \ + check_dev['port'] == test_dev['port'] and \ + check_dev['device'] == test_dev['device']: + print 'Device %d already uses %s:%d/%s.' % \ + (check_dev['id'], check_dev['ip'], check_dev['port'], + check_dev['device']) + exit(EXIT_ERROR) + for key, value in change: + dev[key] = value + new_dev_string = \ + 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev + print 'Device %s is now %s' % (orig_dev_string, new_dev_string) + pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + exit(EXIT_RING_UNCHANGED) + + elif argv[2] == 'remove': + if len(argv) < 4: + print REMOVE_HELP + exit(EXIT_RING_UNCHANGED) + devs = search_devs(builder, argv[3]) + if not devs: + print 'No matching devices found' + exit(EXIT_ERROR) + if len(devs) > 1: + print 'Matched more than one device:' + for dev in devs: + print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ + '"%(meta)s"' % dev + if raw_input('Are you sure you want to remove these %s devices? ' + '(y/N) ' % len(devs)) != 'y': + print 'Aborting device removals' + exit(EXIT_ERROR) + for dev in devs: + builder.remove_dev(dev['id']) + print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ + 'marked for removal and will be removed next rebalance.' % dev + pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + exit(EXIT_RING_UNCHANGED) + + elif argv[2] == 'rebalance': + devs_changed = builder.devs_changed + last_balance = builder.get_balance() + parts, balance = builder.rebalance() + if not parts: + print 'No partitions could be reassigned.' + print 'Either none need to be or none can be due to ' \ + 'min_part_hours [%s].' % builder.min_part_hours + exit(EXIT_RING_UNCHANGED) + if not devs_changed and abs(last_balance - balance) < 1: + print 'Cowardly refusing to save rebalance as it did not change ' \ + 'at least 1%.' + exit(EXIT_RING_UNCHANGED) + builder.validate() + print 'Reassigned %d (%.02f%%) partitions. Balance is now %.02f.' % \ + (parts, 100.0 * parts / builder.parts, balance) + if balance > 5: + print '-' * 79 + print 'NOTE: Balance of %.02f indicates you should push this ' % \ + balance + print ' ring, wait at least %d hours, and rebalance/repush.' \ + % builder.min_part_hours + print '-' * 79 + ts = time() + pickle.dump(builder.get_ring(), + GzipFile(pathjoin(backup_dir, '%d.' % ts + + basename(ring_file)), 'wb'), protocol=2) + pickle.dump(builder, open(pathjoin(backup_dir, + '%d.' % ts + basename(argv[1])), 'wb'), protocol=2) + pickle.dump(builder.get_ring(), GzipFile(ring_file, 'wb'), protocol=2) + pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + exit(EXIT_RING_CHANGED) + + elif argv[2] == 'validate': + builder.validate() + exit(EXIT_RING_UNCHANGED) + + elif argv[2] == 'write_ring': + pickle.dump(builder.get_ring(), + GzipFile(pathjoin(backup_dir, '%d.' % time() + + basename(ring_file)), 'wb'), protocol=2) + pickle.dump(builder.get_ring(), GzipFile(ring_file, 'wb'), protocol=2) + exit(EXIT_RING_CHANGED) + + elif argv[2] == 'pretend_min_part_hours_passed': + builder.pretend_min_part_hours_passed() + pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + exit(EXIT_RING_UNCHANGED) + + elif argv[2] == 'set_min_part_hours': + if len(argv) < 4: + print SET_MIN_PART_HOURS_HELP + exit(EXIT_RING_UNCHANGED) + builder.change_min_part_hours(int(argv[3])) + print 'The minimum number of hours before a partition can be ' \ + 'reassigned is now set to %s' % argv[3] + pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + exit(EXIT_RING_UNCHANGED) + + print 'Unknown command: %s' % argv[2] + exit(EXIT_ERROR) diff --git a/bin/swift-stats-populate.py b/bin/swift-stats-populate.py new file mode 100755 index 0000000000..793c8b6ce4 --- /dev/null +++ b/bin/swift-stats-populate.py @@ -0,0 +1,197 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import traceback +from ConfigParser import ConfigParser +from optparse import OptionParser +from sys import exit, argv +from time import time +from uuid import uuid4 + +from eventlet import GreenPool, patcher, sleep +from eventlet.pools import Pool + +from swift.common.client import Connection, get_auth +from swift.common.ring import Ring +from swift.common.utils import compute_eta, get_time_units + + +def put_container(connpool, container, report): + global retries_done + try: + with connpool.item() as conn: + conn.put_container(container) + retries_done += conn.attempts - 1 + if report: + report(True) + except: + if report: + report(False) + raise + + +def put_object(connpool, container, obj, report): + global retries_done + try: + with connpool.item() as conn: + conn.put_object(container, obj, obj, metadata={'stats': obj}) + retries_done += conn.attempts - 1 + if report: + report(True) + except: + if report: + report(False) + raise + + +def report(success): + global begun, created, item_type, next_report, need_to_create, retries_done + if not success: + traceback.print_exc() + exit('Gave up due to error(s).') + created += 1 + if time() < next_report: + return + next_report = time() + 5 + eta, eta_unit = compute_eta(begun, created, need_to_create) + print '\r\x1B[KCreating %s: %d of %d, %d%s left, %d retries' % (item_type, + created, need_to_create, round(eta), eta_unit, retries_done), + + +if __name__ == '__main__': + global begun, created, item_type, next_report, need_to_create, retries_done + patcher.monkey_patch() + + parser = OptionParser() + parser.add_option('-d', '--dispersion', action='store_true', + dest='dispersion', default=False, + help='Run the dispersion population') + parser.add_option('-p', '--performance', action='store_true', + dest='performance', default=False, + help='Run the performance population') + args = argv[1:] + if not args: + args.append('-h') + (options, args) = parser.parse_args(args) + + conf_file = '/etc/swift/stats.conf' + if args: + conf_file = args[0] + c = ConfigParser() + if not c.read(conf_file): + exit('Unable to read config file: %s' % conf_file) + conf = dict(c.items('stats')) + swift_dir = conf.get('swift_dir', '/etc/swift') + dispersion_coverage = int(conf.get('dispersion_coverage', 1)) + big_container_count = int(conf.get('big_container_count', 1000000)) + retries = int(conf.get('retries', 5)) + concurrency = int(conf.get('concurrency', 50)) + + coropool = GreenPool(size=concurrency) + retries_done = 0 + + url, token = get_auth(conf['auth_url'], conf['auth_user'], + conf['auth_key']) + account = url.rsplit('/', 1)[1] + connpool = Pool(max_size=concurrency) + connpool.create = lambda: Connection(conf['auth_url'], + conf['auth_user'], conf['auth_key'], + retries=retries, + preauthurl=url, preauthtoken=token) + + if options.dispersion: + container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz')) + parts_left = \ + dict((x, x) for x in xrange(container_ring.partition_count)) + item_type = 'containers' + created = 0 + retries_done = 0 + need_to_create = need_to_queue = \ + dispersion_coverage / 100.0 * container_ring.partition_count + begun = next_report = time() + next_report += 2 + while need_to_queue >= 1: + container = 'stats_container_dispersion_%s' % uuid4() + part, _ = container_ring.get_nodes(account, container) + if part in parts_left: + coropool.spawn(put_container, connpool, container, report) + sleep() + del parts_left[part] + need_to_queue -= 1 + coropool.waitall() + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KCreated %d containers for dispersion reporting, ' \ + '%d%s, %d retries' % \ + (need_to_create, round(elapsed), elapsed_unit, retries_done) + + container = 'stats_objects' + put_container(connpool, container, None) + object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz')) + parts_left = dict((x, x) for x in xrange(object_ring.partition_count)) + item_type = 'objects' + created = 0 + retries_done = 0 + need_to_create = need_to_queue = \ + dispersion_coverage / 100.0 * object_ring.partition_count + begun = next_report = time() + next_report += 2 + while need_to_queue >= 1: + obj = 'stats_object_dispersion_%s' % uuid4() + part, _ = object_ring.get_nodes(account, container, obj) + if part in parts_left: + coropool.spawn(put_object, connpool, container, obj, report) + sleep() + del parts_left[part] + need_to_queue -= 1 + coropool.waitall() + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KCreated %d objects for dispersion reporting, ' \ + '%d%s, %d retries' % \ + (need_to_create, round(elapsed), elapsed_unit, retries_done) + + if options.performance: + container = 'big_container' + put_container(connpool, container, None) + item_type = 'objects' + created = 0 + retries_done = 0 + need_to_create = need_to_queue = big_container_count + begun = next_report = time() + next_report += 2 + segments = ['00'] + for x in xrange(big_container_count): + obj = '%s/%02x' % ('/'.join(segments), x) + coropool.spawn(put_object, connpool, container, obj, report) + sleep() + need_to_queue -= 1 + i = 0 + while True: + nxt = int(segments[i], 16) + 1 + if nxt < 10005: + segments[i] = '%02x' % nxt + break + else: + segments[i] = '00' + i += 1 + if len(segments) <= i: + segments.append('00') + break + coropool.waitall() + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KCreated %d objects for performance reporting, ' \ + '%d%s, %d retries' % \ + (need_to_create, round(elapsed), elapsed_unit, retries_done) diff --git a/bin/swift-stats-report.py b/bin/swift-stats-report.py new file mode 100755 index 0000000000..537a731697 --- /dev/null +++ b/bin/swift-stats-report.py @@ -0,0 +1,942 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os +import socket +from ConfigParser import ConfigParser +from httplib import HTTPException +from optparse import OptionParser +from sys import argv, exit, stderr +from time import time +from uuid import uuid4 + +from eventlet import GreenPool, hubs, patcher, sleep, Timeout +from eventlet.pools import Pool + +from swift.common import direct_client +from swift.common.client import ClientException, Connection, get_auth +from swift.common.ring import Ring +from swift.common.utils import compute_eta, get_time_units + + +unmounted = [] + +def get_error_log(prefix): + def error_log(msg_or_exc): + global unmounted + if hasattr(msg_or_exc, 'http_status') and \ + msg_or_exc.http_status == 507: + identifier = '%s:%s/%s' + if identifier not in unmounted: + unmounted.append(identifier) + print >>stderr, 'ERROR: %s:%s/%s is unmounted -- This will ' \ + 'cause replicas designated for that device to be ' \ + 'considered missing until resolved or the ring is ' \ + 'updated.' % (msg_or_exc.http_host, msg_or_exc.http_port, + msg_or_exc.http_device) + if not hasattr(msg_or_exc, 'http_status') or \ + msg_or_exc.http_status not in (404, 507): + print >>stderr, 'ERROR: %s: %s' % (prefix, msg_or_exc) + return error_log + + +def audit(coropool, connpool, account, container_ring, object_ring, options): + begun = time() + with connpool.item() as conn: + estimated_items = [conn.head_account()[0]] + items_completed = [0] + retries_done = [0] + containers_missing_replicas = {} + objects_missing_replicas = {} + next_report = [time() + 2] + def report(): + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = \ + compute_eta(begun, items_completed[0], estimated_items[0]) + print '\r\x1B[KAuditing items: %d of %d, %d%s left, %d ' \ + 'retries' % (items_completed[0], estimated_items[0], + round(eta), eta_unit, retries_done[0]), + def direct_container(container, part, nodes): + estimated_objects = 0 + for node in nodes: + found = False + error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node) + try: + attempts, info = direct_client.retry( + direct_client.direct_head_container, node, + part, account, container, + error_log=error_log, + retries=options.retries) + retries_done[0] += attempts - 1 + found = True + if not estimated_objects: + estimated_objects = info[0] + except ClientException, err: + if err.http_status not in (404, 507): + error_log('Giving up on /%s/%s/%s: %s' % (part, account, + container, err)) + except (Exception, Timeout), err: + error_log('Giving up on /%s/%s/%s: %s' % (part, account, + container, err)) + if not found: + if container in containers_missing_replicas: + containers_missing_replicas[container].append(node) + else: + containers_missing_replicas[container] = [node] + estimated_items[0] += estimated_objects + items_completed[0] += 1 + report() + def direct_object(container, obj, part, nodes): + for node in nodes: + found = False + error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node) + try: + attempts, _ = direct_client.retry( + direct_client.direct_head_object, node, part, + account, container, obj, error_log=error_log, + retries=options.retries) + retries_done[0] += attempts - 1 + found = True + except ClientException, err: + if err.http_status not in (404, 507): + error_log('Giving up on /%s/%s/%s: %s' % (part, account, + container, err)) + except (Exception, Timeout), err: + error_log('Giving up on /%s/%s/%s: %s' % (part, account, + container, err)) + if not found: + opath = '/%s/%s' % (container, obj) + if opath in objects_missing_replicas: + objects_missing_replicas[opath].append(node) + else: + objects_missing_replicas[opath] = [node] + items_completed[0] += 1 + report() + cmarker = '' + while True: + with connpool.item() as conn: + containers = [c['name'] for c in conn.get_account(marker=cmarker)] + if not containers: + break + cmarker = containers[-1] + for container in containers: + part, nodes = container_ring.get_nodes(account, container) + coropool.spawn(direct_container, container, part, nodes) + for container in containers: + omarker = '' + while True: + with connpool.item() as conn: + objects = [o['name'] for o in + conn.get_container(container, marker=omarker)] + if not objects: + break + omarker = objects[-1] + for obj in objects: + part, nodes = object_ring.get_nodes(account, container, obj) + coropool.spawn(direct_object, container, obj, part, nodes) + coropool.waitall() + print '\r\x1B[K\r', + if not containers_missing_replicas and not objects_missing_replicas: + print 'No missing items.' + return + if containers_missing_replicas: + print 'Containers Missing' + print '-' * 78 + for container in sorted(containers_missing_replicas.keys()): + part, _ = container_ring.get_nodes(account, container) + for node in containers_missing_replicas[container]: + print 'http://%s:%s/%s/%s/%s/%s' % (node['ip'], node['port'], + node['device'], part, account, container) + if objects_missing_replicas: + if containers_missing_replicas: + print + print 'Objects Missing' + print '-' * 78 + for opath in sorted(objects_missing_replicas.keys()): + _, container, obj = opath.split('/', 2) + part, _ = object_ring.get_nodes(account, container, obj) + for node in objects_missing_replicas[opath]: + print 'http://%s:%s/%s/%s/%s/%s/%s' % (node['ip'], + node['port'], node['device'], part, account, container, + obj) + + +def container_dispersion_report(coropool, connpool, account, container_ring, + options): + """ Returns (number of containers listed, number of distinct partitions, + number of container copies found) """ + with connpool.item() as conn: + containers = [c['name'] for c in + conn.get_account(prefix='stats_container_dispersion_', + full_listing=True)] + containers_listed = len(containers) + if not containers_listed: + print >>stderr, 'No containers to query. Has stats-populate been run?' + return 0 + retries_done = [0] + containers_queried = [0] + container_copies_found = [0, 0, 0, 0] + begun = time() + next_report = [time() + 2] + def direct(container, part, nodes): + found_count = 0 + for node in nodes: + error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node) + try: + attempts, _ = direct_client.retry( + direct_client.direct_head_container, node, + part, account, container, error_log=error_log, + retries=options.retries) + retries_done[0] += attempts - 1 + found_count += 1 + except ClientException, err: + if err.http_status not in (404, 507): + error_log('Giving up on /%s/%s/%s: %s' % (part, account, + container, err)) + except (Exception, Timeout), err: + error_log('Giving up on /%s/%s/%s: %s' % (part, account, + container, err)) + container_copies_found[found_count] += 1 + containers_queried[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, containers_queried[0], + containers_listed) + print '\r\x1B[KQuerying containers: %d of %d, %d%s left, %d ' \ + 'retries' % (containers_queried[0], containers_listed, + round(eta), eta_unit, retries_done[0]), + container_parts = {} + for container in containers: + part, nodes = container_ring.get_nodes(account, container) + if part not in container_parts: + container_parts[part] = part + coropool.spawn(direct, container, part, nodes) + coropool.waitall() + distinct_partitions = len(container_parts) + copies_expected = distinct_partitions * container_ring.replica_count + copies_found = sum(a * b for a, b in enumerate(container_copies_found)) + value = 100.0 * copies_found / copies_expected + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KQueried %d containers for dispersion reporting, ' \ + '%d%s, %d retries' % (containers_listed, round(elapsed), + elapsed_unit, retries_done[0]) + if containers_listed - distinct_partitions: + print 'There were %d overlapping partitions' % ( + containers_listed - distinct_partitions) + if container_copies_found[2]: + print 'There were %d partitions missing one copy.' % \ + container_copies_found[2] + if container_copies_found[1]: + print '! There were %d partitions missing two copies.' % \ + container_copies_found[1] + if container_copies_found[0]: + print '!!! There were %d partitions missing all copies.' % \ + container_copies_found[0] + print '%.02f%% of container copies found (%d of %d)' % ( + value, copies_found, copies_expected) + print 'Sample represents %.02f%% of the container partition space' % ( + 100.0 * distinct_partitions / container_ring.partition_count) + return value + + +def object_dispersion_report(coropool, connpool, account, object_ring, options): + """ Returns (number of objects listed, number of distinct partitions, + number of object copies found) """ + container = 'stats_objects' + with connpool.item() as conn: + try: + objects = [o['name'] for o in conn.get_container(container, + prefix='stats_object_dispersion_', full_listing=True)] + except ClientException, err: + if err.http_status != 404: + raise + print >>stderr, 'No objects to query. Has stats-populate been run?' + return 0 + objects_listed = len(objects) + if not objects_listed: + print >>stderr, 'No objects to query. Has stats-populate been run?' + return 0 + retries_done = [0] + objects_queried = [0] + object_copies_found = [0, 0, 0, 0] + begun = time() + next_report = [time() + 2] + def direct(obj, part, nodes): + found_count = 0 + for node in nodes: + error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node) + try: + attempts, _ = direct_client.retry( + direct_client.direct_head_object, node, part, + account, container, obj, error_log=error_log, + retries=options.retries) + retries_done[0] += attempts - 1 + found_count += 1 + except ClientException, err: + if err.http_status not in (404, 507): + error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account, + container, obj, err)) + except (Exception, Timeout), err: + error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account, + container, obj, err)) + object_copies_found[found_count] += 1 + objects_queried[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, objects_queried[0], + objects_listed) + print '\r\x1B[KQuerying objects: %d of %d, %d%s left, %d ' \ + 'retries' % (objects_queried[0], objects_listed, round(eta), + eta_unit, retries_done[0]), + object_parts = {} + for obj in objects: + part, nodes = object_ring.get_nodes(account, container, obj) + if part not in object_parts: + object_parts[part] = part + coropool.spawn(direct, obj, part, nodes) + coropool.waitall() + distinct_partitions = len(object_parts) + copies_expected = distinct_partitions * object_ring.replica_count + copies_found = sum(a * b for a, b in enumerate(object_copies_found)) + value = 100.0 * copies_found / copies_expected + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KQueried %d objects for dispersion reporting, ' \ + '%d%s, %d retries' % (objects_listed, round(elapsed), + elapsed_unit, retries_done[0]) + if objects_listed - distinct_partitions: + print 'There were %d overlapping partitions' % ( + objects_listed - distinct_partitions) + if object_copies_found[2]: + print 'There were %d partitions missing one copy.' % \ + object_copies_found[2] + if object_copies_found[1]: + print '! There were %d partitions missing two copies.' % \ + object_copies_found[1] + if object_copies_found[0]: + print '!!! There were %d partitions missing all copies.' % \ + object_copies_found[0] + print '%.02f%% of object copies found (%d of %d)' % ( + value, copies_found, copies_expected) + print 'Sample represents %.02f%% of the object partition space' % ( + 100.0 * distinct_partitions / object_ring.partition_count) + return value + + +def container_put_report(coropool, connpool, count, options): + successes = [0] + failures = [0] + retries_done = [0] + begun = time() + next_report = [time() + 2] + def put(container): + with connpool.item() as conn: + try: + conn.put_container(container) + successes[0] += 1 + except (Exception, Timeout): + failures[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, successes[0] + failures[0], + count) + print '\r\x1B[KCreating containers: %d of %d, %d%s left, %d ' \ + 'retries' % (successes[0] + failures[0], count, eta, + eta_unit, retries_done[0]), + for x in xrange(count): + coropool.spawn(put, 'stats_container_put_%02x' % x) + coropool.waitall() + successes = successes[0] + failures = failures[0] + value = 100.0 * successes / count + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KCreated %d containers for performance reporting, ' \ + '%d%s, %d retries' % (count, round(elapsed), elapsed_unit, + retries_done[0]) + print '%d succeeded, %d failed, %.02f%% success rate' % ( + successes, failures, value) + return value + + +def container_head_report(coropool, connpool, options): + successes = [0] + failures = [0] + retries_done = [0] + begun = time() + next_report = [time() + 2] + with connpool.item() as conn: + containers = [c['name'] for c in + conn.get_account(prefix='stats_container_put_', + full_listing=True)] + count = len(containers) + def head(container): + with connpool.item() as conn: + try: + conn.head_container(container) + successes[0] += 1 + except (Exception, Timeout): + failures[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, successes[0] + failures[0], + count) + print '\r\x1B[KHeading containers: %d of %d, %d%s left, %d ' \ + 'retries' % (successes[0] + failures[0], count, eta, + eta_unit, retries_done[0]), + for container in containers: + coropool.spawn(head, container) + coropool.waitall() + successes = successes[0] + failures = failures[0] + value = 100.0 * successes / len(containers) + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KHeaded %d containers for performance reporting, ' \ + '%d%s, %d retries' % (count, round(elapsed), elapsed_unit, + retries_done[0]) + print '%d succeeded, %d failed, %.02f%% success rate' % ( + successes, failures, value) + return value + + +def container_get_report(coropool, connpool, options): + successes = [0] + failures = [0] + retries_done = [0] + begun = time() + next_report = [time() + 2] + with connpool.item() as conn: + containers = [c['name'] for c in + conn.get_account(prefix='stats_container_put_', + full_listing=True)] + count = len(containers) + def get(container): + with connpool.item() as conn: + try: + conn.get_container(container) + successes[0] += 1 + except (Exception, Timeout): + failures[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, successes[0] + failures[0], + count) + print '\r\x1B[KListing containers: %d of %d, %d%s left, %d ' \ + 'retries' % (successes[0] + failures[0], count, eta, + eta_unit, retries_done[0]), + for container in containers: + coropool.spawn(get, container) + coropool.waitall() + successes = successes[0] + failures = failures[0] + value = 100.0 * successes / len(containers) + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KListing %d containers for performance reporting, ' \ + '%d%s, %d retries' % (count, round(elapsed), elapsed_unit, + retries_done[0]) + print '%d succeeded, %d failed, %.02f%% success rate' % ( + successes, failures, value) + return value + + +def container_standard_listing_report(coropool, connpool, options): + begun = time() + if options.verbose: + print 'Listing big_container', + with connpool.item() as conn: + try: + value = len(conn.get_container('big_container', full_listing=True)) + except ClientException, err: + if err.http_status != 404: + raise + print >>stderr, \ + "big_container doesn't exist. Has stats-populate been run?" + return 0 + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\rGot %d objects (standard listing) in big_container, %d%s' % \ + (value, elapsed, elapsed_unit) + return value + + +def container_prefix_listing_report(coropool, connpool, options): + begun = time() + if options.verbose: + print 'Prefix-listing big_container', + value = 0 + with connpool.item() as conn: + try: + for x in xrange(256): + value += len(conn.get_container('big_container', + prefix=('%02x' % x), full_listing=True)) + except ClientException, err: + if err.http_status != 404: + raise + print >>stderr, \ + "big_container doesn't exist. Has stats-populate been run?" + return 0 + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\rGot %d objects (prefix listing) in big_container, %d%s' % \ + (value, elapsed, elapsed_unit) + return value + + +def container_prefix_delimiter_listing_report(coropool, connpool, options): + begun = time() + if options.verbose: + print 'Prefix-delimiter-listing big_container', + value = [0] + def list(prefix=None): + marker = None + while True: + try: + with connpool.item() as conn: + listing = conn.get_container('big_container', + marker=marker, prefix=prefix, delimiter='/') + except ClientException, err: + if err.http_status != 404: + raise + print >>stderr, "big_container doesn't exist. " \ + "Has stats-populate been run?" + return 0 + if not len(listing): + break + marker = listing[-1].get('name', listing[-1].get('subdir')) + value[0] += len(listing) + subdirs = [] + i = 0 + # Capping the subdirs we'll list per dir to 10 + while len(subdirs) < 10 and i < len(listing): + if 'subdir' in listing[i]: + subdirs.append(listing[i]['subdir']) + i += 1 + del listing + for subdir in subdirs: + coropool.spawn(list, subdir) + sleep() + coropool.spawn(list) + coropool.waitall() + value = value[0] + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\rGot %d objects/subdirs in big_container, %d%s' % (value, + elapsed, elapsed_unit) + return value + + +def container_delete_report(coropool, connpool, options): + successes = [0] + failures = [0] + retries_done = [0] + begun = time() + next_report = [time() + 2] + with connpool.item() as conn: + containers = [c['name'] for c in + conn.get_account(prefix='stats_container_put_', + full_listing=True)] + count = len(containers) + def delete(container): + with connpool.item() as conn: + try: + conn.delete_container(container) + successes[0] += 1 + except (Exception, Timeout): + failures[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, successes[0] + failures[0], + count) + print '\r\x1B[KDeleting containers: %d of %d, %d%s left, %d ' \ + 'retries' % (successes[0] + failures[0], count, eta, + eta_unit, retries_done[0]), + for container in containers: + coropool.spawn(delete, container) + coropool.waitall() + successes = successes[0] + failures = failures[0] + value = 100.0 * successes / len(containers) + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KDeleting %d containers for performance reporting, ' \ + '%d%s, %d retries' % (count, round(elapsed), elapsed_unit, + retries_done[0]) + print '%d succeeded, %d failed, %.02f%% success rate' % ( + successes, failures, value) + return value + + +def object_put_report(coropool, connpool, count, options): + successes = [0] + failures = [0] + retries_done = [0] + begun = time() + next_report = [time() + 2] + def put(obj): + with connpool.item() as conn: + try: + conn.put_object('stats_object_put', obj, '') + successes[0] += 1 + except (Exception, Timeout): + failures[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, successes[0] + failures[0], + count) + print '\r\x1B[KCreating objects: %d of %d, %d%s left, %d ' \ + 'retries' % (successes[0] + failures[0], count, eta, + eta_unit, retries_done[0]), + with connpool.item() as conn: + conn.put_container('stats_object_put') + for x in xrange(count): + coropool.spawn(put, 'stats_object_put_%02x' % x) + coropool.waitall() + successes = successes[0] + failures = failures[0] + value = 100.0 * successes / count + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KCreated %d objects for performance reporting, ' \ + '%d%s, %d retries' % (count, round(elapsed), elapsed_unit, + retries_done[0]) + print '%d succeeded, %d failed, %.02f%% success rate' % ( + successes, failures, value) + return value + + +def object_head_report(coropool, connpool, options): + successes = [0] + failures = [0] + retries_done = [0] + begun = time() + next_report = [time() + 2] + with connpool.item() as conn: + objects = [o['name'] for o in conn.get_container('stats_object_put', + prefix='stats_object_put_', full_listing=True)] + count = len(objects) + def head(obj): + with connpool.item() as conn: + try: + conn.head_object('stats_object_put', obj) + successes[0] += 1 + except (Exception, Timeout): + failures[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, successes[0] + failures[0], + count) + print '\r\x1B[KHeading objects: %d of %d, %d%s left, %d ' \ + 'retries' % (successes[0] + failures[0], count, eta, + eta_unit, retries_done[0]), + for obj in objects: + coropool.spawn(head, obj) + coropool.waitall() + successes = successes[0] + failures = failures[0] + value = 100.0 * successes / len(objects) + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KHeaded %d objects for performance reporting, ' \ + '%d%s, %d retries' % (count, round(elapsed), elapsed_unit, + retries_done[0]) + print '%d succeeded, %d failed, %.02f%% success rate' % ( + successes, failures, value) + return value + + +def object_get_report(coropool, connpool, options): + successes = [0] + failures = [0] + retries_done = [0] + begun = time() + next_report = [time() + 2] + with connpool.item() as conn: + objects = [o['name'] for o in conn.get_container('stats_object_put', + prefix='stats_object_put_', full_listing=True)] + count = len(objects) + def get(obj): + with connpool.item() as conn: + try: + conn.get_object('stats_object_put', obj) + successes[0] += 1 + except (Exception, Timeout): + failures[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, successes[0] + failures[0], + count) + print '\r\x1B[KRetrieving objects: %d of %d, %d%s left, %d ' \ + 'retries' % (successes[0] + failures[0], count, eta, + eta_unit, retries_done[0]), + for obj in objects: + coropool.spawn(get, obj) + coropool.waitall() + successes = successes[0] + failures = failures[0] + value = 100.0 * successes / len(objects) + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KRetrieved %d objects for performance reporting, ' \ + '%d%s, %d retries' % (count, round(elapsed), elapsed_unit, + retries_done[0]) + print '%d succeeded, %d failed, %.02f%% success rate' % ( + successes, failures, value) + return value + + +def object_delete_report(coropool, connpool, options): + successes = [0] + failures = [0] + retries_done = [0] + begun = time() + next_report = [time() + 2] + with connpool.item() as conn: + objects = [o['name'] for o in conn.get_container('stats_object_put', + prefix='stats_object_put_', full_listing=True)] + count = len(objects) + def delete(obj): + with connpool.item() as conn: + try: + conn.delete_object('stats_object_put', obj) + successes[0] += 1 + except (Exception, Timeout): + failures[0] += 1 + if options.verbose and time() >= next_report[0]: + next_report[0] = time() + 5 + eta, eta_unit = compute_eta(begun, successes[0] + failures[0], + count) + print '\r\x1B[KDeleting objects: %d of %d, %d%s left, %d ' \ + 'retries' % (successes[0] + failures[0], count, eta, + eta_unit, retries_done[0]), + for obj in objects: + coropool.spawn(delete, obj) + coropool.waitall() + successes = successes[0] + failures = failures[0] + value = 100.0 * successes / len(objects) + if options.verbose: + elapsed, elapsed_unit = get_time_units(time() - begun) + print '\r\x1B[KDeleted %d objects for performance reporting, ' \ + '%d%s, %d retries' % (count, round(elapsed), elapsed_unit, + retries_done[0]) + print '%d succeeded, %d failed, %.02f%% success rate' % ( + successes, failures, value) + return value + + +if __name__ == '__main__': + patcher.monkey_patch() + hubs.get_hub().debug_exceptions = False + + parser = OptionParser(usage=''' +Usage: %prog [options] [conf_file] + +[conf_file] defaults to /etc/swift/stats.conf'''.strip()) + parser.add_option('-a', '--audit', action='store_true', + dest='audit', default=False, + help='Run the audit checks') + parser.add_option('-d', '--dispersion', action='store_true', + dest='dispersion', default=False, + help='Run the dispersion reports') + parser.add_option('-o', '--output', dest='csv_output', + default=None, + help='Override where the CSV report is written ' + '(default from conf file); the keyword None will ' + 'suppress the CSV report') + parser.add_option('-p', '--performance', action='store_true', + dest='performance', default=False, + help='Run the performance reports') + parser.add_option('-q', '--quiet', action='store_false', dest='verbose', + default=True, help='Suppress status output') + parser.add_option('-r', '--retries', dest='retries', + default=None, + help='Override retry attempts (default from conf file)') + args = argv[1:] + if not args: + args.append('-h') + (options, args) = parser.parse_args(args) + + conf_file = '/etc/swift/stats.conf' + if args: + conf_file = args.pop(0) + c = ConfigParser() + if not c.read(conf_file): + exit('Unable to read config file: %s' % conf_file) + conf = dict(c.items('stats')) + swift_dir = conf.get('swift_dir', '/etc/swift') + dispersion_coverage = int(conf.get('dispersion_coverage', 1)) + container_put_count = int(conf.get('container_put_count', 1000)) + object_put_count = int(conf.get('object_put_count', 1000)) + concurrency = int(conf.get('concurrency', 50)) + if options.retries: + options.retries = int(options.retries) + else: + options.retries = int(conf.get('retries', 5)) + if not options.csv_output: + csv_output = conf.get('csv_output', '/etc/swift/stats.csv') + + coropool = GreenPool(size=concurrency) + + url, token = get_auth(conf['auth_url'], conf['auth_user'], + conf['auth_key']) + account = url.rsplit('/', 1)[1] + connpool = Pool(max_size=concurrency) + connpool.create = lambda: Connection(conf['auth_url'], + conf['auth_user'], conf['auth_key'], + retries=options.retries, preauthurl=url, + preauthtoken=token) + + report = [time(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0] + (R_TIMESTAMP, R_CDR_TIME, R_CDR_VALUE, R_ODR_TIME, R_ODR_VALUE, + R_CPUT_TIME, R_CPUT_RATE, R_CHEAD_TIME, R_CHEAD_RATE, R_CGET_TIME, + R_CGET_RATE, R_CDELETE_TIME, R_CDELETE_RATE, R_CLSTANDARD_TIME, + R_CLPREFIX_TIME, R_CLPREDELIM_TIME, R_OPUT_TIME, R_OPUT_RATE, R_OHEAD_TIME, + R_OHEAD_RATE, R_OGET_TIME, R_OGET_RATE, R_ODELETE_TIME, R_ODELETE_RATE) = \ + xrange(len(report)) + + container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz')) + object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz')) + + if options.audit: + audit(coropool, connpool, account, container_ring, object_ring, options) + if options.verbose and (options.dispersion or options.performance): + print + + if options.dispersion: + begin = time() + report[R_CDR_VALUE] = container_dispersion_report(coropool, connpool, + account, container_ring, options) + report[R_CDR_TIME] = time() - begin + if options.verbose: + print + + begin = time() + report[R_ODR_VALUE] = object_dispersion_report(coropool, connpool, + account, object_ring, options) + report[R_ODR_TIME] = time() - begin + if options.verbose and options.performance: + print + + if options.performance: + begin = time() + report[R_CPUT_RATE] = container_put_report(coropool, connpool, + container_put_count, options) + report[R_CPUT_TIME] = time() - begin + if options.verbose: + print + + begin = time() + report[R_CHEAD_RATE] = \ + container_head_report(coropool, connpool, options) + report[R_CHEAD_TIME] = time() - begin + if options.verbose: + print + + begin = time() + report[R_CGET_RATE] = container_get_report(coropool, connpool, options) + report[R_CGET_TIME] = time() - begin + if options.verbose: + print + + begin = time() + report[R_CDELETE_RATE] = \ + container_delete_report(coropool, connpool, options) + report[R_CDELETE_TIME] = time() - begin + if options.verbose: + print + + begin = time() + container_standard_listing_report(coropool, connpool, options) + report[R_CLSTANDARD_TIME] = time() - begin + if options.verbose: + print + + begin = time() + container_prefix_listing_report(coropool, connpool, options) + report[R_CLPREFIX_TIME] = time() - begin + if options.verbose: + print + + begin = time() + container_prefix_delimiter_listing_report(coropool, connpool, options) + report[R_CLPREDELIM_TIME] = time() - begin + if options.verbose: + print + + begin = time() + report[R_OPUT_RATE] = \ + object_put_report(coropool, connpool, object_put_count, options) + report[R_OPUT_TIME] = time() - begin + if options.verbose: + print + + begin = time() + report[R_OHEAD_RATE] = object_head_report(coropool, connpool, options) + report[R_OHEAD_TIME] = time() - begin + if options.verbose: + print + + begin = time() + report[R_OGET_RATE] = object_get_report(coropool, connpool, options) + report[R_OGET_TIME] = time() - begin + if options.verbose: + print + + begin = time() + report[R_ODELETE_RATE] = \ + object_delete_report(coropool, connpool, options) + report[R_ODELETE_TIME] = time() - begin + + if options.csv_output != 'None': + try: + if not os.path.exists(csv_output): + f = open(csv_output, 'wb') + f.write('Timestamp,' + 'Container Dispersion Report Time,' + 'Container Dispersion Report Value,' + 'Object Dispersion Report Time,' + 'Object Dispersion Report Value,' + 'Container PUT Report Time,' + 'Container PUT Report Success Rate,' + 'Container HEAD Report Time,' + 'Container HEAD Report Success Rate,' + 'Container GET Report Time,' + 'Container GET Report Success Rate' + 'Container DELETE Report Time,' + 'Container DELETE Report Success Rate,' + 'Container Standard Listing Time,' + 'Container Prefix Listing Time,' + 'Container Prefix Delimiter Listing Time,' + 'Object PUT Report Time,' + 'Object PUT Report Success Rate,' + 'Object HEAD Report Time,' + 'Object HEAD Report Success Rate,' + 'Object GET Report Time,' + 'Object GET Report Success Rate' + 'Object DELETE Report Time,' + 'Object DELETE Report Success Rate\r\n') + csv = csv.writer(f) + else: + csv = csv.writer(open(csv_output, 'ab')) + csv.writerow(report) + except Exception, err: + print >>stderr, 'Could not write CSV report:', err diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000000..9fd459f5c9 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +swift (1.0.0) lucid; urgency=low + + * Initial release + + -- Michael Barton Wed, 07 Jul 2010 19:37:44 +0000 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000000..7ed6ff82de --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +5 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000000..331bb404d8 --- /dev/null +++ b/debian/control @@ -0,0 +1,54 @@ +Source: swift +Section: net +Priority: optional +Maintainer: Michael Barton +Build-Depends: debhelper (>> 3.0.0), python (>= 2.6) +Standards-Version: 3.7.2 + +Package: swift +Architecture: all +Depends: python (>= 2.6), python-openssl, python-webob (>= 0.9.7.1~hg20100111-1~racklabs1), python-simplejson, python-xattr, net-tools, python-eventlet (>= 0.9.8pre1-7) +Description: Swift, a distributed virtual object store (common files) + Swift, a distributed virtual object store. + . + Homepage: https://swift.racklabs.com/trac + +Package: swift-proxy +Architecture: all +Depends: swift (=${Source-Version}) +Description: The swift proxy server. + The swift proxy server. + . + Homepage: https://swift.racklabs.com/trac + +Package: swift-object +Architecture: all +Depends: swift (=${Source-Version}) +Description: The swift object server. + The swift object server. + . + Homepage: https://swift.racklabs.com/trac + +Package: swift-container +Architecture: all +Depends: swift (=${Source-Version}) +Description: The swift container server. + The swift container server. + . + Homepage: https://swift.racklabs.com/trac + +Package: swift-account +Architecture: all +Depends: swift (=${Source-Version}) +Description: The swift account server. + The swift account server. + . + Homepage: https://swift.racklabs.com/trac + +Package: swift-auth +Architecture: all +Depends: swift (=${Source-Version}) +Description: The swift auth server. + The swift auth server. + . + Homepage: https://swift.racklabs.com/trac diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000000..70b3474459 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,208 @@ +Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=135 +Name: Swift +Source: https://code.launchpad.net/swift +Files: * +Copyright: 2010, Rackspace, Inc. +License: Apache-2 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000000..285a4ba266 --- /dev/null +++ b/debian/rules @@ -0,0 +1,134 @@ +#!/usr/bin/make -f + +# Verbose mode +export DH_VERBOSE=1 + +PYTHON = "/usr/bin/python" + +clean: + dh_testdir + dh_testroot + $(PYTHON) setup.py clean --all + rm -rf $(CURDIR)/debian/swift + rm -f build-stamp install-stamp + dh_clean + +build: build-stamp +build-stamp: + dh_testdir + $(PYTHON) setup.py build + touch build-stamp + +install: build-stamp + dh_testdir + dh_installdirs + mkdir -p $(CURDIR)/debian/swift/usr/bin + + # Copy files into binary package directories + dh_install --sourcedir=debian/swift + $(PYTHON) setup.py install --root $(CURDIR)/debian/swift + install -m 755 $(CURDIR)/bin/swift-init.py \ + $(CURDIR)/debian/swift/usr/bin/swift-init + install -m 755 $(CURDIR)/bin/swift-ring-builder.py \ + $(CURDIR)/debian/swift/usr/bin/swift-ring-builder + install -m 755 $(CURDIR)/bin/swift-get-nodes.py \ + $(CURDIR)/debian/swift/usr/bin/swift-get-nodes + install -m 755 $(CURDIR)/bin/swift-stats-populate.py \ + $(CURDIR)/debian/swift/usr/bin/swift-stats-populate + install -m 755 $(CURDIR)/bin/swift-stats-report.py \ + $(CURDIR)/debian/swift/usr/bin/swift-stats-report + install -m 644 $(CURDIR)/etc/stats.conf-sample \ + $(CURDIR)/debian/swift/etc/swift + install -m 755 $(CURDIR)/bin/swift-account-audit.py \ + $(CURDIR)/debian/swift/usr/bin/swift-account-audit + install -m 755 $(CURDIR)/bin/st.py \ + $(CURDIR)/debian/swift/usr/bin/st + + # drive-audit + install -m 644 $(CURDIR)/etc/drive-audit.conf-sample \ + $(CURDIR)/debian/swift-object/etc/swift + install -m 755 $(CURDIR)/bin/swift-drive-audit.py \ + $(CURDIR)/debian/swift-object/usr/bin/swift-drive-audit + + # swift-object + install -m 644 $(CURDIR)/etc/object-server.conf-sample \ + $(CURDIR)/debian/swift-object/etc/swift + install -m 755 $(CURDIR)/bin/swift-object-server.py \ + $(CURDIR)/debian/swift-object/usr/bin/swift-object-server + install -m 755 $(CURDIR)/bin/swift-object-replicator.py \ + $(CURDIR)/debian/swift-object/usr/bin/swift-object-replicator + install -m 644 $(CURDIR)/etc/rsyncd.conf-sample \ + $(CURDIR)/debian/swift-object/etc/swift + install -m 755 $(CURDIR)/bin/swift-object-auditor.py \ + $(CURDIR)/debian/swift-object/usr/bin/swift-object-auditor + install -m 755 $(CURDIR)/bin/swift-object-updater.py \ + $(CURDIR)/debian/swift-object/usr/bin/swift-object-updater + install -m 755 $(CURDIR)/bin/swift-object-info.py \ + $(CURDIR)/debian/swift-object/usr/bin/swift-object-info + + # swift-proxy + install -m 644 $(CURDIR)/etc/proxy-server.conf-sample \ + $(CURDIR)/debian/swift-proxy/etc/swift + install -m 755 $(CURDIR)/bin/swift-proxy-server.py \ + $(CURDIR)/debian/swift-proxy/usr/bin/swift-proxy-server + + # swift-container + install -m 644 $(CURDIR)/etc/container-server.conf-sample \ + $(CURDIR)/debian/swift-container/etc/swift + install -m 755 $(CURDIR)/bin/swift-container-server.py \ + $(CURDIR)/debian/swift-container/usr/bin/swift-container-server + install -m 755 $(CURDIR)/bin/swift-container-replicator.py \ + $(CURDIR)/debian/swift-container/usr/bin/swift-container-replicator + install -m 755 $(CURDIR)/bin/swift-container-auditor.py \ + $(CURDIR)/debian/swift-container/usr/bin/swift-container-auditor + install -m 755 $(CURDIR)/bin/swift-container-updater.py \ + $(CURDIR)/debian/swift-container/usr/bin/swift-container-updater + + # swift-account + install -m 644 $(CURDIR)/etc/account-server.conf-sample \ + $(CURDIR)/debian/swift-account/etc/swift + install -m 755 $(CURDIR)/bin/swift-account-server.py \ + $(CURDIR)/debian/swift-account/usr/bin/swift-account-server + install -m 755 $(CURDIR)/bin/swift-account-replicator.py \ + $(CURDIR)/debian/swift-account/usr/bin/swift-account-replicator + install -m 755 $(CURDIR)/bin/swift-account-auditor.py \ + $(CURDIR)/debian/swift-account/usr/bin/swift-account-auditor + install -m 755 $(CURDIR)/bin/swift-account-reaper.py \ + $(CURDIR)/debian/swift-account/usr/bin/swift-account-reaper + + # swift-auth + install -m 644 $(CURDIR)/etc/auth-server.conf-sample \ + $(CURDIR)/debian/swift-auth/etc/swift + install -m 755 $(CURDIR)/bin/swift-auth-server.py \ + $(CURDIR)/debian/swift-auth/usr/bin/swift-auth-server + install -m 755 $(CURDIR)/bin/swift-auth-create-account.py \ + $(CURDIR)/debian/swift-auth/usr/bin/swift-auth-create-account + install -m 755 $(CURDIR)/bin/swift-auth-recreate-accounts.py \ + $(CURDIR)/debian/swift-auth/usr/bin/swift-auth-recreate-accounts + + touch install-stamp + +binary-arch: +binary-indep: install + dh_installinit --no-start + dh_installinit --no-start -pswift-container --init-script=swift-container-replicator + dh_installinit --no-start -pswift-account --init-script=swift-account-replicator + dh_installinit --no-start -pswift-account --init-script=swift-account-reaper + dh_installinit --no-start -pswift-object --init-script=swift-object-auditor + dh_installinit --no-start -pswift-container --init-script=swift-container-auditor + dh_installinit --no-start -pswift-account --init-script=swift-account-auditor + dh_installinit --no-start -pswift-object --init-script=swift-object-updater + dh_installinit --no-start -pswift-object --init-script=swift-object-replicator + dh_installinit --no-start -pswift-container --init-script=swift-container-updater + dh_installcron + dh_installdocs + dh_installchangelogs + dh_compress + dh_fixperms + dh_gencontrol + dh_installdeb + dh_md5sums + dh_builddeb + +binary: binary-arch binary-indep +.PHONY: build clean binary-indep binary-arch binary clean diff --git a/debian/swift-account.dirs b/debian/swift-account.dirs new file mode 100644 index 0000000000..2adfef1f44 --- /dev/null +++ b/debian/swift-account.dirs @@ -0,0 +1,2 @@ +usr/bin +etc/swift diff --git a/debian/swift-account.init b/debian/swift-account.init new file mode 100644 index 0000000000..2298c7edc9 --- /dev/null +++ b/debian/swift-account.init @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-account-server +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift account server +# Description: Account server for swift. +### END INIT INFO + +/usr/bin/swift-init account-server $1 + diff --git a/debian/swift-account.swift-account-auditor b/debian/swift-account.swift-account-auditor new file mode 100644 index 0000000000..4051154004 --- /dev/null +++ b/debian/swift-account.swift-account-auditor @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-account-auditor +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift account auditor server +# Description: Account auditor server for swift. +### END INIT INFO + +/usr/bin/swift-init account-auditor $1 + diff --git a/debian/swift-account.swift-account-reaper b/debian/swift-account.swift-account-reaper new file mode 100644 index 0000000000..0c17f4611b --- /dev/null +++ b/debian/swift-account.swift-account-reaper @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-account-reaper +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift account reaper server +# Description: Account reaper for swift. +### END INIT INFO + +/usr/bin/swift-init account-reaper $1 + diff --git a/debian/swift-account.swift-account-replicator b/debian/swift-account.swift-account-replicator new file mode 100644 index 0000000000..af523b1d6d --- /dev/null +++ b/debian/swift-account.swift-account-replicator @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-account-replicator +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift account replicator +# Description: Account replicator for swift. +### END INIT INFO + +/usr/bin/swift-init account-replicator $1 + diff --git a/debian/swift-auth.dirs b/debian/swift-auth.dirs new file mode 100644 index 0000000000..2adfef1f44 --- /dev/null +++ b/debian/swift-auth.dirs @@ -0,0 +1,2 @@ +usr/bin +etc/swift diff --git a/debian/swift-auth.init b/debian/swift-auth.init new file mode 100644 index 0000000000..41cef971ca --- /dev/null +++ b/debian/swift-auth.init @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-auth-server +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift auth server +# Description: Auth server for swift. +### END INIT INFO + +/usr/bin/swift-init auth-server $1 + diff --git a/debian/swift-container.dirs b/debian/swift-container.dirs new file mode 100644 index 0000000000..2adfef1f44 --- /dev/null +++ b/debian/swift-container.dirs @@ -0,0 +1,2 @@ +usr/bin +etc/swift diff --git a/debian/swift-container.init b/debian/swift-container.init new file mode 100644 index 0000000000..3be4e37e3e --- /dev/null +++ b/debian/swift-container.init @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-container-server +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift container server +# Description: Container server for swift. +### END INIT INFO + +/usr/bin/swift-init container-server $1 + diff --git a/debian/swift-container.swift-container-auditor b/debian/swift-container.swift-container-auditor new file mode 100644 index 0000000000..910dd59bb3 --- /dev/null +++ b/debian/swift-container.swift-container-auditor @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-container-auditor +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift container auditor server +# Description: Container auditor server for swift. +### END INIT INFO + +/usr/bin/swift-init container-auditor $1 + diff --git a/debian/swift-container.swift-container-replicator b/debian/swift-container.swift-container-replicator new file mode 100644 index 0000000000..e23640488f --- /dev/null +++ b/debian/swift-container.swift-container-replicator @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-container-replicator +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift container replicator +# Description: Container replicator for swift. +### END INIT INFO + +/usr/bin/swift-init container-replicator $1 + diff --git a/debian/swift-container.swift-container-updater b/debian/swift-container.swift-container-updater new file mode 100644 index 0000000000..abe356e4a5 --- /dev/null +++ b/debian/swift-container.swift-container-updater @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-container-updater +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift container updater server +# Description: Container updater server for swift. +### END INIT INFO + +/usr/bin/swift-init container-updater $1 + diff --git a/debian/swift-object.dirs b/debian/swift-object.dirs new file mode 100644 index 0000000000..2adfef1f44 --- /dev/null +++ b/debian/swift-object.dirs @@ -0,0 +1,2 @@ +usr/bin +etc/swift diff --git a/debian/swift-object.init b/debian/swift-object.init new file mode 100644 index 0000000000..4c5db84a3f --- /dev/null +++ b/debian/swift-object.init @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-object-server +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift object server +# Description: Object server for swift. +### END INIT INFO + +/usr/bin/swift-init object-server $1 + diff --git a/debian/swift-object.swift-object-auditor b/debian/swift-object.swift-object-auditor new file mode 100644 index 0000000000..d600fb5fbc --- /dev/null +++ b/debian/swift-object.swift-object-auditor @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-object-auditor +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift object auditor server +# Description: Object auditor server for swift. +### END INIT INFO + +/usr/bin/swift-init object-auditor $1 + diff --git a/debian/swift-object.swift-object-replicator b/debian/swift-object.swift-object-replicator new file mode 100644 index 0000000000..aa82fcc427 --- /dev/null +++ b/debian/swift-object.swift-object-replicator @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-object-replicator +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift object replicator server +# Description: Object replicator server for swift. +### END INIT INFO + +/usr/bin/swift-init object-replicator $1 + diff --git a/debian/swift-object.swift-object-updater b/debian/swift-object.swift-object-updater new file mode 100644 index 0000000000..8c655ba76e --- /dev/null +++ b/debian/swift-object.swift-object-updater @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-object-updater +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift object updater server +# Description: Object updater server for swift. +### END INIT INFO + +/usr/bin/swift-init object-updater $1 + diff --git a/debian/swift-proxy.dirs b/debian/swift-proxy.dirs new file mode 100644 index 0000000000..2adfef1f44 --- /dev/null +++ b/debian/swift-proxy.dirs @@ -0,0 +1,2 @@ +usr/bin +etc/swift diff --git a/debian/swift-proxy.init b/debian/swift-proxy.init new file mode 100644 index 0000000000..d11901caf0 --- /dev/null +++ b/debian/swift-proxy.init @@ -0,0 +1,13 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: swift-proxy-server +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Swift proxy server +# Description: Proxy server for swift. +### END INIT INFO + +/usr/bin/swift-init proxy-server $1 + diff --git a/debian/swift.dirs b/debian/swift.dirs new file mode 100644 index 0000000000..9c4a05dc87 --- /dev/null +++ b/debian/swift.dirs @@ -0,0 +1 @@ +etc/swift diff --git a/debian/swift.postinst b/debian/swift.postinst new file mode 100644 index 0000000000..f26869f386 --- /dev/null +++ b/debian/swift.postinst @@ -0,0 +1,8 @@ +#!/bin/sh -e + +# there's probably a better way +python -m compileall `python -c 'import swift;import os;print os.path.dirname(swift.__file__)'` +if ! getent passwd swift > /dev/null ; then + adduser --system --quiet --disabled-login --disabled-password --no-create-home --group swift +fi + diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000000..ebe3817829 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,105 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build +export PYTHONPATH = ../ + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Swift.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Swift.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/source/_static/basic.css b/doc/source/_static/basic.css new file mode 100644 index 0000000000..d909ce37c7 --- /dev/null +++ b/doc/source/_static/basic.css @@ -0,0 +1,416 @@ +/** + * Sphinx stylesheet -- basic theme + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +img { + border: 0; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li div.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable dl, table.indextable dd { + margin-top: 0; + margin-bottom: 0; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +/* -- general body styles --------------------------------------------------- */ + +a.headerlink { + visibility: hidden; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink { + visibility: visible; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.field-list ul { + padding-left: 1em; +} + +.first { +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px 7px 0 7px; + background-color: #ffe; + width: 40%; + float: right; +} + +p.sidebar-title { + font-weight: bold; +} + +/* -- topics ---------------------------------------------------------------- */ + +div.topic { + border: 1px solid #ccc; + padding: 7px 7px 0 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +div.admonition dl { + margin-bottom: 0; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.docutils { + border: 0; + border-collapse: collapse; +} + +table.docutils td, table.docutils th { + padding: 1px 8px 1px 0; + border-top: 0; + border-left: 0; + border-right: 0; + border-bottom: 1px solid #aaa; +} + +table.field-list td, table.field-list th { + border: 0 !important; +} + +table.footnote td, table.footnote th { + border: 0 !important; +} + +th { + text-align: left; + padding-right: 5px; +} + +/* -- other body styles ----------------------------------------------------- */ + +dl { + margin-bottom: 15px; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +dt:target, .highlight { + background-color: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.refcount { + color: #060; +} + +.optional { + font-size: 1.3em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; +} + +td.linenos pre { + padding: 5px 0px; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +tt.descname { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; +} + +tt.descclassname { + background-color: transparent; +} + +tt.xref, a tt { + background-color: transparent; + font-weight: bold; +} + +h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { + background-color: transparent; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} diff --git a/doc/source/_static/default.css b/doc/source/_static/default.css new file mode 100644 index 0000000000..c8091ecb4d --- /dev/null +++ b/doc/source/_static/default.css @@ -0,0 +1,230 @@ +/** + * Sphinx stylesheet -- default theme + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: sans-serif; + font-size: 100%; + background-color: #11303d; + color: #000; + margin: 0; + padding: 0; +} + +div.document { + background-color: #1c4e63; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 230px; +} + +div.body { + background-color: #ffffff; + color: #000000; + padding: 0 20px 30px 20px; +} + +div.footer { + color: #ffffff; + width: 100%; + padding: 9px 0 9px 0; + text-align: center; + font-size: 75%; +} + +div.footer a { + color: #ffffff; + text-decoration: underline; +} + +div.related { + background-color: #133f52; + line-height: 30px; + color: #ffffff; +} + +div.related a { + color: #ffffff; +} + +div.sphinxsidebar { +} + +div.sphinxsidebar h3 { + font-family: 'Trebuchet MS', sans-serif; + color: #ffffff; + font-size: 1.4em; + font-weight: normal; + margin: 0; + padding: 0; +} + +div.sphinxsidebar h3 a { + color: #ffffff; +} + +div.sphinxsidebar h4 { + font-family: 'Trebuchet MS', sans-serif; + color: #ffffff; + font-size: 1.3em; + font-weight: normal; + margin: 5px 0 0 0; + padding: 0; +} + +div.sphinxsidebar p { + color: #ffffff; +} + +div.sphinxsidebar p.topless { + margin: 5px 10px 10px 10px; +} + +div.sphinxsidebar ul { + margin: 10px; + padding: 0; + color: #ffffff; +} + +div.sphinxsidebar a { + color: #98dbcc; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #355f7c; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +div.body p, div.body dd, div.body li { + text-align: left; + line-height: 130%; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: 'Trebuchet MS', sans-serif; + background-color: #f2f2f2; + font-weight: normal; + color: #20435c; + border-bottom: 1px solid #ccc; + margin: 20px -20px 10px -20px; + padding: 3px 0 3px 10px; +} + +div.body h1 { margin-top: 0; font-size: 200%; } +div.body h2 { font-size: 160%; } +div.body h3 { font-size: 140%; } +div.body h4 { font-size: 120%; } +div.body h5 { font-size: 110%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: #c60f0f; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: #c60f0f; + color: white; +} + +div.body p, div.body dd, div.body li { + text-align: left; + line-height: 130%; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.admonition p { + margin-bottom: 5px; +} + +div.admonition pre { + margin-bottom: 5px; +} + +div.admonition ul, div.admonition ol { + margin-bottom: 5px; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.topic { + background-color: #eee; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre { + padding: 5px; + background-color: #eeffcc; + color: #333333; + line-height: 120%; + border: 1px solid #ac9; + border-left: none; + border-right: none; +} + +tt { + background-color: #ecf0f3; + padding: 0 1px 0 1px; + font-size: 0.95em; +} + +.warning tt { + background: #efc2c2; +} + +.note tt { + background: #d6d6d6; +} diff --git a/doc/source/account.rst b/doc/source/account.rst new file mode 100644 index 0000000000..2ddb1f7d33 --- /dev/null +++ b/doc/source/account.rst @@ -0,0 +1,36 @@ +.. _account: + +******* +Account +******* + +.. _account-server: + +Account Server +============== + +.. automodule:: swift.account.server + :members: + :undoc-members: + :show-inheritance: + +.. _account-auditor: + +Account Auditor +=============== + +.. automodule:: swift.account.auditor + :members: + :undoc-members: + :show-inheritance: + +.. _account-reaper: + +Account Reaper +============== + +.. automodule:: swift.account.reaper + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/auth.rst b/doc/source/auth.rst new file mode 100644 index 0000000000..dc5a65ac45 --- /dev/null +++ b/doc/source/auth.rst @@ -0,0 +1,15 @@ +.. _auth: + +************************* +Developer's Authorization +************************* + +.. _auth-server: + +Auth Server +=========== + +.. automodule:: swift.auth.server + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000000..239082023f --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Swift documentation build configuration file, created by +# sphinx-quickstart on Tue May 18 13:50:15 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Swift' +copyright = u'2010, OpenStack, LLC.' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.0' +# The full version, including alpha/beta/rc tags. +release = '1.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Swiftdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'Swift.tex', u'Swift Documentation', + u'Swift Team', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/doc/source/container.rst b/doc/source/container.rst new file mode 100644 index 0000000000..ca6d16c91c --- /dev/null +++ b/doc/source/container.rst @@ -0,0 +1,36 @@ +.. _Container: + +********* +Container +********* + +.. _container-server: + +Container Server +================ + +.. automodule:: swift.container.server + :members: + :undoc-members: + :show-inheritance: + +.. _container-updater: + +Container Updater +================= + +.. automodule:: swift.container.updater + :members: + :undoc-members: + :show-inheritance: + +.. _container-auditor: + +Container Auditor +================= + +.. automodule:: swift.container.auditor + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/db.rst b/doc/source/db.rst new file mode 100644 index 0000000000..268434cf7c --- /dev/null +++ b/doc/source/db.rst @@ -0,0 +1,25 @@ +.. _account_and_container_db: + +*************************** +Account DB and Container DB +*************************** + +.. _db: + +DB +== + +.. automodule:: swift.common.db + :members: + :undoc-members: + :show-inheritance: + +.. _db-replicator: + +DB replicator +============= + +.. automodule:: swift.common.db_replicator + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/development_guidelines.rst b/doc/source/development_guidelines.rst new file mode 100644 index 0000000000..60a70bf68e --- /dev/null +++ b/doc/source/development_guidelines.rst @@ -0,0 +1,54 @@ +====================== +Development Guidelines +====================== + +----------------- +Coding Guidelines +----------------- + +For the most part we try to follow PEP 8 guidelines which can be viewed +here: http://www.python.org/dev/peps/pep-0008/ + +There is a useful pep8 command line tool for checking files for pep8 +compliance which can be installed with ``easy_install pep8``. + +------------------------ +Documentation Guidelines +------------------------ + +The documentation in docstrings should follow the PEP 257 conventions +(as mentioned in the PEP 8 guidelines). + +More specifically: + + 1. Triple qutes should be used for all docstrings. + 2. If the docstring is simple and fits on one line, then just use + one line. + 3. For docstrings that take multiple lines, there should be a newline + after the opening quotes, and before the closing quotes. + 4. Sphinx is used to build documentation, so use the restructured text + markup to designate parameters, return values, etc. Documentation on + the sphinx specific markup can be found here: + http://sphinx.pocoo.org/markup/index.html + +--------------------- +License and Copyright +--------------------- + +Every source file should have the following copyright and license statement at +the top:: + + # Copyright (c) 2010 OpenStack, LLC. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + # implied. + # See the License for the specific language governing permissions and + # limitations under the License. diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst new file mode 100644 index 0000000000..38c753e635 --- /dev/null +++ b/doc/source/development_saio.rst @@ -0,0 +1,445 @@ +======================= +SAIO - Swift All In One +======================= + +----------------------------------- +Instructions for seting up a dev VM +----------------------------------- + +This documents setting up a virtual machine for doing Swift development. The +virtual machine will emulate running a four node Swift cluster. It assumes +you're using *VMware Fusion 3* on *Mac OS X Snow Leopard*, but should give a +good idea what to do on other environments. + +* Get the *Ubuntu 10.04 LTS (Lucid Lynx)* server image from: + http://cdimage.ubuntu.com/releases/10.04/release/ubuntu-10.04-dvd-amd64.iso +* Create guest virtual machine: + + #. `Continue without disc` + #. `Use operating system installation disc image file`, pick the .iso + from above. + #. Select `Linux` and `Ubuntu 64-bit`. + #. Fill in the *Linux Easy Install* details (you should make the user + name match your bzr repo user name). + #. `Customize Settings`, name the image whatever you want + (`SAIO` for instance.) + #. When the `Settings` window comes up, select `Hard Disk`, create an + extra disk (the defaults are fine). + #. Start the virtual machine up and wait for the easy install to + finish. + +* As root on guest (you'll have to log in as you, then `sudo su -`): + + #. `apt-get update` + #. `apt-get install curl gcc bzr memcached python-configobj + python-coverage python-dev python-nose python-setuptools python-simplejson + python-xattr sqlite3 xfsprogs` + #. Install anything else you want, like screen, ssh, vim, etc. + #. `easy_install -U eventlet` + #. `easy_install -U webob` + #. `fdisk /dev/sdb` (set up a single partition) + #. `mkfs.xfs -i size=1024 /dev/sdb1` + #. `mkdir /mnt/sdb1` + #. Edit `/etc/fstab` and add + `/dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0` + #. `mount /mnt/sdb1` + #. `mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 /mnt/sdb1/test` + #. `chown : /mnt/sdb1/*` + #. `for x in {1..4}; do ln -s /mnt/sdb1/$x /srv/$x; done` + #. `mkdir -p /etc/swift/object-server /etc/swift/container-server /etc/swift/account-server /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 /var/run/swift` + #. `chown -R : /etc/swift /srv/[1-4] /var/run/swift` + #. Add to `/etc/rc.local` (before the `exit 0`):: + + mkdir /var/run/swift + chown : /var/run/swift + + #. Create /etc/rsyncd.conf:: + + uid = + gid = + log file = /var/log/rsyncd.log + pid file = /var/run/rsyncd.pid + + + [account6012] + max connections = 25 + path = /srv/1/node/ + read only = false + lock file = /var/lock/account6012.lock + + [account6022] + max connections = 25 + path = /srv/2/node/ + read only = false + lock file = /var/lock/account6022.lock + + [account6032] + max connections = 25 + path = /srv/3/node/ + read only = false + lock file = /var/lock/account6032.lock + + [account6042] + max connections = 25 + path = /srv/4/node/ + read only = false + lock file = /var/lock/account6042.lock + + + [container6011] + max connections = 25 + path = /srv/1/node/ + read only = false + lock file = /var/lock/container6011.lock + + [container6021] + max connections = 25 + path = /srv/2/node/ + read only = false + lock file = /var/lock/container6021.lock + + [container6031] + max connections = 25 + path = /srv/3/node/ + read only = false + lock file = /var/lock/container6031.lock + + [container6041] + max connections = 25 + path = /srv/4/node/ + read only = false + lock file = /var/lock/container6041.lock + + + [object6010] + max connections = 25 + path = /srv/1/node/ + read only = false + lock file = /var/lock/object6010.lock + + [object6020] + max connections = 25 + path = /srv/2/node/ + read only = false + lock file = /var/lock/object6020.lock + + [object6030] + max connections = 25 + path = /srv/3/node/ + read only = false + lock file = /var/lock/object6030.lock + + [object6040] + max connections = 25 + path = /srv/4/node/ + read only = false + lock file = /var/lock/object6040.lock + + #. Edit the following line in /etc/default/rsync:: + + RSYNC_ENABLE=true + + #. `service rsync restart` + +* As you on guest: + + #. `mkdir ~/bin` + #. Create `~/.bazaar/.bazaar.conf`:: + + [DEFAULT] + email = Your Name + + #. Check out your bzr repo of swift, for example: + `bzr branch lp:swift` + #. ``for f in `ls ~/openswift/bin/`; do sudo ln -s /home//openswift/bin/$f /usr/bin/`basename $f .py`; done`` + #. Edit `~/.bashrc` and add to the end:: + + export PYTHONPATH=~/openswift + export PATH_TO_TEST_XFS=/mnt/sdb1/test + export SWIFT_TEST_CONFIG_FILE=/etc/swift/func_test.conf + export PATH=${PATH}:~/bin + + #. `. ~/.bashrc` + #. Create `/etc/swift/auth-server.conf`:: + + [auth-server] + default_cluster_url = http://127.0.0.1:8080/v1 + user = + + #. Create `/etc/swift/proxy-server.conf`:: + + [proxy-server] + bind_port = 8080 + user = + + #. Create `/etc/swift/account-server/1.conf`:: + + [account-server] + devices = /srv/1/node + mount_check = false + bind_port = 6012 + user = + + [account-replicator] + vm_test_mode = yes + + [account-auditor] + + [account-reaper] + + #. Create `/etc/swift/account-server/2.conf`:: + + [account-server] + devices = /srv/2/node + mount_check = false + bind_port = 6022 + user = + + [account-replicator] + vm_test_mode = yes + + [account-auditor] + + [account-reaper] + + #. Create `/etc/swift/account-server/3.conf`:: + + [account-server] + devices = /srv/3/node + mount_check = false + bind_port = 6032 + user = + + [account-replicator] + vm_test_mode = yes + + [account-auditor] + + [account-reaper] + + #. Create `/etc/swift/account-server/4.conf`:: + + [account-server] + devices = /srv/4/node + mount_check = false + bind_port = 6042 + user = + + [account-replicator] + vm_test_mode = yes + + [account-auditor] + + [account-reaper] + + #. Create `/etc/swift/container-server/1.conf`:: + + [container-server] + devices = /srv/1/node + mount_check = false + bind_port = 6011 + user = + + [container-replicator] + vm_test_mode = yes + + [container-updater] + + [container-auditor] + + #. Create `/etc/swift/container-server/2.conf`:: + + [container-server] + devices = /srv/2/node + mount_check = false + bind_port = 6021 + user = + + [container-replicator] + vm_test_mode = yes + + [container-updater] + + [container-auditor] + + #. Create `/etc/swift/container-server/3.conf`:: + + [container-server] + devices = /srv/3/node + mount_check = false + bind_port = 6031 + user = + + [container-replicator] + vm_test_mode = yes + + [container-updater] + + [container-auditor] + + #. Create `/etc/swift/container-server/4.conf`:: + + [container-server] + devices = /srv/4/node + mount_check = false + bind_port = 6041 + user = + + [container-replicator] + vm_test_mode = yes + + [container-updater] + + [container-auditor] + + #. Create `/etc/swift/object-server/1.conf`:: + + [object-server] + devices = /srv/1/node + mount_check = false + bind_port = 6010 + user = + + [object-replicator] + vm_test_mode = yes + + [object-updater] + + [object-auditor] + + #. Create `/etc/swift/object-server/2.conf`:: + + [object-server] + devices = /srv/2/node + mount_check = false + bind_port = 6020 + user = + + [object-replicator] + vm_test_mode = yes + + [object-updater] + + [object-auditor] + + #. Create `/etc/swift/object-server/3.conf`:: + + [object-server] + devices = /srv/3/node + mount_check = false + bind_port = 6030 + user = + + [object-replicator] + vm_test_mode = yes + + [object-updater] + + [object-auditor] + + #. Create `/etc/swift/object-server/4.conf`:: + + [object-server] + devices = /srv/4/node + mount_check = false + bind_port = 6040 + user = + + [object-replicator] + vm_test_mode = yes + + [object-updater] + + [object-auditor] + + #. Create `~/bin/resetswift`:: + + #!/bin/bash + + swift-init all stop + sleep 5 + sudo umount /mnt/sdb1 + sudo mkfs.xfs -f -i size=1024 /dev/sdb1 + sudo mount /mnt/sdb1 + sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 /mnt/sdb1/test + sudo chown : /mnt/sdb1/* + mkdir -p /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 + sudo rm -f /var/log/debug /var/log/messages /var/log/rsyncd.log /var/log/syslog + sudo service rsyslog restart + sudo service memcached restart + + #. Create `~/bin/remakerings`:: + + #!/bin/bash + + cd /etc/swift + + rm *.builder *.ring.gz backups/*.builder backups/*.ring.gz + + swift-ring-builder object.builder create 18 3 1 + swift-ring-builder object.builder add z1-127.0.0.1:6010/sdb1 1 + swift-ring-builder object.builder add z2-127.0.0.1:6020/sdb2 1 + swift-ring-builder object.builder add z3-127.0.0.1:6030/sdb3 1 + swift-ring-builder object.builder add z4-127.0.0.1:6040/sdb4 1 + swift-ring-builder object.builder rebalance + swift-ring-builder container.builder create 18 3 1 + swift-ring-builder container.builder add z1-127.0.0.1:6011/sdb1 1 + swift-ring-builder container.builder add z2-127.0.0.1:6021/sdb2 1 + swift-ring-builder container.builder add z3-127.0.0.1:6031/sdb3 1 + swift-ring-builder container.builder add z4-127.0.0.1:6041/sdb4 1 + swift-ring-builder container.builder rebalance + swift-ring-builder account.builder create 18 3 1 + swift-ring-builder account.builder add z1-127.0.0.1:6012/sdb1 1 + swift-ring-builder account.builder add z2-127.0.0.1:6022/sdb2 1 + swift-ring-builder account.builder add z3-127.0.0.1:6032/sdb3 1 + swift-ring-builder account.builder add z4-127.0.0.1:6042/sdb4 1 + swift-ring-builder account.builder rebalance + + #. Create `~/bin/startmain`:: + + #!/bin/bash + + swift-init auth-server start + swift-init proxy-server start + swift-init account-server start + swift-init container-server start + swift-init object-server start + + #. Create `~/bin/startrest`:: + + #!/bin/bash + + swift-auth-recreate-accounts + swift-init object-updater start + swift-init container-updater start + swift-init object-replicator start + swift-init container-replicator start + swift-init account-replicator start + swift-init object-auditor start + swift-init container-auditor start + swift-init account-auditor start + swift-init account-reaper start + + #. `chmod +x ~/bin/*` + #. `remakerings` + #. `cd ~/openswift; ./.unittests` + #. `startmain` + #. `swift-auth-create-account test tester testing` + #. Get an `X-Storage-Url` and `X-Auth-Token`: `curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0` + #. Check that you can GET account: `curl -v -H 'X-Auth-Token: ' ` + #. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat` + #. Create `/etc/swift/func_test.conf`:: + + auth_host = 127.0.0.1 + auth_port = 11000 + auth_ssl = no + + account = test + username = tester + password = testing + + collate = C + + #. `cd ~/openswift; ./.functests` + #. `cd ~/openswift; ./.probetests` + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000000..d9b4a33aad --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,48 @@ +.. Swift documentation master file, created by + sphinx-quickstart on Tue May 18 13:50:15 2010. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Swift's documentation! +================================= + +Overview: + +.. toctree:: + :maxdepth: 1 + + overview_ring + overview_reaper + overview_auth + overview_replication + +Development: + +.. toctree:: + :maxdepth: 1 + + development_guidelines + development_saio + +Source: + +.. toctree:: + :maxdepth: 2 + + ring + proxy + account + container + db + object + auth + misc + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/misc.rst b/doc/source/misc.rst new file mode 100644 index 0000000000..c45dcf0fc3 --- /dev/null +++ b/doc/source/misc.rst @@ -0,0 +1,99 @@ +.. _misc: + +**** +Misc +**** + +.. _exceptions: + +Exceptions +========== + +.. automodule:: swift.common.exceptions + :members: + :undoc-members: + :show-inheritance: + +.. _constraints: + +Constraints +=========== + +.. automodule:: swift.common.constraints + :members: + :undoc-members: + :show-inheritance: + +.. _utils: + +Utils +===== + +.. automodule:: swift.common.utils + :members: + :show-inheritance: + +.. _common_auth: + +Auth +==== + +.. automodule:: swift.common.auth + :members: + :show-inheritance: + +.. _wsgi: + +WSGI +==== + +.. automodule:: swift.common.wsgi + :members: + :show-inheritance: + +.. _client: + +Client +====== + +.. automodule:: swift.common.client + :members: + :undoc-members: + :show-inheritance: + +.. _direct_client: + +Direct Client +============= + +.. automodule:: swift.common.direct_client + :members: + :undoc-members: + :show-inheritance: + +.. _buffered_http: + +Buffered HTTP +============= + +.. automodule:: swift.common.bufferedhttp + :members: + :show-inheritance: + +.. _healthcheck: + +Healthcheck +=========== + +.. automodule:: swift.common.healthcheck + :members: + :show-inheritance: + +.. _memecached: + +MemCacheD +========= + +.. automodule:: swift.common.memcached + :members: + :show-inheritance: diff --git a/doc/source/object.rst b/doc/source/object.rst new file mode 100644 index 0000000000..9a2643d4a7 --- /dev/null +++ b/doc/source/object.rst @@ -0,0 +1,46 @@ +.. _object: + +****** +Object +****** + +.. _object-server: + +Object Server +============= + +.. automodule:: swift.obj.server + :members: + :undoc-members: + :show-inheritance: + +.. _object-replicator: + +Object Replicator +================= + +.. automodule:: swift.obj.replicator + :members: + :undoc-members: + :show-inheritance: + +.. _object-updater: + +Object Updater +============== + +.. automodule:: swift.obj.updater + :members: + :undoc-members: + :show-inheritance: + +.. _object-auditor: + +Object Auditor +============== + +.. automodule:: swift.obj.auditor + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/overview_auth.rst b/doc/source/overview_auth.rst new file mode 100644 index 0000000000..b34a889411 --- /dev/null +++ b/doc/source/overview_auth.rst @@ -0,0 +1,47 @@ +=============== +The Auth System +=============== + +The auth system for Swift is based on the auth system from an existing +architecture -- actually from a few existing auth systems -- and is therefore a +bit disjointed. The distilled points about it are: + +* The authentication/authorization part is outside Swift itself +* The user of Swift passes in an auth token with each request +* Swift validates each token with the external auth system and caches the + result +* The token does not change from request to request, but does expire + +The token can be passed into Swift using the X-Auth-Token or the +X-Storage-Token header. Both have the same format: just a simple string +representing the token. Some external systems use UUID tokens, some an MD5 hash +of something unique, some use "something else" but the salient point is that +the token is a string which can be sent as-is back to the auth system for +validation. + +The validation call is, for historical reasons, an XMLRPC call. There are two +types of auth systems, type 0 and type 1. With type 0, the XMLRPC call is given +the token and the Swift account name (also known as the account hash because +it's usually of the format _). With type 1, the call is given +the container name and HTTP method as well as the token and account hash. Both +types are also given a service login and password recorded in Swift's +resellers.conf. For a valid token, both auth system types respond with a +session TTL and overall expiration in seconds from now. Swift does not honor +the session TTL but will cache the token up to the expiration time. Tokens can +be purged through a call to Swift's services server. + +How the user gets the token to use with Swift is up to the reseller software +itself. For instance, with Cloud Files the user has a starting URL to an auth +system. The user starts a session by sending a ReST request to that auth system +to receive the auth token, a URL to the Swift system, and a URL to the CDN +system. + +------------------ +History and Future +------------------ + +What's established in Swift for authentication/authorization has history from +before Swift, so that won't be recorded here. It was minimally integrated with +Swift to meet project deadlines, but in the near future Swift should have a +pluggable auth/reseller system to support the above as well as other +architectures. diff --git a/doc/source/overview_reaper.rst b/doc/source/overview_reaper.rst new file mode 100644 index 0000000000..020085dbc2 --- /dev/null +++ b/doc/source/overview_reaper.rst @@ -0,0 +1,64 @@ +================== +The Account Reaper +================== + +The Account Reaper removes data from deleted accounts in the background. + +An account is marked for deletion by a reseller through the services server's +remove_storage_account XMLRPC call. This simply puts the value DELETED into the +status column of the account_stat table in the account database (and replicas), +indicating the data for the account should be deleted later. There is no set +retention time and no undelete; it is assumed the reseller will implement such +features and only call remove_storage_account once it is truly desired the +account's data be removed. + +The account reaper runs on each account server and scans the server +occasionally for account databases marked for deletion. It will only trigger on +accounts that server is the primary node for, so that multiple account servers +aren't all trying to do the same work at the same time. Using multiple servers +to delete one account might improve deletion speed, but requires coordination +so they aren't duplicating effort. Speed really isn't as much of a concern with +data deletion and large accounts aren't deleted that often. + +The deletion process for an account itself is pretty straightforward. For each +container in the account, each object is deleted and then the container is +deleted. Any deletion requests that fail won't stop the overall process, but +will cause the overall process to fail eventually (for example, if an object +delete times out, the container won't be able to be deleted later and therefore +the account won't be deleted either). The overall process continues even on a +failure so that it doesn't get hung up reclaiming cluster space because of one +troublesome spot. The account reaper will keep trying to delete an account +until it evetually becomes empty, at which point the database reclaim process +within the db_replicator will eventually remove the database files. + +------- +History +------- + +At first, a simple approach of deleting an account through completely external +calls was considered as it required no changes to the system. All data would +simply be deleted in the same way the actual user would, through the public +ReST API. However, the downside was that it would use proxy resources and log +everything when it didn't really need to. Also, it would likely need a +dedicated server or two, just for issuing the delete requests. + +A completely bottom-up approach was also considered, where the object and +container servers would occasionally scan the data they held and check if the +account was deleted, removing the data if so. The upside was the speed of +reclamation with no impact on the proxies or logging, but the downside was that +nearly 100% of the scanning would result in no action creating a lot of I/O +load for no reason. + +A more container server centric approach was also considered, where the account +server would mark all the containers for deletion and the container servers +would delete the objects in each container and then themselves. This has the +benefit of still speedy reclamation for accounts with a lot of containers, but +has the downside of a pretty big load spike. The process could be slowed down +to alleviate the load spike possibility, but then the benefit of speedy +reclamation is lost and what's left is just a more complex process. Also, +scanning all the containers for those marked for deletion when the majority +wouldn't be seemed wasteful. The db_replicator could do this work while +performing its replication scan, but it would have to spawn and track deletion +processes which seemed needlessly complex. + +In the end, an account server centric approach seemed best, as described above. diff --git a/doc/source/overview_replication.rst b/doc/source/overview_replication.rst new file mode 100644 index 0000000000..21ba25818e --- /dev/null +++ b/doc/source/overview_replication.rst @@ -0,0 +1,40 @@ +=========== +Replication +=========== + +Since each replica in swift functions independently, and clients generally require only a simple majority of nodes responding to consider an operation successful, transient failures like network partitions can quickly cause replicas to diverge. These differences are eventually reconciled by asynchronous, peer-to-peer replicator processes. The replicator processes traverse their local filesystems, concurrently performing operations in a manner that balances load across physical disks. + +Replication uses a push model, with records and files generally only being copied from local to remote replicas. This is important because data on the node may not belong there (as in the case of handoffs and ring changes), and a replicator can't know what data exists elsewhere in the cluster that it should pull in. It's the duty of any node that contains data to ensure that data gets to where it belongs. Replica placement is handled by the ring. + +Every deleted record or file in the system is marked by a tombstone, so that deletions can be replicated alongside creations. These tombstones are cleaned up by the replication process after a period of time referred to as the consistency window, which is related to replication duration and how long transient failures can remove a node from the cluster. Tombstone cleanup must be tied to replication to reach replica convergence. + +If a replicator detects that a remote drive is has failed, it will use the ring's "get_more_nodes" interface to choose an alternate node to synchronize with. The replicator can generally maintain desired levels of replication in the face of hardware failures, though some replicas may not be in an immediately usable location. + +Replication is an area of active development, and likely rife with potential improvements to speed and correctness. + +There are two major classes of replicator - the db replicator, which replicates accounts and containers, and the object replicator, which replicates object data. + + +-------------- +DB Replication +-------------- + +The first step performed by db replication is a low-cost hash comparison to find out whether or not two replicas already match. Under normal operation, this check is able to verify that most databases in the system are already synchronized very quickly. If the hashes differ, the replicator brings the databases in sync by sharing records added since the last sync point. + +This sync point is a high water mark noting the last record at which two databases were known to be in sync, and is stored in each database as a tuple of the remote database id and record id. Database ids are unique amongst all replicas of the database, and record ids are monotonically increasing integers. After all new records have been pushed to the remote database, the entire sync table of the local database is pushed, so the remote database knows it's now in sync with everyone the local database has previously synchronized with. + +If a replica is found to be missing entirely, the whole local database file is transmitted to the peer using rsync(1) and vested with a new unique id. + +In practice, DB replication can process hundreds of databases per concurrency setting per second (up to the number of available CPUs or disks) and is bound by the number of DB transactions that must be performed. + + +------------------ +Object Replication +------------------ + +The initial implementation of object replication simply performed an rsync to push data from a local partition to all remote servers it was expected to exist on. While this performed adequately at small scale, replication times skyrocketed once directory structures could no longer be held in RAM. We now use a modification of this scheme in which a hash of the contents for each suffix directory is saved to a per-partition hashes file. The hash for a suffix directory is invalidated when the contents of that suffix directory are modified. + +The object replication process reads in these hash files, calculating any invalidated hashes. It then transmits the hashes to each remote server that should hold the partition, and only suffix directories with differing hashes on the remote server are rsynced. After pushing files to the remote server, the replication process notifies it to recalculate hashes for the rsynced suffix directories. + +Performance of object replication is generally bound by the number of uncached directories it has to traverse, usually as a result of invalidated suffix directory hashes. Using write volume and partition counts from our running systems, it was designed so that around 2% of the hash space on a normal node will be invalidated per day, which has experimentally given us acceptable replication speeds. + diff --git a/doc/source/overview_ring.rst b/doc/source/overview_ring.rst new file mode 100644 index 0000000000..ffc90d2f51 --- /dev/null +++ b/doc/source/overview_ring.rst @@ -0,0 +1,234 @@ +========= +The Rings +========= + +The rings determine where data should reside in the cluster. There is a +separate ring for account databases, container databases, and individual +objects but each ring works in the same way. These rings are externally +managed, in that the server processes themselves do not modify the rings, they +are instead given new rings modified by other tools. + +The ring uses a configurable number of bits from a path's MD5 hash as a +partition index that designates a device. The number of bits kept from the hash +is known as the partition power, and 2 to the partition power indicates the +partition count. Partitioning the full MD5 hash ring allows other parts of the +cluster to work in batches of items at once which ends up either more efficient +or at least less complex than working with each item separately or the entire +cluster all at once. + +Another configurable value is the replica count, which indicates how many of +the partition->device assignments comprise a single ring. For a given partition +number, each replica's device will not be in the same zone as any other +replica's device. Zones can be used to group devices based on physical +locations, power separations, network separations, or any other attribute that +would lessen multiple replicas being unavailable at the same time. + +------------ +Ring Builder +------------ + +The rings are built and managed manually by a utility called the ring-builder. +The ring-builder assigns partitions to devices and writes an optimized Python +structure to a gzipped, pickled file on disk for shipping out to the servers. +The server processes just check the modification time of the file occasionally +and reload their in-memory copies of the ring structure as needed. Because of +how the ring-builder manages changes to the ring, using a slightly older ring +usually just means one of the three replicas for a subset of the partitions +will be incorrect, which can be easily worked around. + +The ring-builder also keeps its own builder file with the ring information and +additional data required to build future rings. It is very important to keep +multiple backup copies of these builder files. One option is to copy the +builder files out to every server while copying the ring files themselves. +Another is to upload the builder files into the cluster itself. Complete loss +of a builder file will mean creating a new ring from scratch, nearly all +partitions will end up assigned to different devices, and therefore nearly all +data stored will have to be replicated to new locations. So, recovery from a +builder file loss is possible, but data will definitely be unreachable for an +extended time. + +------------------- +Ring Data Structure +------------------- + +The ring data structure consists of three top level fields: a list of devices +in the cluster, a list of lists of device ids indicating partition to device +assignments, and an integer indicating the number of bits to shift an MD5 hash +to calculate the partition for the hash. + +*************** +List of Devices +*************** + +The list of devices is known internally to the Ring class as devs. Each item in +the list of devices is a dictionary with the following keys: + +====== ======= ============================================================== +id integer The index into the list devices. +zone integer The zone the devices resides in. +weight float The relative weight of the device in comparison to other + devices. This usually corresponds directly to the amount of + disk space the device has compared to other devices. For + instance a device with 1 terabyte of space might have a weight + of 100.0 and another device with 2 terabytes of space might + have a weight of 200.0. This weight can also be used to bring + back into balance a device that has ended up with more or less + data than desired over time. A good average weight of 100.0 + allows flexibility in lowering the weight later if necessary. +ip string The IP address of the server containing the device. +port int The TCP port the listening server process uses that serves + requests for the device. +device string The on disk name of the device on the server. + For example: sdb1 +meta string A general-use field for storing additional information for the + device. This information isn't used directly by the server + processes, but can be useful in debugging. For example, the + date and time of installation and hardware manufacturer could + be stored here. +====== ======= ============================================================== + +Note: The list of devices may contain holes, or indexes set to None, for +devices that have been removed from the cluster. Generally, device ids are not +reused. Also, some devices may be temporarily disabled by setting their weight +to 0.0. To obtain a list of active devices (for uptime polling, for example) +the Python code would look like: ``devices = [device for device in self.devs if +device and device['weight']]`` + +************************* +Partition Assignment List +************************* + +This is a list of array('I') of devices ids. The outermost list contains an +array('I') for each replica. Each array('I') has a length equal to the +partition count for the ring. Each integer in the array('I') is an index into +the above list of devices. The partition list is known internally to the Ring +class as _replica2part2dev_id. + +So, to create a list of device dictionaries assigned to a partition, the Python +code would look like: ``devices = [self.devs[part2dev_id[partition]] for +part2dev_id in self._replica2part2dev_id]`` + +array('I') is used for memory conservation as there may be millions of +partitions. + +********************* +Partition Shift Value +********************* + +The partition shift value is known internally to the Ring class as _part_shift. +This value used to shift an MD5 hash to calculate the partition on which the +data for that hash should reside. Only the top four bytes of the hash is used +in this process. For example, to compute the partition for the path +/account/container/object the Python code might look like: ``partition = +unpack_from('>I', md5('/account/container/object').digest())[0] >> +self._part_shift`` + +----------------- +Building the Ring +----------------- + +The initial building of the ring first calculates the number of partitions that +should ideally be assigned to each device based the device's weight. For +example, if the partition power of 20 the ring will have 1,048,576 partitions. +If there are 1,000 devices of equal weight they will each desire 1,048.576 +partitions. The devices are then sorted by the number of partitions they desire +and kept in order throughout the initialization process. + +Then, the ring builder assigns each partition's replica to the device that +desires the most partitions at that point, with the restriction that the device +is not in the same zone as any other replica for that partition. Once assigned, +the device's desired partition count is decremented and moved to its new sorted +location in the list of devices and the process continues. + +When building a new ring based on an old ring, the desired number of partitions +each device wants is recalculated. Next the partitions to be reassigned are +gathered up. Any removed devices have all their assigned partitions unassigned +and added to the gathered list. Any devices that have more partitions than they +now desire have random partitions unassigned from them and added to the +gathered list. Lastly, the gathered partitions are then reassigned to devices +using a similar method as in the initial assignment described above. + +Whenever a partition has a replica reassigned, the time of the reassignment is +recorded. This is taken into account when gathering partitions to reassign so +that no partition is moved twice in a configurable amount of time. This +configurable amount of time is known internally to the RingBuilder class as +min_part_hours. This restriction is ignored for replicas of partitions on +devices that have been removed, as removing a device only happens on device +failure and there's no choice but to make a reassignment. + +The above processes don't always perfectly rebalance a ring due to the random +nature of gathering partitions for reassignment. To help reach a more balanced +ring, the rebalance process is repeated until near perfect (less 1% off) or +when the balance doesn't improve by at least 1% (indicating we probably can't +get perfect balance due to wildly imbalanced zones or too many partitions +recently moved). + +------- +History +------- + +The ring code went through many iterations before arriving at what it is now +and while it has been stable for a while now, the algorithm may be tweaked or +perhaps even fundamentally changed if new ideas emerge. This section will try +to describe the previous ideas attempted and attempt to explain why they were +discarded. + +A "live ring" option was considered where each server could maintain its own +copy of the ring and the servers would use a gossip protocol to communicate the +changes they made. This was discarded as too complex and error prone to code +correctly in the project time span available. One bug could easily gossip bad +data out to the entire cluster and be difficult to recover from. Having an +externally managed ring simplifies the process, allows full validation of data +before it's shipped out to the servers, and guarantees each server is using a +ring from the same timeline. It also means that the servers themselves aren't +spending a lot of resources maintaining rings. + +A couple of "ring server" options were considered. One was where all ring +lookups would be done by calling a service on a separate server or set of +servers, but this was discarded due to the latency involved. Another was much +like the current process but where servers could submit change requests to the +ring server to have a new ring built and shipped back out to the servers. This +was discarded due to project time constraints and because ring changes are +currently infrequent enough that manual control was sufficient. However, lack +of quick automatic ring changes did mean that other parts of the system had to +be coded to handle devices being unavailable for a period of hours until +someone could manually update the ring. + +The current ring process has each replica of a partition independently assigned +to a device. A version of the ring that used a third of the memory was tried, +where the first replica of a partition was directly assigned and the other two +were determined by "walking" the ring until finding additional devices in other +zones. This was discarded as control was lost as to how many replicas for a +given partition moved at once. Keeping each replica independent allows for +moving only one partition replica within a given time window (except due to +device failures). Using the additional memory was deemed a good tradeoff for +moving data around the cluster much less often. + +Another ring design was tried where the partition to device assignments weren't +stored in a big list in memory but instead each device was assigned a set of +hashes, or anchors. The partition would be determined from the data item's hash +and the nearest device anchors would determine where the replicas should be +stored. However, to get reasonable distribution of data each device had to have +a lot of anchors and walking through those anchors to find replicas started to +add up. In the end, the memory savings wasn't that great and more processing +power was used, so the idea was discarded. + +A completely non-partitioned ring was also tried but discarded as the +partitioning helps many other parts of the system, especially replication. +Replication can be attempted and retried in a partition batch with the other +replicas rather than each data item independently attempted and retried. Hashes +of directory structures can be calculated and compared with other replicas to +reduce directory walking and network traffic. + +Partitioning and independently assigning partition replicas also allowed for +the best balanced cluster. The best of the other strategies tended to give ++-10% variance on device balance with devices of equal weight and +-15% with +devices of varying weights. The current strategy allows us to get +-3% and +-8% +respectively. + +Various hashing algorithms were tried. SHA offers better security, but the ring +doesn't need to be cryptographically secure and SHA is slower. Murmur was much +faster, but MD5 was built-in and hash computation is a small percentage of the +overall request handling time. In all, once it was decided the servers wouldn't +be maintaining the rings themselves anyway and only doing hash lookups, MD5 was +chosen for its general availability, good distribution, and adequate speed. diff --git a/doc/source/proxy.rst b/doc/source/proxy.rst new file mode 100644 index 0000000000..210480d7eb --- /dev/null +++ b/doc/source/proxy.rst @@ -0,0 +1,15 @@ +.. _proxy: + +***** +Proxy +***** + +.. _proxy-server: + +Proxy Server +============ + +.. automodule:: swift.proxy.server + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/ring.rst b/doc/source/ring.rst new file mode 100644 index 0000000000..d8f5a611f4 --- /dev/null +++ b/doc/source/ring.rst @@ -0,0 +1,25 @@ +.. _consistent_hashing_ring: + +******************************** +Partitioned Consistent Hash Ring +******************************** + +.. _ring: + +Ring +==== + +.. automodule:: swift.common.ring.ring + :members: + :undoc-members: + :show-inheritance: + +.. _ring-builder: + +Ring Builder +============ + +.. automodule:: swift.common.ring.builder + :members: + :undoc-members: + :show-inheritance: diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample new file mode 100644 index 0000000000..3ded523ee2 --- /dev/null +++ b/etc/account-server.conf-sample @@ -0,0 +1,51 @@ +[account-server] +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true +# bind_ip = 0.0.0.0 +# bind_port = 6002 +# workers = 1 +# log_facility = LOG_LOCAL0 +# log_level = INFO +# user = swift + +[account-replicator] +# log_facility = LOG_LOCAL0 +# log_level = INFO +# per_diff = 1000 +# concurrency = 8 +# run_pause = 30 +# How long without an error before a node's error count is reset. This will +# also be how long before a node is reenabled after suppression is triggered. +# error_suppression_interval = 60 +# How many errors can accumulate before a node is temporarily ignored. +# error_suppression_limit = 10 +# node_timeout = 10 +# conn_timeout = 0.5 +# The replicator also performs reclamation +# reclaim_age = 86400 + +[account-stats] +# cf_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31 +# container_name = account_stats +# proxy_server_conf = /etc/swift/proxy-server.conf +# log_facility = LOG_LOCAL0 +# log_level = INFO + +[account-auditor] +# Will audit, at most, 1 account per device per interval +# interval = 1800 +# Maximum containers randomly picked for a given account audit +# max_container_count = 100 +# node_timeout = 10 +# conn_timeout = 0.5 +# log_facility = LOG_LOCAL0 +# log_level = INFO + +[account-reaper] +# concurrency = 25 +# interval = 3600 +# node_timeout = 10 +# conn_timeout = 0.5 +# log_facility = LOG_LOCAL0 +# log_level = INFO diff --git a/etc/auth-server.conf-sample b/etc/auth-server.conf-sample new file mode 100644 index 0000000000..29bcc63f1a --- /dev/null +++ b/etc/auth-server.conf-sample @@ -0,0 +1,15 @@ +[auth-server] +# swift_dir = /etc/swift +# bind_ip = 0.0.0.0 +# bind_port = 11000 +# log_facility = LOG_LOCAL0 +# log_level = INFO +# workers = 1 +# reseller_prefix = AUTH +# default_cluster_url = http://127.0.0.1:9000/v1 +# token_life = 86400 +# log_headers = False +# cert_file = Default is no cert; format is path like /etc/swift/auth.crt +# key_file = Default is no key; format is path like /etc/swift/auth.key +# node_timeout = 10 +user = swift diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample new file mode 100644 index 0000000000..086d4d8a0f --- /dev/null +++ b/etc/container-server.conf-sample @@ -0,0 +1,43 @@ +[container-server] +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true +# bind_ip = 0.0.0.0 +# bind_port = 6001 +# workers = 1 +# log_facility = LOG_LOCAL0 +# log_level = INFO +# user = swift +# node_timeout = 3 +# conn_timeout = 0.5 + +[container-replicator] +# log_facility = LOG_LOCAL0 +# log_level = INFO +# per_diff = 1000 +# concurrency = 8 +# run_pause = 30 +# node_timeout = 10 +# conn_timeout = 0.5 +# The replicator also performs reclamation +# reclaim_age = 604800 + +[container-updater] +# interval = 300 +# concurrency = 4 +# node_timeout = 3 +# conn_timeout = 0.5 +# slowdown will sleep that amount between containers +# slowdown = 0.01 +# log_facility = LOG_LOCAL0 +# log_level = INFO + +[container-auditor] +# Will audit, at most, 1 container per device per interval +# interval = 1800 +# Maximum objects randomly picked for a given container audit +# max_object_count = 100 +# node_timeout = 10 +# conn_timeout = 0.5 +# log_facility = LOG_LOCAL0 +# log_level = INFO diff --git a/etc/drive-audit.conf-sample b/etc/drive-audit.conf-sample new file mode 100644 index 0000000000..9ffa94760c --- /dev/null +++ b/etc/drive-audit.conf-sample @@ -0,0 +1,6 @@ +[drive-audit] +# device_dir = /srv/node +# log_facility = LOG_LOCAL0 +# log_level = INFO +# minutes = 60 +# error_limit = 1 diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample new file mode 100644 index 0000000000..cce9ce39a1 --- /dev/null +++ b/etc/object-server.conf-sample @@ -0,0 +1,46 @@ +[object-server] +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true +# bind_ip = 0.0.0.0 +# bind_port = 6000 +# workers = 1 +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_requests = True +# user = swift +# node_timeout = 3 +# conn_timeout = 0.5 +# network_chunk_size = 8192 +# disk_chunk_size = 32768 +# max_upload_time = 86400 +# slow = 1 + +[object-replicator] +# log_facility = LOG_LOCAL0 +# log_level = INFO +# daemonize = on +# run_pause = 30 +# concurrency = 1 +# timeout = 300 +# stats_interval = 3600 +# The replicator also performs reclamation +# reclaim_age = 604800 + +[object-updater] +# interval = 300 +# concurrency = 1 +# node_timeout = 10 +# conn_timeout = 0.5 +# slowdown will sleep that amount between objects +# slowdown = 0.01 +# log_facility = LOG_LOCAL0 +# log_level = INFO + +[object-auditor] +# Will audit, at most, 1 object per device per interval +# interval = 1800 +# node_timeout = 10 +# conn_timeout = 0.5 +# log_facility = LOG_LOCAL0 +# log_level = INFO diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample new file mode 100644 index 0000000000..7b9fb99d74 --- /dev/null +++ b/etc/proxy-server.conf-sample @@ -0,0 +1,35 @@ +[proxy-server] +# bind_ip = 0.0.0.0 +# bind_port = 80 +# cert_file = /etc/swift/proxy.crt +# key_file = /etc/swift/proxy.key +# swift_dir = /etc/swift +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_headers = False +# workers = 1 +# user = swift +# recheck_account_existence = 60 +# recheck_container_existence = 60 +# object_chunk_size = 8192 +# container_chunk_size = 8192 +# account_chunk_size = 8192 +# client_chunk_size = 8192 +# Default for memcache_servers is below, but you can specify multiple servers +# with the format: 10.1.2.3:11211,10.1.2.4:11211 +# memcache_servers = 127.0.0.1:11211 +# node_timeout = 10 +# client_timeout = 60 +# conn_timeout = 0.5 +# How long without an error before a node's error count is reset. This will +# also be how long before a node is reenabled after suppression is triggered. +# error_suppression_interval = 60 +# How many errors can accumulate before a node is temporarily ignored. +# error_suppression_limit = 10 +# How many ops per second to one container (as a float) +# rate_limit = 20000.0 +# How many ops per second for account-level operations +# account_rate_limit = 200.0 +# rate_limit_account_whitelist = acct1,acct2,etc +# rate_limit_account_blacklist = acct3,acct4,etc +# container_put_lock_timeout = 5 diff --git a/etc/rsyncd.conf-sample b/etc/rsyncd.conf-sample new file mode 100644 index 0000000000..2f0c9a84e2 --- /dev/null +++ b/etc/rsyncd.conf-sample @@ -0,0 +1,19 @@ +uid = swift +gid = swift +log file = /var/log/rsyncd.log +pid file = /var/run/rsyncd.pid + +[account] +max connections = 2 +path = /srv/node +read only = false + +[container] +max connections = 4 +path = /srv/node +read only = false + +[object] +max connections = 8 +path = /srv/node +read only = false diff --git a/etc/stats.conf-sample b/etc/stats.conf-sample new file mode 100644 index 0000000000..8ec18d4968 --- /dev/null +++ b/etc/stats.conf-sample @@ -0,0 +1,12 @@ +[stats] +auth_url = http://saio:11000/auth +auth_user = test:tester +auth_key = testing +# swift_dir = /etc/swift +# dispersion_coverage = 1 +# container_put_count = 1000 +# object_put_count = 1000 +# big_container_count = 1000000 +# retries = 5 +# concurrency = 50 +# csv_output = /etc/swift/stats.csv diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..3622960963 --- /dev/null +++ b/setup.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distutils.core import setup + +setup( + name='swift', + version='1.0.0-1', + description='Swift', + license='Apache License (2.0)' + author='OpenStack, LLC.', + url='https://launchpad.net/swift', + packages=['swift', 'swift.common'], + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.6', + 'Environment :: No Input/Output (Daemon)', + ], + scripts=['bin/st.py', 'bin/swift-account-auditor.py', + 'bin/swift-account-audit.py', 'bin/swift-account-reaper.py', + 'bin/swift-account-replicator.py', 'bin/swift-account-server.py', + 'bin/swift-auth-create-account.py', + 'bin/swift-auth-recreate-accounts.py', 'bin/swift-auth-server.py', + 'bin/swift-container-auditor.py', + 'bin/swift-container-replicator.py', + 'bin/swift-container-server.py', 'bin/swift-container-updater.py', + 'bin/swift-drive-audit.py', 'bin/swift-get-nodes.py', + 'bin/swift-init.py', 'bin/swift-object-auditor.py', + 'bin/swift-object-info.py', 'bin/swift-object-server.py', + 'bin/swift-object-updater.py', 'bin/swift-proxy-server.py', + 'bin/swift-ring-builder.py', 'bin/swift-stats-populate.py', + 'bin/swift-stats-report.py'] +) diff --git a/swift/__init__.py b/swift/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/swift/account/__init__.py b/swift/account/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/swift/account/auditor.py b/swift/account/auditor.py new file mode 100644 index 0000000000..94eb5523bf --- /dev/null +++ b/swift/account/auditor.py @@ -0,0 +1,194 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import socket +import time +from random import choice, random +from urllib import quote + +from eventlet import Timeout + +from swift.account import server as account_server +from swift.common.db import AccountBroker +from swift.common.bufferedhttp import http_connect +from swift.common.exceptions import ConnectionTimeout +from swift.common.ring import Ring +from swift.common.utils import get_logger + + +class AuditException(Exception): + pass + + +class AccountAuditor(object): + """Audit accounts.""" + + def __init__(self, server_conf, auditor_conf): + self.logger = get_logger(auditor_conf, 'account-auditor') + self.devices = server_conf.get('devices', '/srv/node') + self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.interval = int(auditor_conf.get('interval', 1800)) + swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') + self.container_ring = None + self.node_timeout = int(auditor_conf.get('node_timeout', 10)) + self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) + self.max_container_count = \ + int(auditor_conf.get('max_container_count', 100)) + self.container_passes = 0 + self.container_failures = 0 + self.container_errors = 0 + + def get_container_ring(self): + """ + Get the container ring. Load the ring if neccesary. + + :returns: container ring + """ + if not self.container_ring: + self.logger.debug( + + 'Loading container ring from %s' % self.container_ring_path) + self.container_ring = Ring(self.container_ring_path) + return self.container_ring + + def audit_forever(self): # pragma: no cover + """Run the account audit until stopped.""" + reported = time.time() + time.sleep(random() * self.interval) + while True: + begin = time.time() + pids = [] + for device in os.listdir(self.devices): + if self.mount_check and not \ + os.path.ismount(os.path.join(self.devices, device)): + self.logger.debug( + 'Skipping %s as it is not mounted' % device) + continue + self.account_audit(device) + if time.time() - reported >= 3600: # once an hour + self.logger.info( + 'Since %s: Remote audits with containers: %s passed ' + 'audit, %s failed audit, %s errors' % + (time.ctime(reported), self.container_passes, + self.container_failures, self.container_errors)) + reported = time.time() + self.container_passes = 0 + self.container_failures = 0 + self.container_errors = 0 + elapsed = time.time() - begin + if elapsed < self.interval: + time.sleep(self.interval - elapsed) + + def audit_once(self): + """Run the account audit once.""" + self.logger.info('Begin account audit "once" mode') + begin = time.time() + for device in os.listdir(self.devices): + if self.mount_check and \ + not os.path.ismount(os.path.join(self.devices, device)): + self.logger.debug( + 'Skipping %s as it is not mounted' % device) + continue + self.account_audit(device) + elapsed = time.time() - begin + self.logger.info( + 'Account audit "once" mode completed: %.02fs' % elapsed) + + def account_audit(self, device): + """ + Audit any accounts found on the device. + + :param device: device to audit + """ + datadir = os.path.join(self.devices, device, account_server.DATADIR) + if not os.path.exists(datadir): + return + broker = None + partition = None + attempts = 100 + while not broker and attempts: + attempts -= 1 + try: + partition = choice(os.listdir(datadir)) + fpath = os.path.join(datadir, partition) + if not os.path.isdir(fpath): + continue + suffix = choice(os.listdir(fpath)) + fpath = os.path.join(fpath, suffix) + if not os.path.isdir(fpath): + continue + hsh = choice(os.listdir(fpath)) + fpath = os.path.join(fpath, hsh) + if not os.path.isdir(fpath): + continue + except IndexError: + continue + for fname in sorted(os.listdir(fpath), reverse=True): + if fname.endswith('.db'): + broker = AccountBroker(os.path.join(fpath, fname)) + if broker.is_deleted(): + broker = None + break + if not broker: + return + info = broker.get_info() + for container in broker.get_random_containers( + max_count=self.max_container_count): + found = False + results = [] + part, nodes = \ + self.get_container_ring().get_nodes(info['account'], container) + for node in nodes: + try: + with ConnectionTimeout(self.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], part, 'HEAD', + '/%s/%s' % (info['account'], container)) + with Timeout(self.node_timeout): + resp = conn.getresponse() + body = resp.read() + if 200 <= resp.status <= 299: + found = True + break + else: + results.append('%s:%s/%s %s %s' % (node['ip'], + node['port'], node['device'], resp.status, + resp.reason)) + except socket.error, err: + results.append('%s:%s/%s Socket Error: %s' % (node['ip'], + node['port'], node['device'], err)) + except ConnectionTimeout: + results.append( + '%(ip)s:%(port)s/%(device)s ConnectionTimeout' % node) + except Timeout: + results.append('%(ip)s:%(port)s/%(device)s Timeout' % node) + except Exception, err: + self.logger.exception('ERROR With remote server ' + '%(ip)s:%(port)s/%(device)s' % node) + results.append('%s:%s/%s Exception: %s' % (node['ip'], + node['port'], node['device'], err)) + if found: + self.container_passes += 1 + self.logger.debug('Audit passed for /%s %s container %s' % + (info['account'], broker.db_file, container)) + else: + self.container_errors += 1 + self.logger.error('ERROR Could not find container /%s/%s ' + 'referenced by %s on any of the primary container ' + 'servers it should be on: %s' % (info['account'], + container, broker.db_file, results)) diff --git a/swift/account/reaper.py b/swift/account/reaper.py new file mode 100644 index 0000000000..0d91fbfae0 --- /dev/null +++ b/swift/account/reaper.py @@ -0,0 +1,407 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random +from logging import DEBUG +from math import sqrt +from time import time + +from eventlet import GreenPool, sleep + +from swift.account.server import DATADIR +from swift.common.db import AccountBroker +from swift.common.direct_client import ClientException, \ + direct_delete_container, direct_delete_object, direct_get_container +from swift.common.ring import Ring +from swift.common.utils import get_logger, whataremyips + + +class AccountReaper(object): + """ + Removes data from status=DELETED accounts. These are accounts that have + been asked to be removed by the reseller via services + remove_storage_account XMLRPC call. + + The account is not deleted immediately by the services call, but instead + the account is simply marked for deletion by setting the status column in + the account_stat table of the account database. This account reaper scans + for such accounts and removes the data in the background. The background + deletion process will occur on the primary account server for the account. + + :param server_conf: The [account-server] dictionary of the account server + configuration file + :param reaper_conf: The [account-reaper] dictionary of the account server + configuration file + + See the etc/account-server.conf-sample for information on the possible + configuration parameters. + """ + + log_name = 'account-reaper' + + def __init__(self, server_conf, reaper_conf): + self.logger = get_logger(reaper_conf, self.log_name) + self.devices = server_conf.get('devices', '/srv/node') + self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.interval = int(reaper_conf.get('interval', 3600)) + swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') + self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') + self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') + self.account_ring = None + self.container_ring = None + self.object_ring = None + self.node_timeout = int(reaper_conf.get('node_timeout', 10)) + self.conn_timeout = float(reaper_conf.get('conn_timeout', 0.5)) + self.myips = whataremyips() + self.concurrency = int(reaper_conf.get('concurrency', 25)) + self.container_concurrency = self.object_concurrency = \ + sqrt(self.concurrency) + self.container_pool = GreenPool(size=self.container_concurrency) + + def get_account_ring(self): + """ The account :class:`swift.common.ring.Ring` for the cluster. """ + if not self.account_ring: + self.logger.debug( + 'Loading account ring from %s' % self.account_ring_path) + self.account_ring = Ring(self.account_ring_path) + return self.account_ring + + def get_container_ring(self): + """ The container :class:`swift.common.ring.Ring` for the cluster. """ + if not self.container_ring: + self.logger.debug( + 'Loading container ring from %s' % self.container_ring_path) + self.container_ring = Ring(self.container_ring_path) + return self.container_ring + + def get_object_ring(self): + """ The object :class:`swift.common.ring.Ring` for the cluster. """ + if not self.object_ring: + self.logger.debug( + 'Loading object ring from %s' % self.object_ring_path) + self.object_ring = Ring(self.object_ring_path) + return self.object_ring + + def reap_forever(self): + """ + Main entry point when running the reaper in its normal daemon mode. + This repeatedly calls :func:`reap_once` no quicker than the + configuration interval. + """ + self.logger.debug('Daemon started.') + sleep(random.random() * self.interval) + while True: + begin = time() + self.reap_once() + elapsed = time() - begin + if elapsed < self.interval: + sleep(self.interval - elapsed) + + def reap_once(self): + """ + Main entry point when running the reaper in 'once' mode, where it will + do a single pass over all accounts on the server. This is called + repeatedly by :func:`reap_forever`. This will call :func:`reap_device` + once for each device on the server. + """ + self.logger.debug('Begin devices pass: %s' % self.devices) + begin = time() + for device in os.listdir(self.devices): + if self.mount_check and \ + not os.path.ismount(os.path.join(self.devices, device)): + self.logger.debug( + 'Skipping %s as it is not mounted' % device) + continue + self.reap_device(device) + elapsed = time() - begin + self.logger.info('Devices pass completed: %.02fs' % elapsed) + + def reap_device(self, device): + """ + Called once per pass for each device on the server. This will scan the + accounts directory for the device, looking for partitions this device + is the primary for, then looking for account databases that are marked + status=DELETED and still have containers and calling + :func:`reap_account`. Account databases marked status=DELETED that no + longer have containers will eventually be permanently removed by the + reclaim process within the account replicator (see + :mod:`swift.db_replicator`). + + :param device: The device to look for accounts to be deleted. + """ + datadir = os.path.join(self.devices, device, DATADIR) + if not os.path.exists(datadir): + return + for partition in os.listdir(datadir): + partition_path = os.path.join(datadir, partition) + if not partition.isdigit(): + continue + nodes = self.get_account_ring().get_part_nodes(int(partition)) + if nodes[0]['ip'] not in self.myips or \ + not os.path.isdir(partition_path): + continue + for suffix in os.listdir(partition_path): + suffix_path = os.path.join(partition_path, suffix) + if not os.path.isdir(suffix_path): + continue + for hsh in os.listdir(suffix_path): + hsh_path = os.path.join(suffix_path, hsh) + if not os.path.isdir(hsh_path): + continue + for fname in sorted(os.listdir(hsh_path), reverse=True): + if fname.endswith('.ts'): + break + elif fname.endswith('.db'): + broker = \ + AccountBroker(os.path.join(hsh_path, fname)) + if broker.is_status_deleted() and \ + not broker.empty(): + self.reap_account(broker, partition, nodes) + + def reap_account(self, broker, partition, nodes): + """ + Called once per pass for each account this server is the primary for + and attempts to delete the data for the given account. The reaper will + only delete one account at any given time. It will call + :func:`reap_container` up to sqrt(self.concurrency) times concurrently + while reaping the account. + + If there is any exception while deleting a single container, the + process will continue for any other containers and the failed + containers will be tried again the next time this function is called + with the same parameters. + + If there is any exception while listing the containers for deletion, + the process will stop (but will obviously be tried again the next time + this function is called with the same parameters). This isn't likely + since the listing comes from the local database. + + After the process completes (successfully or not) statistics about what + was accomplished will be logged. + + This function returns nothing and should raise no exception but only + update various self.stats_* values for what occurs. + + :param broker: The AccountBroker for the account to delete. + :param partition: The partition in the account ring the account is on. + :param nodes: The primary node dicts for the account to delete. + + * See also: :class:`swift.common.db.AccountBroker` for the broker class. + * See also: :func:`swift.common.ring.Ring.get_nodes` for a description + of the node dicts. + """ + begin = time() + account = broker.get_info()['account'] + self.logger.info('Beginning pass on account %s' % account) + self.stats_return_codes = {} + self.stats_containers_deleted = 0 + self.stats_objects_deleted = 0 + self.stats_containers_remaining = 0 + self.stats_objects_remaining = 0 + self.stats_containers_possibly_remaining = 0 + self.stats_objects_possibly_remaining = 0 + try: + marker = '' + while True: + containers = \ + list(broker.list_containers_iter(1000, marker, None, None)) + if not containers: + break + try: + for (container, _, _, _) in containers: + self.container_pool.spawn(self.reap_container, account, + partition, nodes, container) + self.container_pool.waitall() + except Exception: + self.logger.exception( + 'Exception with containers for account %s' % account) + marker = containers[-1][0] + log = 'Completed pass on account %s' % account + except Exception: + self.logger.exception( + 'Exception with account %s' % account) + log = 'Incomplete pass on account %s' % account + if self.stats_containers_deleted: + log += ', %s containers deleted' % self.stats_containers_deleted + if self.stats_objects_deleted: + log += ', %s objects deleted' % self.stats_objects_deleted + if self.stats_containers_remaining: + log += ', %s containers remaining' % self.stats_containers_remaining + if self.stats_objects_remaining: + log += ', %s objects remaining' % self.stats_objects_remaining + if self.stats_containers_possibly_remaining: + log += ', %s containers possibly remaining' % \ + self.stats_containers_possibly_remaining + if self.stats_objects_possibly_remaining: + log += ', %s objects possibly remaining' % \ + self.stats_objects_possibly_remaining + if self.stats_return_codes: + log += ', return codes: ' + for code in sorted(self.stats_return_codes.keys()): + log += '%s %sxxs, ' % (self.stats_return_codes[code], code) + log = log[:-2] + log += ', elapsed: %.02fs' % (time() - begin) + self.logger.info(log) + + def reap_container(self, account, account_partition, account_nodes, + container): + """ + Deletes the data and the container itself for the given container. This + will call :func:`reap_object` up to sqrt(self.concurrency) times + concurrently for the objects in the container. + + If there is any exception while deleting a single object, the process + will continue for any other objects in the container and the failed + objects will be tried again the next time this function is called with + the same parameters. + + If there is any exception while listing the objects for deletion, the + process will stop (but will obviously be tried again the next time this + function is called with the same parameters). This is a possibility + since the listing comes from querying just the primary remote container + server. + + Once all objects have been attempted to be deleted, the container + itself will be attempted to be deleted by sending a delete request to + all container nodes. The format of the delete request is such that each + container server will update a corresponding account server, removing + the container from the account's listing. + + This function returns nothing and should raise no exception but only + update various self.stats_* values for what occurs. + + :param account: The name of the account for the container. + :param account_partition: The partition for the account on the account + ring. + :param account_nodes: The primary node dicts for the account. + :param container: The name of the container to delete. + + * See also: :func:`swift.common.ring.Ring.get_nodes` for a description + of the account node dicts. + """ + account_nodes = list(account_nodes) + part, nodes = self.get_container_ring().get_nodes(account, container) + node = nodes[-1] + pool = GreenPool(size=self.object_concurrency) + marker = '' + while True: + objects = None + try: + objects = direct_get_container(node, part, account, container, + marker=marker, conn_timeout=self.conn_timeout, + response_timeout=self.node_timeout) + self.stats_return_codes[2] = \ + self.stats_return_codes.get(2, 0) + 1 + except ClientException, err: + if self.logger.getEffectiveLevel() <= DEBUG: + self.logger.exception( + 'Exception with %(ip)s:%(port)s/%(device)s' % node) + self.stats_return_codes[err.http_status / 100] = \ + self.stats_return_codes.get(err.http_status / 100, 0) + 1 + if not objects: + break + try: + for obj in objects: + if isinstance(obj['name'], unicode): + obj['name'] = obj['name'].encode('utf8') + pool.spawn(self.reap_object, account, container, part, + nodes, obj['name']) + pool.waitall() + except Exception: + self.logger.exception('Exception with objects for container ' + '%s for account %s' % (container, account)) + marker = objects[-1]['name'] + successes = 0 + failures = 0 + for node in nodes: + anode = account_nodes.pop() + try: + direct_delete_container(node, part, account, container, + conn_timeout=self.conn_timeout, + response_timeout=self.node_timeout, + headers={'X-Account-Host': '%(ip)s:%(port)s' % anode, + 'X-Account-Partition': str(account_partition), + 'X-Account-Device': anode['device'], + 'X-Account-Override-Deleted': 'yes'}) + successes += 1 + self.stats_return_codes[2] = \ + self.stats_return_codes.get(2, 0) + 1 + except ClientException, err: + if self.logger.getEffectiveLevel() <= DEBUG: + self.logger.exception( + 'Exception with %(ip)s:%(port)s/%(device)s' % node) + failures += 1 + self.stats_return_codes[err.http_status / 100] = \ + self.stats_return_codes.get(err.http_status / 100, 0) + 1 + if successes > failures: + self.stats_containers_deleted += 1 + elif not successes: + self.stats_containers_remaining += 1 + else: + self.stats_containers_possibly_remaining += 1 + + def reap_object(self, account, container, container_partition, + container_nodes, obj): + """ + Deletes the given object by issuing a delete request to each node for + the object. The format of the delete request is such that each object + server will update a corresponding container server, removing the + object from the container's listing. + + This function returns nothing and should raise no exception but only + update various self.stats_* values for what occurs. + + :param account: The name of the account for the object. + :param container: The name of the container for the object. + :param container_partition: The partition for the container on the + container ring. + :param container_nodes: The primary node dicts for the container. + :param obj: The name of the object to delete. + + * See also: :func:`swift.common.ring.Ring.get_nodes` for a description + of the container node dicts. + """ + container_nodes = list(container_nodes) + part, nodes = self.get_object_ring().get_nodes(account, container, obj) + successes = 0 + failures = 0 + for node in nodes: + cnode = container_nodes.pop() + try: + direct_delete_object(node, part, account, container, obj, + conn_timeout=self.conn_timeout, + response_timeout=self.node_timeout, + headers={'X-Container-Host': '%(ip)s:%(port)s' % cnode, + 'X-Container-Partition': str(container_partition), + 'X-Container-Device': cnode['device']}) + successes += 1 + self.stats_return_codes[2] = \ + self.stats_return_codes.get(2, 0) + 1 + except ClientException, err: + if self.logger.getEffectiveLevel() <= DEBUG: + self.logger.exception( + 'Exception with %(ip)s:%(port)s/%(device)s' % node) + failures += 1 + self.stats_return_codes[err.http_status / 100] = \ + self.stats_return_codes.get(err.http_status / 100, 0) + 1 + if successes > failures: + self.stats_objects_deleted += 1 + elif not successes: + self.stats_objects_remaining += 1 + else: + self.stats_objects_possibly_remaining += 1 diff --git a/swift/account/server.py b/swift/account/server.py new file mode 100644 index 0000000000..9654670cad --- /dev/null +++ b/swift/account/server.py @@ -0,0 +1,295 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import errno +import os +import time +import traceback +from datetime import datetime +from urllib import unquote +from swift.common.utils import get_logger + +import sqlite3 +from webob import Request, Response +from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ + HTTPCreated, HTTPForbidden, HTTPInternalServerError, \ + HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, HTTPPreconditionFailed +import simplejson +from xml.sax import saxutils + +from swift.common import ACCOUNT_LISTING_LIMIT +from swift.common.db import AccountBroker +from swift.common.exceptions import MessageTimeout +from swift.common.utils import get_param, split_path, storage_directory, \ + hash_path +from swift.common.constraints import check_mount, check_float, \ + check_xml_encodable +from swift.common.healthcheck import healthcheck +from swift.common.db_replicator import ReplicatorRpc + + +DATADIR = 'accounts' + + +class AccountController(object): + """WSGI controller for the account server.""" + log_name = 'account' + + def __init__(self, conf): + self.logger = get_logger(conf, self.log_name) + self.root = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.replicator_rpc = \ + ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check) + + def _get_account_broker(self, drive, part, account): + hsh = hash_path(account) + db_dir = storage_directory(DATADIR, part, hsh) + db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') + return AccountBroker(db_path, account=account, logger=self.logger) + + def DELETE(self, req): + """Handle HTTP DELETE request.""" + try: + drive, part, account = split_path(unquote(req.path), 3) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + if 'x-timestamp' not in req.headers or \ + not check_float(req.headers['x-timestamp']): + return HTTPBadRequest(body='Missing timestamp', request=req, + content_type='text/plain') + broker = self._get_account_broker(drive, part, account) + if broker.is_deleted(): + return HTTPNotFound(request=req) + broker.delete_db(req.headers['x-timestamp']) + return HTTPNoContent(request=req) + + def PUT(self, req): + """Handle HTTP PUT request.""" + drive, part, account, container = split_path(unquote(req.path), 3, 4) + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + broker = self._get_account_broker(drive, part, account) + if container: # put account container + if 'x-cf-trans-id' in req.headers: + broker.pending_timeout = 3 + if req.headers.get('x-account-override-deleted', 'no').lower() != \ + 'yes' and broker.is_deleted(): + return HTTPNotFound(request=req) + broker.put_container(container, req.headers['x-put-timestamp'], + req.headers['x-delete-timestamp'], + req.headers['x-object-count'], + req.headers['x-bytes-used']) + if req.headers['x-delete-timestamp'] > \ + req.headers['x-put-timestamp']: + return HTTPNoContent(request=req) + else: + return HTTPCreated(request=req) + else: # put account + if not os.path.exists(broker.db_file): + broker.initialize(req.headers['x-timestamp']) + return HTTPCreated(request=req) + elif broker.is_status_deleted(): + return HTTPForbidden(request=req, body='Recently deleted') + else: + broker.update_put_timestamp(req.headers['x-timestamp']) + return HTTPAccepted(request=req) + + def HEAD(self, req): + """Handle HTTP HEAD request.""" + # TODO: Refactor: The account server used to provide a 'account and + # container existence check all-in-one' call by doing a HEAD with a + # container path. However, container existence is now checked with the + # container servers directly so this is no longer needed. We should + # refactor out the container existence check here and retest + # everything. + try: + drive, part, account, container = split_path(unquote(req.path), 3, 4) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + broker = self._get_account_broker(drive, part, account) + if not container: + broker.pending_timeout = 0.1 + broker.stale_reads_ok = True + if broker.is_deleted(): + return HTTPNotFound(request=req) + info = broker.get_info() + headers = { + 'X-Account-Container-Count': info['container_count'], + 'X-Account-Object-Count': info['object_count'], + 'X-Account-Bytes-Used': info['bytes_used'], + 'X-Timestamp': info['created_at'], + 'X-PUT-Timestamp': info['put_timestamp'], + } + if container: + container_ts = broker.get_container_timestamp(container) + if container_ts is not None: + headers['X-Container-Timestamp'] = container_ts + return HTTPNoContent(request=req, headers=headers) + + def GET(self, req): + """Handle HTTP GET request.""" + try: + drive, part, account = split_path(unquote(req.path), 3) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + broker = self._get_account_broker(drive, part, account) + broker.pending_timeout = 0.1 + broker.stale_reads_ok = True + if broker.is_deleted(): + return HTTPNotFound(request=req) + info = broker.get_info() + resp_headers = { + 'X-Account-Container-Count': info['container_count'], + 'X-Account-Object-Count': info['object_count'], + 'X-Account-Bytes-Used': info['bytes_used'], + 'X-Timestamp': info['created_at'], + 'X-PUT-Timestamp': info['put_timestamp'] + } + try: + prefix = get_param(req, 'prefix') + delimiter = get_param(req, 'delimiter') + if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): + # delimiters can be made more flexible later + return HTTPPreconditionFailed(body='Bad delimiter') + limit = ACCOUNT_LISTING_LIMIT + given_limit = get_param(req, 'limit') + if given_limit and given_limit.isdigit(): + limit = int(given_limit) + if limit > ACCOUNT_LISTING_LIMIT: + return HTTPPreconditionFailed(request=req, + body='Maximum limit is %d' % ACCOUNT_LISTING_LIMIT) + marker = get_param(req, 'marker', '') + query_format = get_param(req, 'format') + except UnicodeDecodeError, err: + return HTTPBadRequest(body='parameters not utf8', + content_type='text/plain', request=req) + header_format = req.accept.first_match(['text/plain', + 'application/json', + 'application/xml']) + format = query_format if query_format else header_format + if format.startswith('application/'): + format = format[12:] + account_list = broker.list_containers_iter(limit, marker, prefix, + delimiter) + if format == 'json': + out_content_type = 'application/json' + json_pattern = ['"name":%s', '"count":%s', '"bytes":%s'] + json_pattern = '{' + ','.join(json_pattern) + '}' + json_out = [] + for (name, object_count, bytes_used, is_subdir) in account_list: + name = simplejson.dumps(name) + if is_subdir: + json_out.append('{"subdir":%s}'% name) + else: + json_out.append(json_pattern % + (name, object_count, bytes_used)) + account_list = '[' + ','.join(json_out) + ']' + elif format == 'xml': + out_content_type = 'application/xml' + output_list = ['', + ''%account] + for (name, object_count, bytes_used, is_subdir) in account_list: + name = saxutils.escape(name) + if is_subdir: + output_list.append('' % name) + else: + item = '%s%s' \ + '%s' % \ + (name, object_count, bytes_used) + output_list.append(item) + output_list.append('') + account_list = '\n'.join(output_list) + else: + if not account_list: + return HTTPNoContent(request=req, headers=resp_headers) + out_content_type = 'text/plain' + account_list = '\n'.join(r[0] for r in account_list) + '\n' + ret = Response(body=account_list, request=req, headers=resp_headers) + ret.content_type = out_content_type + ret.charset = 'utf8' + return ret + + def POST(self, req): + """ + Handle HTTP POST request. + Handler for RPC calls for account replication. + """ + try: + post_args = split_path(unquote(req.path), 3) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + drive, partition, hash = post_args + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + try: + args = simplejson.load(req.body_file) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain') + ret = self.replicator_rpc.dispatch(post_args, args) + ret.request = req + return ret + + def __call__(self, env, start_response): + start_time = time.time() + req = Request(env) + if req.path_info == '/healthcheck': + return healthcheck(req)(env, start_response) + elif not check_xml_encodable(req.path_info): + res = HTTPPreconditionFailed(body='Invalid UTF8') + else: + try: + if hasattr(self, req.method): + res = getattr(self, req.method)(req) + else: + res = HTTPMethodNotAllowed() + except: + self.logger.exception('ERROR __call__ error with %s %s ' + 'transaction %s' % (env.get('REQUEST_METHOD', '-'), + env.get('PATH_INFO', '-'), env.get('HTTP_X_CF_TRANS_ID', + '-'))) + res = HTTPInternalServerError(body=traceback.format_exc()) + trans_time = '%.4f' % (time.time() - start_time) + additional_info = '' + if res.headers.get('x-container-timestamp') is not None: + additional_info += 'x-container-timestamp: %s' % \ + res.headers['x-container-timestamp'] + log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % ( + req.remote_addr, + time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), + req.method, req.path, + res.status.split()[0], res.content_length or '-', + req.headers.get('x-cf-trans-id', '-'), + req.referer or '-', req.user_agent or '-', + trans_time, + additional_info) + if req.method.upper() == 'POST': + self.logger.debug(log_message) + else: + self.logger.info(log_message) + return res(env, start_response) + diff --git a/swift/auth/__init__.py b/swift/auth/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/swift/auth/server.py b/swift/auth/server.py new file mode 100644 index 0000000000..b58064f571 --- /dev/null +++ b/swift/auth/server.py @@ -0,0 +1,503 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import errno +import os +import socket +from contextlib import contextmanager +from time import gmtime, strftime, time +from urllib import unquote, quote +from uuid import uuid4 + +from webob import Request, Response +from webob.exc import HTTPBadRequest, HTTPNoContent, HTTPUnauthorized, \ + HTTPServiceUnavailable, HTTPNotFound + +from swift.common.bufferedhttp import http_connect +from swift.common.db import DatabaseConnectionError, get_db_connection +from swift.common.ring import Ring +from swift.common.utils import get_logger, normalize_timestamp, split_path + + +class AuthController(object): + """ + Sample implementation of an authorization server for development work. This + server only implements the basic functionality and isn't written for high + availability or to scale to thousands (or even hundreds) of requests per + second. It is mainly for use by developers working on the rest of the + system. + + The design of the auth system was restricted by a couple of existing + systems. + + This implementation stores an account name, user name, and password (in + plain text!) as well as a corresponding Swift cluster url and account hash. + One existing auth system used account, user, and password whereas another + used just account and an "API key". Here, we support both systems with + their various, sometimes colliding headers. + + The most common use case is by the end user: + + * The user makes a ReST call to the auth server requesting a token and url + to use to access the Swift cluster. + * The auth system validates the user info and returns a token and url for + the user to use with the Swift cluster. + * The user makes a ReST call to the Swift cluster using the url given with + the token as the X-Auth-Token header. + * The Swift cluster makes an ReST call to the auth server to validate the + token for the given account hash, caching the result for future requests + up to the expiration the auth server returns. + * The auth server validates the token / account hash given and returns the + expiration for the token. + * The Swift cluster completes the user's request. + + Another use case is creating a new account: + + * The developer makes a ReST call to create a new account. + * The auth server makes ReST calls to the Swift cluster's account servers + to create a new account on its end. + * The auth server records the information in its database. + + A last use case is recreating existing accounts; this is really only useful + on a development system when the drives are reformatted quite often but + the auth server's database is retained: + + * A developer makes an ReST call to have the existing accounts recreated. + * For each account in its database, the auth server makes ReST calls to + the Swift cluster's account servers to create a specific account on its + end. + + :param conf: The [auth-server] dictionary of the auth server configuration + file + :param ring: Overrides loading the account ring from a file; useful for + testing. + + See the etc/auth-server.conf-sample for information on the possible + configuration parameters. + """ + + log_name = 'auth' + + def __init__(self, conf, ring=None): + self.logger = get_logger(conf, self.log_name) + self.swift_dir = conf.get('swift_dir', '/etc/swift') + self.default_cluster_url = \ + conf.get('default_cluster_url', 'http://127.0.0.1:9000/v1') + self.token_life = int(conf.get('token_life', 86400)) + self.log_headers = conf.get('log_headers') == 'True' + if ring: + self.account_ring = ring + else: + self.account_ring = \ + Ring(os.path.join(self.swift_dir, 'account.ring.gz')) + self.db_file = os.path.join(self.swift_dir, 'auth.db') + self.conn = get_db_connection(self.db_file, okay_to_create=True) + self.conn.execute('''CREATE TABLE IF NOT EXISTS account ( + account TEXT, url TEXT, cfaccount TEXT, + user TEXT, password TEXT)''') + self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account + ON account (account)''') + self.conn.execute('''CREATE TABLE IF NOT EXISTS token ( + cfaccount TEXT, token TEXT, created FLOAT)''') + self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount + ON token (cfaccount)''') + self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created + ON token (created)''') + self.conn.commit() + + def add_storage_account(self, account_name=''): + """ + Creates an account within the Swift cluster by making a ReST call to + each of the responsible account servers. + + :param account_name: The desired name for the account; if omitted a + UUID4 will be used. + :returns: False upon failure, otherwise the name of the account + within the Swift cluster. + """ + begin = time() + orig_account_name = account_name + if not account_name: + account_name = str(uuid4()) + partition, nodes = self.account_ring.get_nodes(account_name) + headers = {'X-Timestamp': normalize_timestamp(time()), + 'x-cf-trans-id': 'tx' + str(uuid4())} + statuses = [] + for node in nodes: + try: + conn = None + conn = http_connect(node['ip'], node['port'], node['device'], + partition, 'PUT', '/'+account_name, headers) + source = conn.getresponse() + statuses.append(source.status) + if source.status >= 500: + self.logger.error('ERROR With account server %s:%s/%s: ' + 'Response %s %s: %s' % + (node['ip'], node['port'], node['device'], + source.status, source.reason, source.read(1024))) + conn = None + except BaseException, err: + log_call = self.logger.exception + msg = 'ERROR With account server ' \ + '%(ip)s:%(port)s/%(device)s (will retry later): ' % node + if isinstance(err, socket.error): + if err[0] == errno.ECONNREFUSED: + log_call = self.logger.error + msg += 'Connection refused' + elif err[0] == errno.EHOSTUNREACH: + log_call = self.logger.error + msg += 'Host unreachable' + log_call(msg) + rv = False + if len([s for s in statuses if (200 <= s < 300)]) > len(nodes) / 2: + rv = account_name + return rv + + @contextmanager + def get_conn(self): + """ + Returns a DB API connection instance to the auth server's SQLite + database. This is a contextmanager call to be use with the 'with' + statement. It takes no parameters. + """ + if not self.conn: + # We go ahead and make another db connection even if this is a + # reentry call; just in case we had an error that caused self.conn + # to become None. Even if we make an extra conn, we'll only keep + # one after the 'with' block. + self.conn = get_db_connection(self.db_file) + conn = self.conn + self.conn = None + try: + yield conn + conn.rollback() + self.conn = conn + except Exception, err: + try: + conn.close() + except: + pass + self.conn = get_db_connection(self.db_file) + raise err + + def purge_old_tokens(self): + """ + Removes tokens that have expired from the auth server's database. This + is called by :func:`validate_token` and :func:`GET` to help keep the + database clean. + """ + with self.get_conn() as conn: + conn.execute('DELETE FROM token WHERE created < ?', + (time() - self.token_life,)) + conn.commit() + + def validate_token(self, token, account_hash): + """ + Tests if the given token is a valid token + + :param token: The token to validate + :param account_hash: The account hash the token is being used with + :returns: TTL if valid, False otherwise + """ + begin = time() + self.purge_old_tokens() + rv = False + with self.get_conn() as conn: + row = conn.execute(''' + SELECT created FROM token + WHERE cfaccount = ? AND token = ?''', + (account_hash, token)).fetchone() + if row is not None: + created = row[0] + if time() - created >= self.token_life: + conn.execute(''' + DELETE FROM token + WHERE cfaccount = ? AND token = ?''', + (account_hash, token)) + conn.commit() + else: + rv = self.token_life - (time() - created) + self.logger.info('validate_token(%s, %s, _, _) = %s [%.02f]' % + (repr(token), repr(account_hash), repr(rv), + time() - begin)) + return rv + + def create_account(self, new_account, new_user, new_password): + """ + Handles the create_account call for developers, used to request + an account be created both on a Swift cluster and in the auth server + database. + + This will make ReST requests to the Swift cluster's account servers + to have an account created on its side. The resulting account hash + along with the URL to use to access the account, the account name, the + user name, and the password is recorded in the auth server's database. + The url is constructed now and stored separately to support changing + the configuration file's default_cluster_url for directing new accounts + to a different Swift cluster while still supporting old accounts going + to the Swift clusters they were created on. + + :param new_account: The name for the new account + :param new_user: The name for the new user + :param new_password: The password for the new account + + :returns: False if the create fails, storage url if successful + """ + begin = time() + if not all((new_account, new_user, new_password)): + return False + account_hash = self.add_storage_account() + if not account_hash: + self.logger.info( + 'FAILED create_account(%s, %s, _,) [%.02f]' % + (repr(new_account), repr(new_user), time() - begin)) + return False + url = self.default_cluster_url.rstrip('/') + '/' + account_hash + with self.get_conn() as conn: + conn.execute('''INSERT INTO account + (account, url, cfaccount, user, password) + VALUES (?, ?, ?, ?, ?)''', + (new_account, url, account_hash, new_user, new_password)) + conn.commit() + self.logger.info( + 'SUCCESS create_account(%s, %s, _) = %s [%.02f]' % + (repr(new_account), repr(new_user), repr(url), time() - begin)) + return url + + def recreate_accounts(self): + """ + Recreates the accounts from the existing auth database in the Swift + cluster. This is useful on a development system when the drives are + reformatted quite often but the auth server's database is retained. + + :returns: A string indicating accounts and failures + """ + begin = time() + with self.get_conn() as conn: + account_hashes = [r[0] for r in + conn.execute('SELECT cfaccount FROM account').fetchall()] + failures = [] + for i, account_hash in enumerate(account_hashes): + if not self.add_storage_account(account_hash): + failures.append(account_hash) + rv = '%d accounts, failures %s' % (len(account_hashes), repr(failures)) + self.logger.info('recreate_accounts(_, _) = %s [%.02f]' % + (rv, time() - begin)) + return rv + + def handle_token(self, request): + """ + Hanles ReST request from Swift to validate tokens + + Valid URL paths: + * GET /token// + + If the HTTP equest returns with a 204, then the token is valid, + and the TTL of the token will be available in the X-Auth-Ttl header. + + :param request: webob.Request object + """ + try: + _, account_hash, token = split_path(request.path, minsegs=3) + except ValueError: + return HTTPBadRequest() + ttl = self.validate_token(token, account_hash) + if not ttl: + return HTTPNotFound() + return HTTPNoContent(headers={'x-auth-ttl': ttl}) + + def handle_account_create(self, request): + """ + Handles Rest requests from developers to have an account created. + + Valid URL paths: + * PUT /account// - create the account + + Valid headers: + * X-Auth-Key: (Only required when creating an account) + + If the HTTP request returns with a 204, then the account was created, + and the storage url will be available in the X-Storage-Url header. + + :param request: webob.Request object + """ + try: + _, account_name, user_name = split_path(request.path, minsegs=3) + except ValueError: + return HTTPBadRequest() + if 'X-Auth-Key' not in request.headers: + return HTTPBadRequest('X-Auth-Key is required') + password = request.headers['x-auth-key'] + storage_url = self.create_account(account_name, user_name, password) + if not storage_url: + return HTTPServiceUnavailable() + return HTTPNoContent(headers={'x-storage-url': storage_url}) + + def handle_account_recreate(self, request): + """ + Handles ReST requests from developers to have accounts in the Auth + system recreated in Swift. I know this is bad ReST style, but this + isn't production right? :) + + Valid URL paths: + * POST /recreate_accounts + + :param request: webob.Request object + """ + result = self.recreate_accounts() + return Response(result, 200, request = request) + + def handle_auth(self, request): + """ + Handles ReST requests from end users for a Swift cluster url and auth + token. This can handle all the various headers and formats that + existing auth systems used, so it's a bit of a chameleon. + + Valid URL paths: + * GET /v1//auth + * GET /auth + * GET /v1.0 + + Valid headers: + * X-Auth-User: : + * X-Auth-Key: + * X-Storage-User: [:] + The [:] is only optional here if the + /v1//auth path is used. + * X-Storage-Pass: + + The (currently) preferred method is to use /v1.0 path and the + X-Auth-User and X-Auth-Key headers. + + :param request: A webob.Request instance. + """ + pathsegs = \ + split_path(request.path, minsegs=1, maxsegs=3, rest_with_last=True) + if pathsegs[0] == 'v1' and pathsegs[2] == 'auth': + account = pathsegs[1] + user = request.headers.get('x-storage-user') + if not user: + user = request.headers.get('x-auth-user') + if not user or ':' not in user: + return HTTPUnauthorized() + account2, user = user.split(':', 1) + if account != account2: + return HTTPUnauthorized() + password = request.headers.get('x-storage-pass') + if not password: + password = request.headers.get('x-auth-key') + elif pathsegs[0] in ('auth', 'v1.0'): + user = request.headers.get('x-auth-user') + if not user: + user = request.headers.get('x-storage-user') + if not user or ':' not in user: + return HTTPUnauthorized() + account, user = user.split(':', 1) + password = request.headers.get('x-auth-key') + if not password: + password = request.headers.get('x-storage-pass') + else: + return HTTPBadRequest() + if not all((account, user, password)): + return HTTPUnauthorized() + self.purge_old_tokens() + with self.get_conn() as conn: + row = conn.execute(''' + SELECT cfaccount, url FROM account + WHERE account = ? AND user = ? AND password = ?''', + (account, user, password)).fetchone() + if row is None: + return HTTPUnauthorized() + cfaccount = row[0] + url = row[1] + row = conn.execute('SELECT token FROM token WHERE cfaccount = ?', + (cfaccount,)).fetchone() + if row: + token = row[0] + else: + token = 'tk' + str(uuid4()) + conn.execute(''' + INSERT INTO token (cfaccount, token, created) + VALUES (?, ?, ?)''', + (cfaccount, token, time())) + conn.commit() + return HTTPNoContent(headers={'x-auth-token': token, + 'x-storage-token': token, + 'x-storage-url': url}) + + + def handleREST(self, env, start_response): + """ + Handles routing of ReST requests. This handler also logs all requests. + + :param env: WSGI environment + :param start_response: WSGI start_response function + """ + req = Request(env) + logged_headers = None + if self.log_headers: + logged_headers = '\n'.join('%s: %s' % (k, v) + for k, v in req.headers.items()).replace('"', "#042") + start_time = time() + # Figure out how to handle the request + try: + if req.method == 'GET' and req.path.startswith('/v1') or \ + req.path.startswith('/auth'): + handler = self.handle_auth + elif req.method == 'GET' and req.path.startswith('/token/'): + handler = self.handle_token + elif req.method == 'PUT' and req.path.startswith('/account/'): + handler = self.handle_account_create + elif req.method == 'POST' and \ + req.path == '/recreate_accounts': + handler = self.handle_account_recreate + else: + return HTTPBadRequest(request=env)(env, start_response) + response = handler(req) + except: + self.logger.exception('ERROR Unhandled exception in ReST request') + return HTTPServiceUnavailable(request=req)(env, start_response) + trans_time = '%.4f' % (time() - start_time) + if not response.content_length and response.app_iter and \ + hasattr(response.app_iter, '__len__'): + response.content_length = sum(map(len, response.app_iter)) + the_request = '%s %s' % (req.method, quote(unquote(req.path))) + if req.query_string: + the_request = the_request + '?' + req.query_string + the_request += ' ' + req.environ['SERVER_PROTOCOL'] + client = req.headers.get('x-cluster-client-ip') + if not client and 'x-forwarded-for' in req.headers: + client = req.headers['x-forwarded-for'].split(',')[0].strip() + if not client: + client = req.remote_addr + self.logger.info( + '%s - - [%s] "%s" %s %s "%s" "%s" - - - - - - - - - "-" "%s" ' + '"%s" %s' % ( + client, + strftime('%d/%b/%Y:%H:%M:%S +0000', gmtime()), + the_request, + response.status_int, + response.content_length or '-', + req.referer or '-', + req.user_agent or '-', + req.remote_addr, + logged_headers or '-', + trans_time)) + return response(env, start_response) + + def __call__(self, env, start_response): + """ Used by the eventlet.wsgi.server """ + return self.handleREST(env, start_response) diff --git a/swift/common/__init__.py b/swift/common/__init__.py new file mode 100644 index 0000000000..fe1925beb1 --- /dev/null +++ b/swift/common/__init__.py @@ -0,0 +1,6 @@ +""" Code common to all of Swift. """ + +ACCOUNT_LISTING_LIMIT = 10000 +CONTAINER_LISTING_LIMIT = 10000 +FILE_SIZE_LIMIT = 5368709122 + diff --git a/swift/common/auth.py b/swift/common/auth.py new file mode 100644 index 0000000000..1e7be05da3 --- /dev/null +++ b/swift/common/auth.py @@ -0,0 +1,98 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser, NoOptionError +import os +import time + +from webob.request import Request +from webob.exc import HTTPUnauthorized, HTTPPreconditionFailed +from eventlet.timeout import Timeout + +from swift.common.utils import split_path +from swift.common.bufferedhttp import http_connect_raw as http_connect + + +class DevAuthMiddleware(object): + """ + Auth Middleware that uses the dev auth server + """ + def __init__(self, app, conf, memcache_client, logger): + self.app = app + self.memcache_client = memcache_client + self.logger = logger + self.conf = conf + self.auth_host = conf.get('bind_ip', '127.0.0.1') + self.auth_port = int(conf.get('bind_port', 11000)) + self.timeout = int(conf.get('node_timeout', 10)) + + def __call__(self, env, start_response): + req = Request(env) + if req.path != '/healthcheck': + if 'x-storage-token' in req.headers and \ + 'x-auth-token' not in req.headers: + req.headers['x-auth-token'] = req.headers['x-storage-token'] + version, account, container, obj = split_path(req.path, 1, 4, True) + if account is None: + return HTTPPreconditionFailed(request=req, body='Bad URL')( + env, start_response) + if not req.headers.get('x-auth-token'): + return HTTPPreconditionFailed(request=req, + body='Missing Auth Token')(env, start_response) + if account is None: + return HTTPPreconditionFailed( + request=req, body='Bad URL')(env, start_response) + if not self.auth(account, req.headers['x-auth-token']): + return HTTPUnauthorized(request=req)(env, start_response) + + # If we get here, then things should be good. + return self.app(env, start_response) + + def auth(self, account, token): + """ + Dev authorization implmentation + + :param account: account name + :param token: auth token + + :returns: True if authorization is successful, False otherwise + """ + key = 'auth/%s/%s' % (account, token) + now = time.time() + cached_auth_data = self.memcache_client.get(key) + if cached_auth_data: + start, expiration = cached_auth_data + if now - start <= expiration: + return True + try: + with Timeout(self.timeout): + conn = http_connect(self.auth_host, self.auth_port, 'GET', + '/token/%s/%s' % (account, token)) + resp = conn.getresponse() + resp.read() + conn.close() + if resp.status == 204: + validated = float(resp.getheader('x-auth-ttl')) + else: + validated = False + except: + self.logger.exception('ERROR with auth') + return False + if not validated: + return False + else: + val = (now, validated) + self.memcache_client.set(key, val, timeout=validated) + return True diff --git a/swift/common/bufferedhttp.py b/swift/common/bufferedhttp.py new file mode 100644 index 0000000000..ed5f0cec2d --- /dev/null +++ b/swift/common/bufferedhttp.py @@ -0,0 +1,158 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Monkey Patch httplib.HTTPResponse to buffer reads of headers. This can improve +performance when making large numbers of small HTTP requests. This module +also provides helper functions to make HTTP connections using +BufferedHTTPResponse. + +.. warning:: + + If you use this, be sure that the libraries you are using do not access + the socket directly (xmlrpclib, I'm looking at you :/), and instead + make all calls through httplib. +""" + +from urllib import quote +import logging +import time + +from eventlet.green.httplib import HTTPConnection, HTTPResponse, _UNKNOWN, \ + CONTINUE, HTTPMessage + + +class BufferedHTTPResponse(HTTPResponse): + """HTTPResponse class that buffers reading of headers""" + + def __init__(self, sock, debuglevel=0, strict=0, + method=None): # pragma: no cover + self.sock = sock + self.fp = sock.makefile('rb') + self.debuglevel = debuglevel + self.strict = strict + self._method = method + + self.msg = None + + # from the Status-Line of the response + self.version = _UNKNOWN # HTTP-Version + self.status = _UNKNOWN # Status-Code + self.reason = _UNKNOWN # Reason-Phrase + + self.chunked = _UNKNOWN # is "chunked" being used? + self.chunk_left = _UNKNOWN # bytes left to read in current chunk + self.length = _UNKNOWN # number of bytes left in response + self.will_close = _UNKNOWN # conn will close at end of response + + def expect_response(self): + self.fp = self.sock.makefile('rb', 0) + version, status, reason = self._read_status() + if status != CONTINUE: + self._read_status = lambda: (version, status, reason) + self.begin() + else: + self.status = status + self.reason = reason.strip() + self.version = 11 + self.msg = HTTPMessage(self.fp, 0) + self.msg.fp = None + + +class BufferedHTTPConnection(HTTPConnection): + """HTTPConnection class that uses BufferedHTTPResponse""" + response_class = BufferedHTTPResponse + + def connect(self): + self._connected_time = time.time() + return HTTPConnection.connect(self) + + def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0): + self._method = method + self._path = url + self._txn_id = '-' + return HTTPConnection.putrequest(self, method, url, skip_host, + skip_accept_encoding) + + def putheader(self, header, value): + if header.lower() == 'x-cf-trans-id': + self._txn_id = value + return HTTPConnection.putheader(self, header, value) + + def getexpect(self): + response = BufferedHTTPResponse(self.sock, strict=self.strict, + method=self._method) + response.expect_response() + return response + + def getresponse(self): + response = HTTPConnection.getresponse(self) + logging.debug("HTTP PERF: %.5f seconds to %s %s:%s %s (%s)" % + (time.time() - self._connected_time, self._method, self.host, + self.port, self._path, self._txn_id)) + return response + + +def http_connect(ipaddr, port, device, partition, method, path, + headers=None, query_string=None): + """ + Helper function to create a HTTPConnection object that is buffered + for backend Swift services. + + :param ipaddr: IPv4 address to connect to + :param port: port to connect to + :param device: device of the node to query + :param partition: partition on the device + :param method: HTTP method to request ('GET', 'PUT', 'POST', etc.) + :param path: request path + :param headers: dictionary of headers + :param query_string: request query string + :returns: HTTPConnection object + """ + conn = BufferedHTTPConnection('%s:%s' % (ipaddr, port)) + path = quote('/' + device + '/' + str(partition) + path) + if query_string: + path += '?' + query_string + conn.path = path + conn.putrequest(method, path) + if headers: + for header, value in headers.iteritems(): + conn.putheader(header, value) + conn.endheaders() + return conn + +def http_connect_raw(ipaddr, port, method, path, headers=None, + query_string=None): + """ + Helper function to create a HTTPConnection object that is buffered. + + :param ipaddr: IPv4 address to connect to + :param port: port to connect to + :param method: HTTP method to request ('GET', 'PUT', 'POST', etc.) + :param path: request path + :param headers: dictionary of headers + :param query_string: request query string + :returns: HTTPConnection object + """ + conn = BufferedHTTPConnection('%s:%s' % (ipaddr, port)) + if query_string: + path += '?' + query_string + conn.path = path + conn.putrequest(method, path) + if headers: + for header, value in headers.iteritems(): + conn.putheader(header, value) + conn.endheaders() + return conn diff --git a/swift/common/client.py b/swift/common/client.py new file mode 100644 index 0000000000..3f3b9d8b4c --- /dev/null +++ b/swift/common/client.py @@ -0,0 +1,718 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Cloud Files client library used internally +""" +import socket +from cStringIO import StringIO +from httplib import HTTPConnection, HTTPException, HTTPSConnection +from re import compile, DOTALL +from tokenize import generate_tokens, STRING, NAME, OP +from urllib import quote as _quote, unquote +from urlparse import urlparse, urlunparse + +try: + from eventlet import sleep +except: + from time import sleep + + +def quote(value, safe='/'): + """ + Patched version of urllib.quote that encodes utf8 strings before quoting + """ + if isinstance(value, unicode): + value = value.encode('utf8') + return _quote(value, safe) + + +# look for a real json parser first +try: + # simplejson is popular and pretty good + from simplejson import loads as json_loads +except ImportError: + try: + # 2.6 will have a json module in the stdlib + from json import loads as json_loads + except ImportError: + # fall back on local parser otherwise + comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL) + + def json_loads(string): + ''' + Fairly competent json parser exploiting the python tokenizer and + eval(). -- From python-cloudfiles + + _loads(serialized_json) -> object + ''' + try: + res = [] + consts = {'true': True, 'false': False, 'null': None} + string = '(' + comments.sub('', string) + ')' + for type, val, _, _, _ in \ + generate_tokens(StringIO(string).readline): + if (type == OP and val not in '[]{}:,()-') or \ + (type == NAME and val not in consts): + raise AttributeError() + elif type == STRING: + res.append('u') + res.append(val.replace('\\/', '/')) + else: + res.append(val) + return eval(''.join(res), {}, consts) + except: + raise AttributeError() + + +class ClientException(Exception): + + def __init__(self, msg, http_scheme='', http_host='', http_port='', + http_path='', http_query='', http_status=0, http_reason='', + http_device=''): + Exception.__init__(self, msg) + self.msg = msg + self.http_scheme = http_scheme + self.http_host = http_host + self.http_port = http_port + self.http_path = http_path + self.http_query = http_query + self.http_status = http_status + self.http_reason = http_reason + self.http_device = http_device + + def __str__(self): + a = self.msg + b = '' + if self.http_scheme: + b += '%s://' % self.http_scheme + if self.http_host: + b += self.http_host + if self.http_port: + b += ':%s' % self.http_port + if self.http_path: + b += self.http_path + if self.http_query: + b += '?%s' % self.http_query + if self.http_status: + if b: + b = '%s %s' % (b, self.http_status) + else: + b = str(self.http_status) + if self.http_reason: + if b: + b = '%s %s' % (b, self.http_reason) + else: + b = '- %s' % self.http_reason + if self.http_device: + if b: + b = '%s: device %s' % (b, self.http_device) + else: + b = 'device %s' % self.http_device + return b and '%s: %s' % (a, b) or a + + +def http_connection(url): + """ + Make an HTTPConnection or HTTPSConnection + + :param url: url to connect to + :returns: tuple of (parsed url, connection object) + :raises ClientException: Unable to handle protocol scheme + """ + parsed = urlparse(url) + if parsed.scheme == 'http': + conn = HTTPConnection(parsed.netloc) + elif parsed.scheme == 'https': + conn = HTTPSConnection(parsed.netloc) + else: + raise ClientException('Cannot handle protocol scheme %s for url %s' % + (parsed.scheme, repr(url))) + return parsed, conn + + +def get_auth(url, user, key, snet=False): + """ + Get authentication credentials + + :param url: authentication URL + :param user: user to auth as + :param key: key or passowrd for auth + :param snet: use SERVICENET internal network default is False + :returns: tuple of (storage URL, storage token, auth token) + :raises ClientException: HTTP GET request to auth URL failed + """ + parsed, conn = http_connection(url) + conn.request('GET', parsed.path, '', + {'X-Auth-User': user, 'X-Auth-Key': key}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Auth GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_status=resp.status, + http_reason=resp.reason) + url = resp.getheader('x-storage-url') + if snet: + parsed = list(urlparse(url)) + # Second item in the list is the netloc + parsed[1] = 'snet-' + parsed[1] + url = urlunparse(parsed) + return url, resp.getheader('x-storage-token', + resp.getheader('x-auth-token')) + + +def get_account(url, token, marker=None, limit=None, prefix=None, + http_conn=None, full_listing=False): + """ + Get a listing of containers for the account. + + :param url: storage URL + :param token: auth token + :param marker: marker query + :param limit: limit query + :param prefix: prefix query + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param full_listing: if True, return a full listing, else returns a max + of 10000 listings + :returns: a list of accounts + :raises ClientException: HTTP GET request failed + """ + if not http_conn: + http_conn = http_connection(url) + if full_listing: + rv = [] + listing = get_account(url, token, marker, limit, prefix, http_conn) + while listing: + rv.extend(listing) + marker = listing[-1]['name'] + listing = get_account(url, token, marker, limit, prefix, http_conn) + return rv + parsed, conn = http_conn + qs = 'format=json' + if marker: + qs += '&marker=%s' % quote(marker) + if limit: + qs += '&limit=%d' % limit + if prefix: + qs += '&prefix=%s' % quote(prefix) + conn.request('GET', '%s?%s' % (parsed.path, qs), '', + {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Account GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_query=qs, http_status=resp.status, + http_reason=resp.reason) + if resp.status == 204: + resp.read() + return [] + return json_loads(resp.read()) + + +def head_account(url, token, http_conn=None): + """ + Get account stats. + + :param url: storage URL + :param token: auth token + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a tuple of (container count, object count, bytes used) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_status=resp.status, + http_reason=resp.reason) + return int(resp.getheader('x-account-container-count', 0)), \ + int(resp.getheader('x-account-object-count', 0)), \ + int(resp.getheader('x-account-bytes-used', 0)) + + +def get_container(url, token, container, marker=None, limit=None, + prefix=None, delimiter=None, http_conn=None, + full_listing=False): + """ + Get a listing of objects for the container. + + :param url: storage URL + :param token: auth token + :param container: container name to get a listing for + :param marker: marker query + :param limit: limit query + :param prefix: prefix query + :param delimeter: string to delimit the queries on + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param full_listing: if True, return a full listing, else returns a max + of 10000 listings + :returns: a list of objects + :raises ClientException: HTTP GET request failed + """ + if not http_conn: + http_conn = http_connection(url) + if full_listing: + rv = [] + listing = get_container(url, token, container, marker, limit, prefix, + delimiter, http_conn) + while listing: + rv.extend(listing) + if not delimiter: + marker = listing[-1]['name'] + else: + marker = listing[-1].get('name', listing[-1].get('subdir')) + listing = get_container(url, token, container, marker, limit, + prefix, delimiter, http_conn) + return rv + parsed, conn = http_conn + path = '%s/%s' % (parsed.path, quote(container)) + qs = 'format=json' + if marker: + qs += '&marker=%s' % quote(marker) + if limit: + qs += '&limit=%d' % limit + if prefix: + qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) + conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Container GET failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_query=qs, + http_status=resp.status, http_reason=resp.reason) + if resp.status == 204: + resp.read() + return [] + return json_loads(resp.read()) + + +def head_container(url, token, container, http_conn=None): + """ + Get container stats. + + :param url: storage URL + :param token: auth token + :param container: container name to get stats for + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a tuple of (object count, bytes used) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + conn.request('HEAD', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container HEAD failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + return int(resp.getheader('x-container-object-count', 0)), \ + int(resp.getheader('x-container-bytes-used', 0)) + + +def put_container(url, token, container, http_conn=None): + """ + Create a container + + :param url: storage URL + :param token: auth token + :param container: container name to create + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP PUT request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + conn.request('PUT', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container PUT failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + +def delete_container(url, token, container, http_conn=None): + """ + Delete a container + + :param url: storage URL + :param token: auth token + :param container: container name to delete + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP DELETE request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + conn.request('DELETE', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container DELETE failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + +def get_object(url, token, container, name, http_conn=None, + resp_chunk_size=None): + """ + Get an object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to get + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param resp_chunk_size: if defined, chunk size of data to read + :returns: a list of objects + :raises ClientException: HTTP GET request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('GET', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Object GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + metadata = {} + for key, value in resp.getheaders(): + if key.lower().startswith('x-object-meta-'): + metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) + if resp_chunk_size: + + def _object_body(): + buf = resp.read(resp_chunk_size) + while buf: + yield buf + buf = resp.read(resp_chunk_size) + object_body = _object_body() + else: + object_body = resp.read() + return resp.getheader('content-type'), \ + int(resp.getheader('content-length', 0)), \ + resp.getheader('last-modified'), \ + resp.getheader('etag').strip('"'), \ + metadata, \ + object_body + + +def head_object(url, token, container, name, http_conn=None): + """ + Get object info + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to get info for + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a tuple of (content type, content length, last modfied, etag, + dictionary of metadata) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('HEAD', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + metadata = {} + for key, value in resp.getheaders(): + if key.lower().startswith('x-object-meta-'): + metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) + return resp.getheader('content-type'), \ + int(resp.getheader('content-length', 0)), \ + resp.getheader('last-modified'), \ + resp.getheader('etag').strip('"'), \ + metadata + + +def put_object(url, token, container, name, contents, metadata={}, + content_length=None, etag=None, chunk_size=65536, + content_type=None, http_conn=None): + """ + Put an object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to put + :param contents: file like object to read object data from + :param metadata: dictionary of object metadata + :param content_length: value to send as content-length header + :param etag: etag of contents + :param chunk_size: chunk size of data to write + :param content_type: value to send as content-type header + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: etag from server response + :raises ClientException: HTTP PUT request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + headers = {'X-Auth-Token': token} + for key, value in metadata.iteritems(): + headers['X-Object-Meta-%s' % quote(key)] = quote(value) + if etag: + headers['ETag'] = etag.strip('"') + if content_length is not None: + headers['Content-Length'] = str(content_length) + if content_type is not None: + headers['Content-Type'] = content_type + if not contents: + headers['Content-Length'] = '0' + if hasattr(contents, 'read'): + conn.putrequest('PUT', path) + for header, value in headers.iteritems(): + conn.putheader(header, value) + if not content_length: + conn.putheader('Transfer-Encoding', 'chunked') + conn.endheaders() + chunk = contents.read(chunk_size) + while chunk: + if not content_length: + conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) + else: + conn.send(chunk) + chunk = contents.read(chunk_size) + if not content_length: + conn.send('0\r\n\r\n') + else: + conn.request('PUT', path, contents, headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object PUT failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + return resp.getheader('etag').strip('"') + + +def post_object(url, token, container, name, metadata, http_conn=None): + """ + Change object metadata + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to change + :param metadata: dictionary of object metadata + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + headers = {'X-Auth-Token': token} + for key, value in metadata.iteritems(): + headers['X-Object-Meta-%s' % quote(key)] = quote(value) + conn.request('POST', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object POST failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + + +def delete_object(url, token, container, name, http_conn=None): + """ + Delete object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to delete + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP DELETE request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('DELETE', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object DELETE failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + +class Connection(object): + """Convenience class to make requests that will also retry the request""" + + def __init__(self, authurl, user, key, retries=5, preauthurl=None, + preauthtoken=None, snet=False): + """ + :param authurl: authenitcation URL + :param user: user name to authenticate as + :param key: key/password to authenticate with + :param retries: Number of times to retry the request before failing + :param preauthurl: storage URL (if you have already authenticated) + :param preauthtoken: authentication token (if you have already + authenticated) + :param snet: use SERVICENET internal network default is False + """ + self.authurl = authurl + self.user = user + self.key = key + self.retries = retries + self.http_conn = None + self.url = preauthurl + self.token = preauthtoken + self.attempts = 0 + self.snet = snet + + def _retry(self, func, *args, **kwargs): + kwargs['http_conn'] = self.http_conn + self.attempts = 0 + backoff = 1 + while self.attempts <= self.retries: + self.attempts += 1 + try: + if not self.url or not self.token: + self.url, self.token = \ + get_auth(self.authurl, self.user, self.key, snet=self.snet) + self.http_conn = None + if not self.http_conn: + self.http_conn = http_connection(self.url) + kwargs['http_conn'] = self.http_conn + rv = func(self.url, self.token, *args, **kwargs) + return rv + except (socket.error, HTTPException): + if self.attempts > self.retries: + raise + self.http_conn = None + except ClientException, err: + if self.attempts > self.retries: + raise + if err.http_status == 401: + self.url = self.token = None + if self.attempts > 1: + raise + elif 500 <= err.http_status <= 599: + pass + else: + raise + sleep(backoff) + backoff *= 2 + + def head_account(self): + """Wrapper for head_account""" + return self._retry(head_account) + + def get_account(self, marker=None, limit=None, prefix=None, + full_listing=False): + """Wrapper for get_account""" + # TODO: With full_listing=True this will restart the entire listing + # with each retry. Need to make a better version that just retries + # where it left off. + return self._retry(get_account, marker=marker, limit=limit, + prefix=prefix, full_listing=full_listing) + + def head_container(self, container): + """Wrapper for head_container""" + return self._retry(head_container, container) + + def get_container(self, container, marker=None, limit=None, prefix=None, + delimiter=None, full_listing=False): + """Wrapper for get_container""" + # TODO: With full_listing=True this will restart the entire listing + # with each retry. Need to make a better version that just retries + # where it left off. + return self._retry(get_container, container, marker=marker, + limit=limit, prefix=prefix, delimiter=delimiter, + full_listing=full_listing) + + def put_container(self, container): + """Wrapper for put_container""" + return self._retry(put_container, container) + + def delete_container(self, container): + """Wrapper for delete_container""" + return self._retry(delete_container, container) + + def head_object(self, container, obj): + """Wrapper for head_object""" + return self._retry(head_object, container, obj) + + def get_object(self, container, obj, resp_chunk_size=None): + """Wrapper for get_object""" + return self._retry(get_object, container, obj, + resp_chunk_size=resp_chunk_size) + + def put_object(self, container, obj, contents, metadata={}, + content_length=None, etag=None, chunk_size=65536, + content_type=None): + """Wrapper for put_object""" + return self._retry(put_object, container, obj, contents, + metadata=metadata, content_length=content_length, etag=etag, + chunk_size=chunk_size, content_type=content_type) + + def post_object(self, container, obj, metadata): + """Wrapper for post_object""" + return self._retry(post_object, container, obj, metadata) + + def delete_object(self, container, obj): + """Wrapper for delete_object""" + return self._retry(delete_object, container, obj) diff --git a/swift/common/constraints.py b/swift/common/constraints.py new file mode 100644 index 0000000000..8fdacb5a93 --- /dev/null +++ b/swift/common/constraints.py @@ -0,0 +1,152 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from webob.exc import HTTPBadRequest, HTTPLengthRequired, \ + HTTPRequestEntityTooLarge + + +#: Max file size allowed for objects +MAX_FILE_SIZE = 5 * 1024 * 1024 * 1024 + 2 +#: Max length of the name of a key for metadata +MAX_META_NAME_LENGTH = 128 +#: Max length of the value of a key for metadata +MAX_META_VALUE_LENGTH = 256 +#: Max number of metadata items +MAX_META_COUNT = 90 +#: Max overall size of metadata +MAX_META_OVERALL_SIZE = 4096 +#: Max object name length +MAX_OBJECT_NAME_LENGTH = 1024 + + +def check_metadata(req): + """ + Check metadata sent for objects in the request headers. + + :param req: request object + :raises HTTPBadRequest: bad metadata + """ + meta_count = 0 + meta_size = 0 + for key, value in req.headers.iteritems(): + if not key.lower().startswith('x-object-meta-'): + continue + key = key[len('x-object-meta-'):] + if not key: + return HTTPBadRequest(body='Metadata name cannot be empty', + request=req, content_type='text/plain') + meta_count += 1 + meta_size += len(key) + len(value) + if len(key) > MAX_META_NAME_LENGTH: + return HTTPBadRequest( + body='Metadata name too long; max %d' + % MAX_META_NAME_LENGTH, + request=req, content_type='text/plain') + elif len(value) > MAX_META_VALUE_LENGTH: + return HTTPBadRequest( + body='Metadata value too long; max %d' + % MAX_META_VALUE_LENGTH, + request=req, content_type='text/plain') + elif meta_count > MAX_META_COUNT: + return HTTPBadRequest( + body='Too many metadata items; max %d' % MAX_META_COUNT, + request=req, content_type='text/plain') + elif meta_size > MAX_META_OVERALL_SIZE: + return HTTPBadRequest( + body='Total metadata too large; max %d' + % MAX_META_OVERALL_SIZE, + request=req, content_type='text/plain') + return None + + +def check_object_creation(req, object_name): + """ + Check to ensure that everything is alright about an object to be created. + + :param req: HTTP request object + :param object_name: name of object to be created + :raises HTTPRequestEntityTooLarge: the object is too large + :raises HTTPLengthRequered: missing content-length header and not + a chunked request + :raises HTTPBadRequest: missing or bad content-type header, or + bad metadata + """ + if req.content_length and req.content_length > MAX_FILE_SIZE: + return HTTPRequestEntityTooLarge(body='Your request is too large.', + request=req, content_type='text/plain') + if req.content_length is None and \ + req.headers.get('transfer-encoding') != 'chunked': + return HTTPLengthRequired(request=req) + if len(object_name) > MAX_OBJECT_NAME_LENGTH: + return HTTPBadRequest(body='Object name length of %d longer than %d' % + (len(object_name), MAX_OBJECT_NAME_LENGTH), request=req, + content_type='text/plain') + if 'Content-Type' not in req.headers: + return HTTPBadRequest(request=req, content_type='text/plain', + body='No content type') + if not check_xml_encodable(req.headers['Content-Type']): + return HTTPBadRequest(request=req, body='Invalid Content-Type', + content_type='text/plain') + return check_metadata(req) + + +def check_mount(root, drive): + """ + Verify that the path to the device is a mount point and mounted. This + allows us to fast fail on drives that have been unmounted because of + issues, and also prevents us for accidently filling up the root partition. + + :param root: base path where the devices are mounted + :param drive: drive name to be checked + :returns: True if it is a valid mounted device, False otherwise + """ + if not drive.isalnum(): + return False + path = os.path.join(root, drive) + return os.path.exists(path) and os.path.ismount(path) + + +def check_float(string): + """ + Helper function for checking if a string can be converted to a float. + + :param string: string to be verified as a float + :returns: True if the string can be converted to a float, False otherwise + """ + try: + float(string) + return True + except ValueError: + return False + + +_invalid_xml = re.compile(ur'[^\x09\x0a\x0d\x20-\uD7FF\uE000-\uFFFD%s-%s]' % + (unichr(0x10000), unichr(0x10FFFF))) + + +def check_xml_encodable(string): + """ + Validate if a string can be encoded in xml. + + :param string: string to be validated + :returns: True if the string can be encoded in xml, False otherwise + """ + try: + return not _invalid_xml.search(string.decode('UTF-8')) + except UnicodeDecodeError: + return False diff --git a/swift/common/db.py b/swift/common/db.py new file mode 100644 index 0000000000..3464d451b9 --- /dev/null +++ b/swift/common/db.py @@ -0,0 +1,1463 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Database code for Swift """ + +from __future__ import with_statement +from contextlib import contextmanager +import hashlib +import logging +import operator +import os +from uuid import uuid4 +import time +import cPickle as pickle +import errno +from random import randint +from tempfile import mkstemp +import math + +from eventlet import sleep +import sqlite3 + +from swift.common.utils import normalize_timestamp, renamer, \ + mkdirs, lock_parent_directory, fallocate +from swift.common.exceptions import LockTimeout + + +#: Timeout for trying to connect to a DB +BROKER_TIMEOUT = 25 +#: Pickle protocol to use +PICKLE_PROTOCOL = 2 +#: Max number of pending entries +PENDING_CAP = 131072 + + +class DatabaseConnectionError(sqlite3.DatabaseError): + """More friendly error messages for DB Errors.""" + def __init__(self, path, msg, timeout=0): + self.path = path + self.timeout = timeout + self.msg = msg + + def __str__(self): + return 'DB connection error (%s, %s):\n%s' % ( + self.path, self.timeout, self.msg) + + +class GreenDBConnection(sqlite3.Connection): + """SQLite DB Connection handler that plays well with eventlet.""" + def __init__(self, *args, **kwargs): + self.timeout = kwargs.get('timeout', BROKER_TIMEOUT) + kwargs['timeout'] = 0 + self.db_file = args and args[0] or '-' + sqlite3.Connection.__init__(self, *args, **kwargs) + + def _timeout(self, call): + with LockTimeout(self.timeout, self.db_file): + while True: + try: + return call() + except sqlite3.OperationalError, e: + if 'locked' not in str(e): + raise + sleep(0.05) + + def execute(self, *args, **kwargs): + return self._timeout(lambda: sqlite3.Connection.execute( + self, *args, **kwargs)) + + def commit(self): + return self._timeout(lambda: sqlite3.Connection.commit(self)) + + +def dict_factory(crs, row): + """ + This should only be used when you need a real dict, + i.e. when you're going to serialize the results. + """ + return dict( + ((col[0], row[idx]) for idx, col in enumerate(crs.description))) + + +def chexor(old, name, timestamp): + """ + Each entry in the account and container databases is XORed by the 128-bit + hash on insert or delete. This serves as a rolling, order-independent hash + of the contents. (check + XOR) + + :param old: hex representation of the current DB hash + :param name: name of the object or container being inserted + :param timestamp: timestamp of the new record + :returns: a hex representation of the new hash value + """ + if name is None: + raise Exception('name is None!') + old = old.decode('hex') + new = hashlib.md5(('%s-%s' % (name, timestamp)).encode('utf_8')).digest() + response = ''.join( + map(chr, map(operator.xor, map(ord, old), map(ord, new)))) + return response.encode('hex') + + +def get_db_connection(path, timeout=30, okay_to_create=False): + """ + Returns a properly configured SQLite database connection. + + :param path: path to DB + :param timeout: timeout for connection + :param okay_to_create: if True, create the DB if it doesn't exist + :returns: DB connection object + """ + try: + connect_time = time.time() + conn = sqlite3.connect(path, check_same_thread=False, + factory=GreenDBConnection, timeout=timeout) + if path != ':memory:' and not okay_to_create: + # attempt to detect and fail when connect creates the db file + stat = os.stat(path) + if stat.st_size == 0 and stat.st_ctime >= connect_time: + os.unlink(path) + raise DatabaseConnectionError(path, + 'DB file created by connect?') + conn.row_factory = sqlite3.Row + conn.text_factory = str + conn.execute('PRAGMA synchronous = NORMAL') + conn.execute('PRAGMA count_changes = OFF') + conn.execute('PRAGMA temp_store = MEMORY') + conn.create_function('chexor', 3, chexor) + except sqlite3.DatabaseError: + import traceback + raise DatabaseConnectionError(path, traceback.format_exc(), + timeout=timeout) + return conn + + +class DatabaseBroker(object): + """Encapsulates working with a database.""" + + def __init__(self, db_file, timeout=BROKER_TIMEOUT, logger=None, + account=None, container=None, pending_timeout=10, + stale_reads_ok=False): + """ Encapsulates working with a database. """ + self.conn = None + self.db_file = db_file + self.pending_file = self.db_file + '.pending' + self.pending_timeout = pending_timeout + self.stale_reads_ok = stale_reads_ok + self.db_dir = os.path.dirname(db_file) + self.timeout = timeout + self.logger = logger or logging.getLogger() + self.account = account + self.container = container + + def initialize(self, put_timestamp=None): + """ + Create the DB + + :param put_timestamp: timestamp of initial PUT request + """ + if self.db_file == ':memory:': + tmp_db_file = None + conn = get_db_connection(self.db_file, self.timeout) + else: + mkdirs(self.db_dir) + fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir) + os.close(fd) + conn = sqlite3.connect(tmp_db_file, check_same_thread=False, + factory=GreenDBConnection, timeout=0) + # creating dbs implicitly does a lot of transactions, so we + # pick fast, unsafe options here and do a big fsync at the end. + conn.execute('PRAGMA synchronous = OFF') + conn.execute('PRAGMA temp_store = MEMORY') + conn.execute('PRAGMA journal_mode = MEMORY') + conn.create_function('chexor', 3, chexor) + conn.row_factory = sqlite3.Row + conn.text_factory = str + conn.executescript(""" + CREATE TABLE outgoing_sync ( + remote_id TEXT UNIQUE, + sync_point INTEGER, + updated_at TEXT DEFAULT 0 + ); + CREATE TABLE incoming_sync ( + remote_id TEXT UNIQUE, + sync_point INTEGER, + updated_at TEXT DEFAULT 0 + ); + CREATE TRIGGER outgoing_sync_insert AFTER INSERT ON outgoing_sync + BEGIN + UPDATE outgoing_sync + SET updated_at = STRFTIME('%s', 'NOW') + WHERE ROWID = new.ROWID; + END; + CREATE TRIGGER outgoing_sync_update AFTER UPDATE ON outgoing_sync + BEGIN + UPDATE outgoing_sync + SET updated_at = STRFTIME('%s', 'NOW') + WHERE ROWID = new.ROWID; + END; + CREATE TRIGGER incoming_sync_insert AFTER INSERT ON incoming_sync + BEGIN + UPDATE incoming_sync + SET updated_at = STRFTIME('%s', 'NOW') + WHERE ROWID = new.ROWID; + END; + CREATE TRIGGER incoming_sync_update AFTER UPDATE ON incoming_sync + BEGIN + UPDATE incoming_sync + SET updated_at = STRFTIME('%s', 'NOW') + WHERE ROWID = new.ROWID; + END; + """) + if not put_timestamp: + put_timestamp = normalize_timestamp(0) + self._initialize(conn, put_timestamp) + conn.commit() + if tmp_db_file: + conn.close() + with open(tmp_db_file, 'r+b') as fp: + os.fsync(fp.fileno()) + with lock_parent_directory(self.db_file, self.pending_timeout): + if os.path.exists(self.db_file): + # It's as if there was a "condition" where different parts + # of the system were "racing" each other. + raise DatabaseConnectionError(self.db_file, + 'DB created by someone else while working?') + renamer(tmp_db_file, self.db_file) + self.conn = get_db_connection(self.db_file, self.timeout) + else: + self.conn = conn + + def delete_db(self, timestamp): + """ + Mark the DB as deleted + + :param timestamp: delete timestamp + """ + timestamp = normalize_timestamp(timestamp) + with self.get() as conn: + self._delete_db(conn, timestamp) + conn.commit() + + @contextmanager + def get(self): + """Use with the "with" statement; returns a database connection.""" + if not self.conn: + if self.db_file != ':memory:' and os.path.exists(self.db_file): + self.conn = get_db_connection(self.db_file, self.timeout) + else: + raise DatabaseConnectionError(self.db_file, "DB doesn't exist") + conn = self.conn + self.conn = None + try: + yield conn + conn.rollback() + self.conn = conn + except: + conn.close() + raise + + @contextmanager + def lock(self): + """Use with the "with" statement; locks a database.""" + if not self.conn: + if self.db_file != ':memory:' and os.path.exists(self.db_file): + self.conn = get_db_connection(self.db_file, self.timeout) + else: + raise DatabaseConnectionError(self.db_file, "DB doesn't exist") + conn = self.conn + self.conn = None + orig_isolation_level = conn.isolation_level + conn.isolation_level = None + conn.execute('BEGIN IMMEDIATE') + try: + yield True + except: + pass + try: + conn.execute('ROLLBACK') + conn.isolation_level = orig_isolation_level + self.conn = conn + except: # pragma: no cover + logging.exception( + 'Broker error trying to rollback locked connection') + conn.close() + + def newid(self, remote_id): + """ + Re-id the database. This should be called after an rsync. + + :param remote_id: the ID of the remote database being rsynced in + """ + with self.get() as conn: + row = conn.execute(''' + UPDATE %s_stat SET id=? + ''' % self.db_type, (str(uuid4()),)) + row = conn.execute(''' + SELECT ROWID FROM %s ORDER BY ROWID DESC LIMIT 1 + ''' % self.db_contains_type).fetchone() + sync_point = row['ROWID'] if row else -1 + conn.execute(''' + INSERT OR REPLACE INTO incoming_sync (sync_point, remote_id) + VALUES (?, ?) + ''', (sync_point, remote_id)) + self._newid(conn) + conn.commit() + + def _newid(self, conn): + # Override for additional work when receiving an rsynced db. + pass + + def merge_timestamps(self, created_at, put_timestamp, delete_timestamp): + """ + Used in replication to handle updating timestamps. + + :param created_at: create timestamp + :param put_timestamp: put timestamp + :param delete_timestamp: delete timestamp + """ + with self.get() as conn: + row = conn.execute(''' + UPDATE %s_stat SET created_at=MIN(?, created_at), + put_timestamp=MAX(?, put_timestamp), + delete_timestamp=MAX(?, delete_timestamp) + ''' % self.db_type, (created_at, put_timestamp, delete_timestamp)) + conn.commit() + + def get_items_since(self, start, count): + """ + Get a list of objects in the database between start and end. + + :param start: start ROWID + :param count: number to get + :returns: list of objects between start and end + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + curs = conn.execute(''' + SELECT * FROM %s WHERE ROWID > ? ORDER BY ROWID ASC LIMIT ? + ''' % self.db_contains_type, (start, count)) + curs.row_factory = dict_factory + return [r for r in curs] + + def get_sync(self, id, incoming=True): + """ + Gets the most recent sync point for a server from the sync table. + + :param id: remote ID to get the sync_point for + :param incoming: if True, get the last incoming sync, otherwise get + the last outgoing sync + :returns: the sync point, or -1 if the id doesn't exist. + """ + with self.get() as conn: + row = conn.execute( + "SELECT sync_point FROM %s_sync WHERE remote_id=?" + % ('incoming' if incoming else 'outgoing'), (id,)).fetchone() + if not row: + return -1 + return row['sync_point'] + + def get_syncs(self, incoming=True): + """ + Get a serialized copy of the sync table. + + :param incoming: if True, get the last incoming sync, otherwise get + the last outgoing sync + :returns: list of {'remote_id', 'sync_point'} + """ + with self.get() as conn: + curs = conn.execute(''' + SELECT remote_id, sync_point FROM %s_sync + ''' % 'incoming' if incoming else 'outgoing') + result = [] + for row in curs: + result.append({'remote_id': row[0], 'sync_point': row[1]}) + return result + + def get_replication_info(self): + """ + Get information about the DB required for replication. + + :returns: tuple of (hash, id, created_at, put_timestamp, + delete_timestamp) from the DB + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + curs = conn.execute(''' + SELECT hash, id, created_at, put_timestamp, delete_timestamp, + %s_count AS count, + CASE WHEN SQLITE_SEQUENCE.seq IS NOT NULL + THEN SQLITE_SEQUENCE.seq ELSE -1 END AS max_row + FROM (%s_stat LEFT JOIN SQLITE_SEQUENCE + ON SQLITE_SEQUENCE.name == '%s') LIMIT 1 + ''' % (self.db_contains_type, self.db_type, self.db_contains_type)) + curs.row_factory = dict_factory + return curs.fetchone() + + def _commit_puts(self): + pass # stub to be overridden if need be + + def merge_syncs(self, sync_points, incoming=True): + """ + Merge a list of sync points with the incoming sync table. + + :param sync_points: list of sync points where a sync point is a dict of + {'sync_point', 'remote_id'} + :param incoming: if True, get the last incoming sync, otherwise get + the last outgoing sync + """ + with self.get() as conn: + for rec in sync_points: + try: + conn.execute(''' + INSERT INTO %s_sync (sync_point, remote_id) + VALUES (?, ?) + ''' % ('incoming' if incoming else 'outgoing'), + (rec['sync_point'], rec['remote_id'])) + except sqlite3.IntegrityError: + conn.execute(''' + UPDATE %s_sync SET sync_point=max(?, sync_point) + WHERE remote_id=? + ''' % ('incoming' if incoming else 'outgoing'), + (rec['sync_point'], rec['remote_id'])) + conn.commit() + + def _preallocate(self): + """ + The idea is to allocate space in front of an expanding db. If it gets + within 512k of a boundary, it allocates to the next boundary. + Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after. + """ + if self.db_file == ':memory:': + return + MB = (1024 * 1024) + + def prealloc_points(): + for pm in (1, 2, 5, 10, 25, 50): + yield pm * MB + while True: + pm += 50 + yield pm * MB + + stat = os.stat(self.db_file) + file_size = stat.st_size + allocated_size = stat.st_blocks * 512 + for point in prealloc_points(): + if file_size <= point - MB / 2: + prealloc_size = point + break + if allocated_size < prealloc_size: + with open(self.db_file, 'rb+') as fp: + fallocate(fp.fileno(), int(prealloc_size)) + + +class ContainerBroker(DatabaseBroker): + """Encapsulates working with a container database.""" + db_type = 'container' + db_contains_type = 'object' + + def _initialize(self, conn, put_timestamp): + """Creates a brand new database (tables, indices, triggers, etc.)""" + if not self.account: + raise ValueError( + 'Attempting to create a new database with no account set') + if not self.container: + raise ValueError( + 'Attempting to create a new database with no container set') + self.create_object_table(conn) + self.create_container_stat_table(conn, put_timestamp) + + def create_object_table(self, conn): + """ + Create the object table which is specifc to the container DB. + + :param conn: DB connection object + """ + conn.executescript(""" + CREATE TABLE object ( + ROWID INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE, + created_at TEXT, + size INTEGER, + content_type TEXT, + etag TEXT, + deleted INTEGER DEFAULT 0 + ); + + CREATE INDEX ix_object_deleted ON object (deleted); + + CREATE TRIGGER object_insert AFTER INSERT ON object + BEGIN + UPDATE container_stat + SET object_count = object_count + (1 - new.deleted), + bytes_used = bytes_used + new.size, + hash = chexor(hash, new.name, new.created_at); + END; + + CREATE TRIGGER object_update BEFORE UPDATE ON object + BEGIN + SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); + END; + + CREATE TRIGGER object_delete AFTER DELETE ON object + BEGIN + UPDATE container_stat + SET object_count = object_count - (1 - old.deleted), + bytes_used = bytes_used - old.size, + hash = chexor(hash, old.name, old.created_at); + END; + """) + + def create_container_stat_table(self, conn, put_timestamp=None): + """ + Create the container_stat table which is specifc to the container DB. + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + if put_timestamp is None: + put_timestamp = normalize_timestamp(0) + conn.executescript(""" + CREATE TABLE container_stat ( + account TEXT, + container TEXT, + created_at TEXT, + put_timestamp TEXT DEFAULT '0', + delete_timestamp TEXT DEFAULT '0', + object_count INTEGER, + bytes_used INTEGER, + reported_put_timestamp TEXT DEFAULT '0', + reported_delete_timestamp TEXT DEFAULT '0', + reported_object_count INTEGER DEFAULT 0, + reported_bytes_used INTEGER DEFAULT 0, + hash TEXT default '00000000000000000000000000000000', + id TEXT, + status TEXT DEFAULT '', + status_changed_at TEXT DEFAULT '0' + ); + + INSERT INTO container_stat (object_count, bytes_used) + VALUES (0, 0); + """) + conn.execute(''' + UPDATE container_stat + SET account = ?, container = ?, created_at = ?, id = ?, + put_timestamp = ? + ''', (self.account, self.container, normalize_timestamp(time.time()), + str(uuid4()), put_timestamp)) + + def _newid(self, conn): + conn.execute(''' + UPDATE container_stat + SET reported_put_timestamp = 0, reported_delete_timestamp = 0, + reported_object_count = 0, reported_bytes_used = 0''') + + def update_put_timestamp(self, timestamp): + """ + Update the put_timestamp. Only modifies it if it is greater than + the current timestamp. + + :param timestamp: put timestamp + """ + with self.get() as conn: + conn.execute(''' + UPDATE container_stat SET put_timestamp = ? + WHERE put_timestamp < ? ''', (timestamp, timestamp)) + conn.commit() + + def _delete_db(self, conn, timestamp): + """ + Mark the DB as deleted + + :param conn: DB connection object + :param timestamp: timestamp to mark as deleted + """ + conn.execute(""" + UPDATE container_stat + SET delete_timestamp = ?, + status = 'DELETED', + status_changed_at = ? + WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) + + def empty(self): + """ + Check if the DB is empty. + + :returns: True if the database has no active objects, False otherwise + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + row = conn.execute( + 'SELECT object_count from container_stat').fetchone() + return (row[0] == 0) + + def _commit_puts(self, item_list=None): + """Handles commiting rows in .pending files.""" + if self.db_file == ':memory:' or not os.path.exists(self.pending_file): + return + if item_list is None: + item_list = [] + with lock_parent_directory(self.pending_file, self.pending_timeout): + self._preallocate() + if not os.path.getsize(self.pending_file): + if item_list: + self.merge_items(item_list) + return + with open(self.pending_file, 'r+b') as fp: + for entry in fp.read().split(':'): + if entry: + try: + (name, timestamp, size, content_type, etag, + deleted) = pickle.loads(entry.decode('base64')) + item_list.append({'name': name, 'created_at': + timestamp, 'size': size, 'content_type': + content_type, 'etag': etag, + 'deleted': deleted}) + except: + self.logger.exception( + 'Invalid pending entry %s: %s' + % (self.pending_file, entry)) + if item_list: + self.merge_items(item_list) + try: + os.ftruncate(fp.fileno(), 0) + except OSError, err: + if err.errno != errno.ENOENT: + raise + + def reclaim(self, object_timestamp, sync_timestamp): + """ + Delete rows from the object table that are marked deleted and + whose created_at timestamp is < object_timestamp. Also deletes rows + from incoming_sync and outgoing_sync where the updated_at timestamp is + < sync_timestamp. + + :param object_timestamp: max created_at timestamp of object rows to + delete + :param sync_timestamp: max update_at timestamp of sync rows to delete + """ + self._commit_puts() + with self.get() as conn: + conn.execute(""" + DELETE FROM object + WHERE deleted = 1 + AND created_at < ?""", (object_timestamp,)) + try: + conn.execute(''' + DELETE FROM outgoing_sync WHERE updated_at < ? + ''', (sync_timestamp,)) + conn.execute(''' + DELETE FROM incoming_sync WHERE updated_at < ? + ''', (sync_timestamp,)) + except sqlite3.OperationalError, err: + # Old dbs didn't have updated_at in the _sync tables. + if 'no such column: updated_at' not in str(err): + raise + conn.commit() + + def delete_object(self, name, timestamp): + """ + Mark an object deleted. + + :param name: object name to be deleted + :param timestamp: timestamp when the object was marked as deleted + """ + self.put_object(name, timestamp, 0, 'application/deleted', 'noetag', 1) + + def put_object(self, name, timestamp, size, content_type, etag, deleted=0): + """ + Creates an object in the DB with its metadata. + + :param name: object name to be created + :param timestamp: timestamp of when the object was created + :param size: object size + :param content_type: object content-type + :param etag: object etag + :param deleted: if True, marks the object as deleted and sets the + deteleted_at timestamp to timestamp + """ + record = {'name': name, 'created_at': timestamp, 'size': size, + 'content_type': content_type, 'etag': etag, + 'deleted': deleted} + if self.db_file == ':memory:': + self.merge_items([record]) + return + if not os.path.exists(self.db_file): + raise DatabaseConnectionError(self.db_file, "DB doesn't exist") + pending_size = 0 + try: + pending_size = os.path.getsize(self.pending_file) + except OSError, err: + if err.errno != errno.ENOENT: + raise + if pending_size > PENDING_CAP: + self._commit_puts([record]) + else: + with lock_parent_directory( + self.pending_file, self.pending_timeout): + with open(self.pending_file, 'a+b') as fp: + # Colons aren't used in base64 encoding; so they are our + # delimiter + fp.write(':') + fp.write(pickle.dumps( + (name, timestamp, size, content_type, etag, deleted), + protocol=PICKLE_PROTOCOL).encode('base64')) + fp.flush() + + def is_deleted(self, timestamp=None): + """ + Check if the DB is considered to be deleted. + + :returns: True if the DB is considered to be deleted, False otherwise + """ + if self.db_file != ':memory:' and not os.path.exists(self.db_file): + return True + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + row = conn.execute(''' + SELECT put_timestamp, delete_timestamp, object_count + FROM container_stat''').fetchone() + # leave this db as a tombstone for a consistency window + if timestamp and row['delete_timestamp'] > timestamp: + return False + # The container is considered deleted if the delete_timestamp + # value is greater than the put_timestamp, and there are no + # objects in the container. + return (row['object_count'] in (None, '', 0, '0')) and \ + (float(row['delete_timestamp']) > float(row['put_timestamp'])) + + def get_info(self): + """ + Get global data for the container. + + :returns: a tuple of (account, container, created_at, put_timestamp, + delete_timestamp, object_count, bytes_used, + reported_put_timestamp, reported_delete_timestamp, + reported_object_count, reported_bytes_used, hash, id) + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + return conn.execute(''' + SELECT account, container, created_at, put_timestamp, + delete_timestamp, object_count, bytes_used, + reported_put_timestamp, reported_delete_timestamp, + reported_object_count, reported_bytes_used, hash, id + FROM container_stat + ''').fetchone() + + def reported(self, put_timestamp, delete_timestamp, object_count, + bytes_used): + """ + Update reported stats. + + :param put_timestamp: put_timestamp to update + :param delete_timestamp: delete_timestamp to update + :param object_count: object_count to update + :param bytes_used: bytes_used to update + """ + with self.get() as conn: + conn.execute(''' + UPDATE container_stat + SET reported_put_timestamp = ?, reported_delete_timestamp = ?, + reported_object_count = ?, reported_bytes_used = ? + ''', (put_timestamp, delete_timestamp, object_count, bytes_used)) + conn.commit() + + def get_random_objects(self, max_count=100): + """ + Get random objects from the DB. This is used by the container_auditor + when testing random objects for existence. + + :param max_count: maximum number of objects to get + + :returns: list of object names + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + rv = [] + with self.get() as conn: + row = conn.execute(''' + SELECT ROWID FROM object ORDER BY ROWID DESC LIMIT 1 + ''').fetchone() + if not row: + return [] + max_rowid = row['ROWID'] + for _ in xrange(min(max_count, max_rowid)): + row = conn.execute(''' + SELECT name FROM object WHERE ROWID >= ? AND +deleted = 0 + LIMIT 1 + ''', (randint(0, max_rowid),)).fetchone() + if row: + rv.append(row['name']) + return list(set(rv)) + + def list_objects_iter(self, limit, marker, prefix, delimiter, path=None, + format=None): + """ + Get a list of objects sorted by name starting at marker onward, up + to limit entries. Entries will begin with the prefix and will not + have the delimiter after the prefix. + + :param limit: maximum number of entries to get + :param marker: marker query + :param prefix: prefix query + :param delimeter: delimeter for query + :param path: if defined, will set the prefix and delimter based on + the path + :param format: TOOD: remove as it is no longer used + + :returns: list of tuples of (name, created_at, size, content_type, + etag) + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + if path is not None: + prefix = path + if path: + prefix = path = path.rstrip('/') + '/' + delimiter = '/' + elif delimiter and not prefix: + prefix = '' + orig_marker = marker + with self.get() as conn: + results = [] + while len(results) < limit: + query = '''SELECT name, created_at, size, content_type, etag + FROM object WHERE''' + query_args = [] + if marker and marker >= prefix: + query += ' name > ? AND' + query_args.append(marker) + elif prefix: + query += ' name >= ? AND' + query_args.append(prefix) + query += ' +deleted = 0 ORDER BY name LIMIT ?' + query_args.append(limit - len(results)) + curs = conn.execute(query, query_args) + curs.row_factory = None + + if prefix is None: + return [r for r in curs] + if not delimiter: + return [r for r in curs if r[0].startswith(prefix)] + rowcount = 0 + for row in curs: + rowcount += 1 + marker = name = row[0] + if len(results) >= limit or not name.startswith(prefix): + curs.close() + return results + end = name.find(delimiter, len(prefix)) + if path is not None: + if name == path: + continue + if end >= 0 and len(name) > end + len(delimiter): + marker = name[:end] + chr(ord(delimiter) + 1) + curs.close() + break + elif end > 0: + marker = name[:end] + chr(ord(delimiter) + 1) + dir_name = name[:end + 1] + if dir_name != orig_marker: + results.append([dir_name, '0', 0, None, '']) + curs.close() + break + results.append(row) + if not rowcount: + break + return results + + def merge_items(self, item_list, source=None): + """ + Merge items into the object table. + + :param item_list: list of dictionaries of {'name', 'created_at', + 'size', 'content_type', 'etag', 'deleted'} + :param source: if defined, update incoming_sync with the source + """ + with self.get() as conn: + max_rowid = -1 + for rec in item_list: + curs = conn.execute(''' + DELETE FROM object WHERE name = ? AND + (created_at < ?) + ''', (rec['name'], rec['created_at'])) + try: + conn.execute(''' + INSERT INTO object (name, created_at, size, + content_type, etag, deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', ([rec['name'], rec['created_at'], rec['size'], + rec['content_type'], rec['etag'], rec['deleted']])) + except sqlite3.IntegrityError: + pass + if source: + max_rowid = max(max_rowid, rec['ROWID']) + if source: + try: + conn.execute(''' + INSERT INTO incoming_sync (sync_point, remote_id) + VALUES (?, ?) + ''', (max_rowid, source)) + except sqlite3.IntegrityError: + conn.execute(''' + UPDATE incoming_sync SET sync_point=max(?, sync_point) + WHERE remote_id=? + ''', (max_rowid, source)) + conn.commit() + + +class AccountBroker(DatabaseBroker): + """Encapsulates working with a account database.""" + db_type = 'account' + db_contains_type = 'container' + + def _initialize(self, conn, put_timestamp): + """ + Create a brand new database (tables, indices, triggers, etc.) + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + if not self.account: + raise ValueError( + 'Attempting to create a new database with no account set') + self.create_container_table(conn) + self.create_account_stat_table(conn, put_timestamp) + + def create_container_table(self, conn): + """ + Create container table which is specific to the account DB. + + :param conn: DB connection object + """ + conn.executescript(""" + CREATE TABLE container ( + ROWID INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE, + put_timestamp TEXT, + delete_timestamp TEXT, + object_count INTEGER, + bytes_used INTEGER, + deleted INTEGER DEFAULT 0 + ); + + CREATE INDEX ix_container_deleted ON container (deleted); + CREATE INDEX ix_container_name ON container (name); + CREATE TRIGGER container_insert AFTER INSERT ON container + BEGIN + UPDATE account_stat + SET container_count = container_count + (1 - new.deleted), + object_count = object_count + new.object_count, + bytes_used = bytes_used + new.bytes_used, + hash = chexor(hash, new.name, + new.put_timestamp || '-' || + new.delete_timestamp || '-' || + new.object_count || '-' || new.bytes_used); + END; + + CREATE TRIGGER container_update BEFORE UPDATE ON container + BEGIN + SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); + END; + + + CREATE TRIGGER container_delete AFTER DELETE ON container + BEGIN + UPDATE account_stat + SET container_count = container_count - (1 - old.deleted), + object_count = object_count - old.object_count, + bytes_used = bytes_used - old.bytes_used, + hash = chexor(hash, old.name, + old.put_timestamp || '-' || + old.delete_timestamp || '-' || + old.object_count || '-' || old.bytes_used); + END; + """) + + def create_account_stat_table(self, conn, put_timestamp): + """ + Create account_stat table which is specific to the account DB. + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + conn.executescript(""" + CREATE TABLE account_stat ( + account TEXT, + created_at TEXT, + put_timestamp TEXT DEFAULT '0', + delete_timestamp TEXT DEFAULT '0', + container_count INTEGER, + object_count INTEGER DEFAULT 0, + bytes_used INTEGER DEFAULT 0, + hash TEXT default '00000000000000000000000000000000', + id TEXT, + status TEXT DEFAULT '', + status_changed_at TEXT DEFAULT '0' + ); + + INSERT INTO account_stat (container_count) VALUES (0); + """) + + conn.execute(''' + UPDATE account_stat SET account = ?, created_at = ?, id = ?, + put_timestamp = ? + ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), + put_timestamp)) + + def update_put_timestamp(self, timestamp): + """ + Update the put_timestamp. Only modifies it if it is greater than + the current timestamp. + + :param timestamp: put timestamp + """ + with self.get() as conn: + conn.execute(''' + UPDATE account_stat SET put_timestamp = ? + WHERE put_timestamp < ? ''', (timestamp, timestamp)) + conn.commit() + + def _delete_db(self, conn, timestamp, force=False): + """ + Mark the DB as deleted. + + :param conn: DB connection object + :param timestamp: timestamp to mark as deleted + """ + conn.execute(""" + UPDATE account_stat + SET delete_timestamp = ?, + status = 'DELETED', + status_changed_at = ? + WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) + + def _commit_puts(self, item_list=None): + """Handles commiting rows in .pending files.""" + if self.db_file == ':memory:' or not os.path.exists(self.pending_file): + return + if item_list is None: + item_list = [] + with lock_parent_directory(self.pending_file, self.pending_timeout): + self._preallocate() + if not os.path.getsize(self.pending_file): + if item_list: + self.merge_items(item_list) + return + with open(self.pending_file, 'r+b') as fp: + for entry in fp.read().split(':'): + if entry: + try: + (name, put_timestamp, delete_timestamp, + object_count, bytes_used, deleted) = \ + pickle.loads(entry.decode('base64')) + item_list.append({'name': name, + 'put_timestamp': put_timestamp, + 'delete_timestamp': delete_timestamp, + 'object_count': object_count, + 'bytes_used': bytes_used, + 'deleted': deleted}) + except: + self.logger.exception( + 'Invalid pending entry %s: %s' + % (self.pending_file, entry)) + if item_list: + self.merge_items(item_list) + try: + os.ftruncate(fp.fileno(), 0) + except OSError, err: + if err.errno != errno.ENOENT: + raise + + def empty(self): + """ + Check if the account DB is empty. + + :returns: True if the database has no active containers. + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + row = conn.execute( + 'SELECT container_count from account_stat').fetchone() + return (row[0] == 0) + + def reclaim(self, container_timestamp, sync_timestamp): + """ + Delete rows from the container table that are marked deleted and + whose created_at timestamp is < object_timestamp. Also deletes rows + from incoming_sync and outgoing_sync where the updated_at timestamp is + < sync_timestamp. + + :param object_timestamp: max created_at timestamp of container rows to + delete + :param sync_timestamp: max update_at timestamp of sync rows to delete + """ + + self._commit_puts() + with self.get() as conn: + conn.execute(''' + DELETE FROM container WHERE + deleted = 1 AND delete_timestamp < ? + ''', (container_timestamp,)) + try: + conn.execute(''' + DELETE FROM outgoing_sync WHERE updated_at < ? + ''', (sync_timestamp,)) + conn.execute(''' + DELETE FROM incoming_sync WHERE updated_at < ? + ''', (sync_timestamp,)) + except sqlite3.OperationalError, err: + # Old dbs didn't have updated_at in the _sync tables. + if 'no such column: updated_at' not in str(err): + raise + conn.commit() + + def get_container_timestamp(self, container_name): + """ + Get the put_timestamp of a container. + + :param container_name: container name + + :returns: put_timestamp of the container + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + ret = conn.execute(''' + SELECT put_timestamp FROM container + WHERE name = ? AND deleted != 1''', + (container_name,)).fetchone() + if ret: + ret = ret[0] + return ret + + def put_container(self, name, put_timestamp, delete_timestamp, + object_count, bytes_used): + """ + Create a container with the given attributes. + + :param name: name of the container to create + :param put_timestamp: put_timestamp of the container to create + :param delete_timestamp: delete_timestamp of the container to create + :param object_count: number of objects in the container + :param bytes_used: number of bytes used by the container + """ + if delete_timestamp > put_timestamp and \ + object_count in (None, '', 0, '0'): + deleted = 1 + else: + deleted = 0 + record = {'name': name, 'put_timestamp': put_timestamp, + 'delete_timestamp': delete_timestamp, + 'object_count': object_count, + 'bytes_used': bytes_used, + 'deleted': deleted} + if self.db_file == ':memory:': + self.merge_items([record]) + return + commit = False + with lock_parent_directory(self.pending_file, self.pending_timeout): + with open(self.pending_file, 'a+b') as fp: + # Colons aren't used in base64 encoding; so they are our + # delimiter + fp.write(':') + fp.write(pickle.dumps( + (name, put_timestamp, delete_timestamp, object_count, + bytes_used, deleted), + protocol=PICKLE_PROTOCOL).encode('base64')) + fp.flush() + if fp.tell() > PENDING_CAP: + commit = True + if commit: + self._commit_puts() + + def can_delete_db(self, cutoff): + """ + Check if the accont DB can be deleted. + + :returns: True if the account can be deleted, False otherwise + """ + self._commit_puts() + with self.get() as conn: + row = conn.execute(''' + SELECT status, put_timestamp, delete_timestamp, container_count + FROM account_stat''').fetchone() + # The account is considered deleted if its status is marked + # as 'DELETED" and the delete_timestamp is older than the supplied + # cutoff date; or if the delete_timestamp value is greater than + # the put_timestamp, and there are no containers for the account + status_del = (row['status'] == 'DELETED') + deltime = float(row['delete_timestamp']) + past_cutoff = (deltime < cutoff) + time_later = (row['delete_timestamp'] > row['put_timestamp']) + no_containers = (row['container_count'] in (None, '', 0, '0')) + return ( + (status_del and past_cutoff) or (time_later and no_containers)) + + def is_deleted(self): + """ + Check if the account DB is considered to be deleted. + + :returns: True if the account DB is considered to be deleted, False + otherwise + """ + if self.db_file != ':memory:' and not os.path.exists(self.db_file): + return True + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + row = conn.execute(''' + SELECT put_timestamp, delete_timestamp, container_count, status + FROM account_stat''').fetchone() + return row['status'] == 'DELETED' or ( + row['container_count'] in (None, '', 0, '0') and + row['delete_timestamp'] > row['put_timestamp']) + + def is_status_deleted(self): + """Only returns true if the status field is set to DELETED.""" + with self.get() as conn: + row = conn.execute(''' + SELECT status + FROM account_stat''').fetchone() + return (row['status'] == "DELETED") + + def get_info(self): + """ + Get global data for the account. + + :returns: a tuple of (account, created_at, put_timestamp, + delete_timestamp, container_count, object_count, + bytes_used, hash, id) + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + with self.get() as conn: + return conn.execute(''' + SELECT account, created_at, put_timestamp, delete_timestamp, + container_count, object_count, bytes_used, hash, id + FROM account_stat + ''').fetchone() + + def get_random_containers(self, max_count=100): + """ + Get random containers from the DB. This is used by the + account_auditor when testing random containerss for existence. + + :param max_count: maximum number of containers to get + + :returns: list of container names + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + rv = [] + with self.get() as conn: + row = conn.execute(''' + SELECT ROWID FROM container ORDER BY ROWID DESC LIMIT 1 + ''').fetchone() + if not row: + return [] + max_rowid = row['ROWID'] + for _ in xrange(min(max_count, max_rowid)): + row = conn.execute(''' + SELECT name FROM container WHERE + ROWID >= ? AND +deleted = 0 + LIMIT 1 + ''', (randint(0, max_rowid),)).fetchone() + if row: + rv.append(row['name']) + return list(set(rv)) + + def list_containers_iter(self, limit, marker, prefix, delimiter): + """ + Get a list of containerss sorted by name starting at marker onward, up + to limit entries. Entries will begin with the prefix and will not + have the delimiter after the prefix. + + :param limit: maximum number of entries to get + :param marker: marker query + :param prefix: prefix query + :param delimeter: delimeter for query + + :returns: list of tuples of (name, object_count, bytes_used, 0) + """ + try: + self._commit_puts() + except LockTimeout: + if not self.stale_reads_ok: + raise + if delimiter and not prefix: + prefix = '' + orig_marker = marker + with self.get() as conn: + results = [] + while len(results) < limit: + query = """ + SELECT name, object_count, bytes_used, 0 + FROM container + WHERE deleted = 0 AND """ + query_args = [] + if marker and marker >= prefix: + query += ' name > ? AND' + query_args.append(marker) + elif prefix: + query += ' name >= ? AND' + query_args.append(prefix) + query += ' +deleted = 0 ORDER BY name LIMIT ?' + query_args.append(limit - len(results)) + curs = conn.execute(query, query_args) + curs.row_factory = None + + if prefix is None: + return [r for r in curs] + if not delimiter: + return [r for r in curs if r[0].startswith(prefix)] + rowcount = 0 + for row in curs: + rowcount += 1 + marker = name = row[0] + if len(results) >= limit or not name.startswith(prefix): + curs.close() + return results + end = name.find(delimiter, len(prefix)) + if end > 0: + marker = name[:end] + chr(ord(delimiter) + 1) + dir_name = name[:end + 1] + if dir_name != orig_marker: + results.append([dir_name, 0, 0, 1]) + curs.close() + break + results.append(row) + if not rowcount: + break + return results + + def merge_items(self, item_list, source=None): + """ + Merge items into the container table. + + :param item_list: list of dictionaries of {'name', 'put_timestamp', + 'delete_timestamp', 'object_count', 'bytes_used', + 'deleted'} + :param source: if defined, update incoming_sync with the source + """ + with self.get() as conn: + max_rowid = -1 + for rec in item_list: + record = [rec['name'], rec['put_timestamp'], + rec['delete_timestamp'], rec['object_count'], + rec['bytes_used'], rec['deleted']] + try: + conn.execute(''' + INSERT INTO container (name, put_timestamp, + delete_timestamp, object_count, bytes_used, + deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', record) + except sqlite3.IntegrityError: + curs = conn.execute(''' + SELECT name, put_timestamp, delete_timestamp, + object_count, bytes_used, deleted + FROM container WHERE name = ? AND + (put_timestamp < ? OR delete_timestamp < ? OR + object_count != ? OR bytes_used != ?)''', + (rec['name'], rec['put_timestamp'], + rec['delete_timestamp'], rec['object_count'], + rec['bytes_used'])) + curs.row_factory = None + row = curs.fetchone() + if row: + row = list(row) + for i in xrange(5): + if record[i] is None and row[i] is not None: + record[i] = row[i] + if row[1] > record[1]: # Keep newest put_timestamp + record[1] = row[1] + if row[2] > record[2]: # Keep newest delete_timestamp + record[2] = row[2] + conn.execute('DELETE FROM container WHERE name = ?', + (record[0],)) + # If deleted, mark as such + if record[2] > record[1] and \ + record[3] in (None, '', 0, '0'): + record[5] = 1 + else: + record[5] = 0 + try: + conn.execute(''' + INSERT INTO container (name, put_timestamp, + delete_timestamp, object_count, bytes_used, + deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', record) + except sqlite3.IntegrityError: + continue + if source: + max_rowid = max(max_rowid, rec['ROWID']) + if source: + try: + conn.execute(''' + INSERT INTO incoming_sync (sync_point, remote_id) + VALUES (?, ?) + ''', (max_rowid, source)) + except sqlite3.IntegrityError: + conn.execute(''' + UPDATE incoming_sync SET sync_point=max(?, sync_point) + WHERE remote_id=? + ''', (max_rowid, source)) + conn.commit() diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py new file mode 100644 index 0000000000..b8df33038e --- /dev/null +++ b/swift/common/db_replicator.py @@ -0,0 +1,526 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import sys +import os +import random +import math +import time +import shutil + +from eventlet import GreenPool, sleep, Timeout +from eventlet.green import subprocess +import simplejson +from webob import Response +from webob.exc import HTTPNotFound, HTTPNoContent, HTTPAccepted, \ + HTTPInsufficientStorage, HTTPBadRequest + +from swift.common.utils import get_logger, whataremyips, storage_directory, \ + renamer, mkdirs, lock_parent_directory, unlink_older_than, LoggerFileObject +from swift.common import ring +from swift.common.bufferedhttp import BufferedHTTPConnection +from swift.common.exceptions import DriveNotMounted, ConnectionTimeout + + +def quarantine_db(object_file, server_type): + """ + In the case that a corrupt file is found, move it to a quarantined area to + allow replication to fix it. + + :param object_file: path to corrupt file + :param server_type: type of file that is corrupt + ('container' or 'account') + """ + object_dir = os.path.dirname(object_file) + quarantine_dir = os.path.abspath(os.path.join(object_dir, '..', + '..', '..', '..', 'quarantined', server_type + 's', + os.path.basename(object_dir))) + renamer(object_dir, quarantine_dir) + + +class ReplConnection(BufferedHTTPConnection): + """ + Helper to simplify POSTing to a remote server. + """ + def __init__(self, node, partition, hash_, logger): + "" + self.logger = logger + self.node = node + BufferedHTTPConnection.__init__(self, '%(ip)s:%(port)s' % node) + self.path = '/%s/%s/%s' % (node['device'], partition, hash_) + + def post(self, *args): + """ + Make an HTTP POST request + + :param args: list of json-encodable objects + + :returns: httplib response object + """ + try: + body = simplejson.dumps(args) + self.request('POST', self.path, body, + {'Content-Type': 'application/json'}) + response = self.getresponse() + response.data = response.read() + return response + except: + self.logger.exception( + 'ERROR reading HTTP response from %s' % self.node) + return None + + +class Replicator(object): + """ + Implements the logic for directing db replication. + """ + + def __init__(self, server_conf, replicator_conf): + self.logger = \ + get_logger(replicator_conf, '%s-replicator' % self.server_type) + # log uncaught exceptions + sys.excepthook = lambda *exc_info: \ + self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + sys.stdout = sys.stderr = LoggerFileObject(self.logger) + self.root = server_conf.get('devices', '/srv/node') + self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.port = int(server_conf.get('bind_port', self.default_port)) + concurrency = int(replicator_conf.get('concurrency', 8)) + self.cpool = GreenPool(size=concurrency) + swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file)) + self.per_diff = int(replicator_conf.get('per_diff', 1000)) + self.run_pause = int(replicator_conf.get('run_pause', 30)) + self.vm_test_mode = replicator_conf.get( + 'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1') + self.node_timeout = int(replicator_conf.get('node_timeout', 10)) + self.conn_timeout = float(replicator_conf.get('conn_timeout', 0.5)) + self.reclaim_age = float(replicator_conf.get('reclaim_age', 86400 * 7)) + self._zero_stats() + + def _zero_stats(self): + """Zero out the stats.""" + self.stats = {'attempted': 0, 'success': 0, 'failure': 0, 'ts_repl': 0, + 'no_change': 0, 'hashmatch': 0, 'rsync': 0, 'diff': 0, + 'remove': 0, 'empty': 0, 'remote_merge': 0, + 'start': time.time()} + + def _report_stats(self): + """Report the current stats to the logs.""" + self.logger.info( + 'Attempted to replicate %d dbs in %.5f seconds (%.5f/s)' + % (self.stats['attempted'], time.time() - self.stats['start'], + self.stats['attempted'] / + (time.time() - self.stats['start'] + 0.0000001))) + self.logger.info('Removed %(remove)d dbs' % self.stats) + self.logger.info('%(success)s successes, %(failure)s failures' + % self.stats) + self.logger.info(' '.join(['%s:%s' % item for item in + self.stats.items() if item[0] in + ('no_change', 'hashmatch', 'rsync', 'diff', 'ts_repl', 'empty')])) + + def _rsync_file(self, db_file, remote_file, whole_file=True): + """ + Sync a single file using rsync. Used by _rsync_db to handle syncing. + + :param db_file: file to be synced + :param remote_file: remote location to sync the DB file to + :param whole-file: if True, uses rsync's --whole-file flag + + :returns: True if the sync was successful, False otherwise + """ + popen_args = ['rsync', '--quiet', '--no-motd', + '--timeout=%s' % int(math.ceil(self.node_timeout)), + '--contimeout=%s' % int(math.ceil(self.conn_timeout))] + if whole_file: + popen_args.append('--whole-file') + popen_args.extend([db_file, remote_file]) + proc = subprocess.Popen(popen_args) + proc.communicate() + if proc.returncode != 0: + self.logger.error('ERROR rsync failed with %s: %s' % + (proc.returncode, popen_args)) + return proc.returncode == 0 + + def _rsync_db(self, broker, device, http, local_id, + post_method='complete_rsync', post_timeout=None): + """ + Sync a whole db using rsync. + + :param broker: DB broker object of DB to be synced + :param device: device to sync to + :param http: ReplConnection object + :param local_id: unique ID of the local database replica + :param post_method: remote operation to perform after rsync + :param post_timeout: timeout to wait in seconds + """ + if self.vm_test_mode: + remote_file = '%s::%s%s/%s/tmp/%s' % (device['ip'], + self.server_type, device['port'], device['device'], + local_id) + else: + remote_file = '%s::%s/%s/tmp/%s' % (device['ip'], + self.server_type, device['device'], local_id) + mtime = os.path.getmtime(broker.db_file) + if not self._rsync_file(broker.db_file, remote_file): + return False + # perform block-level sync if the db was modified during the first sync + if os.path.exists(broker.db_file + '-journal') or \ + os.path.getmtime(broker.db_file) > mtime: + # grab a lock so nobody else can modify it + with broker.lock(): + if not self._rsync_file(broker.db_file, remote_file, False): + return False + with Timeout(post_timeout or self.node_timeout): + response = http.post(post_method, local_id) + return response and response.status >= 200 and response.status < 300 + + def _usync_db(self, point, broker, http, remote_id, local_id): + """ + Sync a db by sending all records since the last sync. + + :param point: synchronization high water mark between the replicas + :param broker: database broker object + :param http: ReplConnection object for the remote server + :param remote_id: database id for the remote replica + :param local_id: database id for the local replica + + :returns: boolean indicating completion and success + """ + self.stats['diff'] += 1 + self.logger.debug('Syncing chunks with %s', http.host) + sync_table = broker.get_syncs() + objects = broker.get_items_since(point, self.per_diff) + while len(objects): + with Timeout(self.node_timeout): + response = http.post('merge_items', objects, local_id) + if not response or response.status >= 300 or response.status < 200: + if response: + self.logger.error('ERROR Bad response %s from %s' % + (response.status, http.host)) + return False + point = objects[-1]['ROWID'] + objects = broker.get_items_since(point, self.per_diff) + with Timeout(self.node_timeout): + response = http.post('merge_syncs', sync_table) + if response and response.status >= 200 and response.status < 300: + broker.merge_syncs([{'remote_id': remote_id, + 'sync_point': point}], incoming=False) + return True + return False + + def _in_sync(self, rinfo, info, broker, local_sync): + """ + Determine whether or not two replicas of a databases are considered + to be in sync. + + :param rinfo: remote database info + :param info: local database info + :param broker: database broker object + :param local_sync: cached last sync point between replicas + + :returns: boolean indicating whether or not the replicas are in sync + """ + if max(rinfo['point'], local_sync) >= info['max_row']: + self.stats['no_change'] += 1 + return True + if rinfo['hash'] == info['hash']: + self.stats['hashmatch'] += 1 + broker.merge_syncs([{'remote_id': rinfo['id'], + 'sync_point': rinfo['point']}], incoming=False) + return True + + def _http_connect(self, node, partition, db_file): + """ + Make an http_connection using ReplConnection + + :param node: node dictionary from the ring + :param partition: partition partition to send in the url + :param db_file: DB file + + :returns: ReplConnection object + """ + return ReplConnection(node, partition, + os.path.basename(db_file).split('.', 1)[0], self.logger) + + def _repl_to_node(self, node, broker, partition, info): + """ + Replicate a database to a node. + + :param node: node dictionary from the ring to be replicated to + :param broker: DB broker for the DB to be replication + :param partition: partition on the node to replicate to + :param info: DB info as a dictionary of {'max_row', 'hash', 'id', + 'created_at', 'put_timestamp', 'delete_timestamp'} + + :returns: True if successful, False otherwise + """ + with ConnectionTimeout(self.conn_timeout): + http = self._http_connect(node, partition, broker.db_file) + if not http: + self.logger.error( + 'ERROR Unable to connect to remote server: %s' % node) + return False + with Timeout(self.node_timeout): + response = http.post('sync', info['max_row'], info['hash'], + info['id'], info['created_at'], info['put_timestamp'], + info['delete_timestamp']) + if not response: + return False + elif response.status == HTTPNotFound.code: # completely missing, rsync + self.stats['rsync'] += 1 + return self._rsync_db(broker, node, http, info['id']) + elif response.status == HTTPInsufficientStorage.code: + raise DriveNotMounted() + elif response.status >= 200 and response.status < 300: + rinfo = simplejson.loads(response.data) + local_sync = broker.get_sync(rinfo['id'], incoming=False) + if self._in_sync(rinfo, info, broker, local_sync): + return True + # if the difference in rowids between the two differs by + # more than 50%, rsync then do a remote merge. + if rinfo['max_row'] / float(info['max_row']) < 0.5: + self.stats['remote_merge'] += 1 + return self._rsync_db(broker, node, http, info['id'], + post_method='rsync_then_merge', + post_timeout=(info['count'] / 2000)) + # else send diffs over to the remote server + return self._usync_db(max(rinfo['point'], local_sync), + broker, http, rinfo['id'], info['id']) + + def _replicate_object(self, partition, object_file, node_id): + """ + Replicate the db, choosing method based on whether or not it + already exists on peers. + + :param partition: partition to be replicated to + :param object_file: DB file name to be replicated + :param node_id: node id of the node to be replicated to + """ + self.logger.debug('Replicating db %s' % object_file) + self.stats['attempted'] += 1 + try: + broker = self.brokerclass(object_file, pending_timeout=30) + broker.reclaim(time.time() - self.reclaim_age, + time.time() - (self.reclaim_age * 2)) + info = broker.get_replication_info() + except Exception, e: + if 'no such table' in str(e): + self.logger.error('Quarantining DB %s' % object_file) + quarantine_db(broker.db_file, broker.db_type) + else: + self.logger.exception('ERROR reading db %s' % object_file) + self.stats['failure'] += 1 + return + # The db is considered deleted if the delete_timestamp value is greater + # than the put_timestamp, and there are no objects. + delete_timestamp = 0 + try: + delete_timestamp = float(info['delete_timestamp']) + except ValueError: + pass + put_timestamp = 0 + try: + put_timestamp = float(info['put_timestamp']) + except ValueError: + pass + if delete_timestamp < (time.time() - self.reclaim_age) and \ + delete_timestamp > put_timestamp and \ + info['count'] in (None, '', 0, '0'): + with lock_parent_directory(object_file): + shutil.rmtree(os.path.dirname(object_file), True) + self.stats['remove'] += 1 + return + responses = [] + nodes = self.ring.get_part_nodes(int(partition)) + shouldbehere = bool([n for n in nodes if n['id'] == node_id]) + repl_nodes = [n for n in nodes if n['id'] != node_id] + more_nodes = self.ring.get_more_nodes(int(partition)) + for node in repl_nodes: + success = False + try: + success = self._repl_to_node(node, broker, partition, info) + except DriveNotMounted: + repl_nodes.append(more_nodes.next()) + self.logger.error('ERROR Remote drive not mounted %s' % node) + except: + self.logger.exception('ERROR syncing %s with node %s' % + (object_file, node)) + self.stats['success' if success else 'failure'] += 1 + responses.append(success) + if not shouldbehere and all(responses): + # If the db shouldn't be on this node and has been successfully + # synced to all of its peers, it can be removed. + with lock_parent_directory(object_file): + shutil.rmtree(os.path.dirname(object_file), True) + self.stats['remove'] += 1 + + def roundrobin_datadirs(self, datadirs): + """ + Generator to walk the data dirs in a round robin manner, evenly + hitting each device on the system. + + :param datadirs: a list of paths to walk + """ + def walk_datadir(datadir, node_id): + partitions = os.listdir(datadir) + random.shuffle(partitions) + for partition in partitions: + part_dir = os.path.join(datadir, partition) + for root, dirs, files in os.walk(part_dir, topdown=False): + for fname in (f for f in files if f.endswith('.db')): + object_file = os.path.join(root, fname) + yield (partition, object_file, node_id) + its = [walk_datadir(datadir, node_id) for datadir, node_id in datadirs] + while its: + for it in its: + try: + yield it.next() + except StopIteration: + its.remove(it) + + def replicate_once(self): + """Run a replication pass once.""" + self._zero_stats() + dirs = [] + ips = whataremyips() + if not ips: + self.logger.error('ERROR Failed to get my own IPs?') + return + for node in self.ring.devs: + if node and node['ip'] in ips and node['port'] == self.port: + if self.mount_check and not os.path.ismount( + os.path.join(self.root, node['device'])): + self.logger.warn( + 'Skipping %(device)s as it is not mounted' % node) + continue + unlink_older_than( + os.path.join(self.root, node['device'], 'tmp'), + time.time() - self.reclaim_age) + datadir = os.path.join(self.root, node['device'], self.datadir) + if os.path.isdir(datadir): + dirs.append((datadir, node['id'])) + self.logger.info('Beginning replication run') + for part, object_file, node_id in self.roundrobin_datadirs(dirs): + self.cpool.spawn_n( + self._replicate_object, part, object_file, node_id) + self.cpool.waitall() + self.logger.info('Replication run OVER') + self._report_stats() + + def replicate_forever(self): + """ + Replicate dbs under the given root in an infinite loop. + """ + while True: + try: + self.replicate_once() + except: + self.logger.exception('ERROR trying to replicate') + sleep(self.run_pause) + + +class ReplicatorRpc(object): + """Handle Replication RPC calls. TODO: redbo document please :)""" + + def __init__(self, root, datadir, broker_class, mount_check=True): + self.root = root + self.datadir = datadir + self.broker_class = broker_class + self.mount_check = mount_check + + def dispatch(self, post_args, args): + if not hasattr(args, 'pop'): + return HTTPBadRequest(body='Invalid object type') + op = args.pop(0) + drive, partition, hsh = post_args + if self.mount_check and \ + not os.path.ismount(os.path.join(self.root, drive)): + return Response(status='507 %s is not mounted' % drive) + db_file = os.path.join(self.root, drive, + storage_directory(self.datadir, partition, hsh), hsh + '.db') + if op == 'rsync_then_merge': + return self.rsync_then_merge(drive, db_file, args) + if op == 'complete_rsync': + return self.complete_rsync(drive, db_file, args) + else: + # someone might be about to rsync a db to us, + # make sure there's a tmp dir to receive it. + mkdirs(os.path.join(self.root, drive, 'tmp')) + if not os.path.exists(db_file): + return HTTPNotFound() + return getattr(self, op)(self.broker_class(db_file), args) + + def sync(self, broker, args): + (remote_sync, hash_, id_, created_at, put_timestamp, + delete_timestamp) = args + try: + info = broker.get_replication_info() + except Exception, e: + if 'no such table' in str(e): + # TODO find a real logger + print "Quarantining DB %s" % broker.db_file + quarantine_db(broker.db_file, broker.db_type) + return HTTPNotFound() + raise + if info['put_timestamp'] != put_timestamp or \ + info['created_at'] != created_at or \ + info['delete_timestamp'] != delete_timestamp: + broker.merge_timestamps( + created_at, put_timestamp, delete_timestamp) + info['point'] = broker.get_sync(id_) + if hash_ == info['hash'] and info['point'] < remote_sync: + broker.merge_syncs([{'remote_id': id_, + 'sync_point': remote_sync}]) + info['point'] = remote_sync + return Response(simplejson.dumps(info)) + + def merge_syncs(self, broker, args): + broker.merge_syncs(args[0]) + return HTTPAccepted() + + def merge_items(self, broker, args): + broker.merge_items(args[0], args[1]) + return HTTPAccepted() + + def complete_rsync(self, drive, db_file, args): + old_filename = os.path.join(self.root, drive, 'tmp', args[0]) + if os.path.exists(db_file): + return HTTPNotFound() + if not os.path.exists(old_filename): + return HTTPNotFound() + broker = self.broker_class(old_filename) + broker.newid(args[0]) + renamer(old_filename, db_file) + return HTTPNoContent() + + def rsync_then_merge(self, drive, db_file, args): + old_filename = os.path.join(self.root, drive, 'tmp', args[0]) + if not os.path.exists(db_file) or not os.path.exists(old_filename): + return HTTPNotFound() + new_broker = self.broker_class(old_filename) + existing_broker = self.broker_class(db_file) + point = -1 + objects = existing_broker.get_items_since(point, 1000) + while len(objects): + new_broker.merge_items(objects) + point = objects[-1]['ROWID'] + objects = existing_broker.get_items_since(point, 1000) + sleep() + new_broker.newid(args[0]) + renamer(old_filename, db_file) + return HTTPNoContent() diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py new file mode 100644 index 0000000000..1d7030c09d --- /dev/null +++ b/swift/common/direct_client.py @@ -0,0 +1,303 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Internal client library for making calls directly to the servers rather than +through the proxy. +""" + +import socket +from httplib import HTTPException +from time import time +from urllib import quote as _quote, unquote + +from eventlet import sleep, Timeout + +from swift.common.bufferedhttp import http_connect +from swift.common.client import ClientException, json_loads +from swift.common.utils import normalize_timestamp + + +def quote(value, safe='/'): + if isinstance(value, unicode): + value = value.encode('utf8') + return _quote(value, safe) + + +def direct_head_container(node, part, account, container, conn_timeout=5, + response_timeout=15): + """ + Request container information directly from the container server. + + :param node: node dictionary from the ring + :param part: partition the container is on + :param account: account name + :param container: container name + :param conn_timeout: timeout in seconds for establishing the connection + :param response_timeout: timeout in seconds for getting the response + :returns: tuple of (object count, bytes used) + """ + path = '/%s/%s' % (account, container) + with Timeout(conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], part, + 'HEAD', path) + with Timeout(response_timeout): + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException( + 'Container server %s:%s direct HEAD %s gave status %s' % + (node['ip'], node['port'], + repr('/%s/%s%s' % (node['device'], part, path)), + resp.status), + http_host=node['ip'], http_port=node['port'], + http_device=node['device'], http_status=resp.status, + http_reason=resp.reason) + return int(resp.getheader('x-container-object-count')), \ + int(resp.getheader('x-container-bytes-used')) + + +def direct_get_container(node, part, account, container, marker=None, + limit=None, prefix=None, delimiter=None, + conn_timeout=5, response_timeout=15): + """ + Get container listings directly from the container server. + + :param node: node dictionary from the ring + :param part: partition the container is on + :param account: account name + :param container: container name + :param marker: marker query + :param limit: query limit + :param prefix: prefix query + :param delimeter: delimeter for the query + :param conn_timeout: timeout in seconds for establishing the connection + :param response_timeout: timeout in seconds for getting the response + :returns: list of objects + """ + path = '/%s/%s' % (account, container) + qs = 'format=json' + if marker: + qs += '&marker=%s' % quote(marker) + if limit: + qs += '&limit=%d' % limit + if prefix: + qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) + with Timeout(conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], part, + 'GET', path, query_string='format=json') + with Timeout(response_timeout): + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException( + 'Container server %s:%s direct GET %s gave stats %s' % (node['ip'], + node['port'], repr('/%s/%s%s' % (node['device'], part, path)), + resp.status), + http_host=node['ip'], http_port=node['port'], + http_device=node['device'], http_status=resp.status, + http_reason=resp.reason) + if resp.status == 204: + resp.read() + return [] + return json_loads(resp.read()) + + +def direct_delete_container(node, part, account, container, conn_timeout=5, + response_timeout=15, headers={}): + path = '/%s/%s' % (account, container) + headers['X-Timestamp'] = normalize_timestamp(time()) + with Timeout(conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], part, + 'DELETE', path, headers) + with Timeout(response_timeout): + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + raise ClientException( + 'Container server %s:%s direct DELETE %s gave status %s' % + (node['ip'], node['port'], + repr('/%s/%s%s' % (node['device'], part, path)), + resp.status), + http_host=node['ip'], http_port=node['port'], + http_device=node['device'], http_status=resp.status, + http_reason=resp.reason) + return resp + + +def direct_head_object(node, part, account, container, obj, conn_timeout=5, + response_timeout=15): + """ + Request object information directly from the object server. + + :param node: node dictionary from the ring + :param part: partition the container is on + :param account: account name + :param container: container name + :param obj: object name + :param conn_timeout: timeout in seconds for establishing the connection + :param response_timeout: timeout in seconds for getting the response + :returns: tuple of (content-type, object size, last modified timestamp, + etag, metadata dictionary) + """ + path = '/%s/%s/%s' % (account, container, obj) + with Timeout(conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], part, + 'HEAD', path) + with Timeout(response_timeout): + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException( + 'Object server %s:%s direct HEAD %s gave status %s' % + (node['ip'], node['port'], + repr('/%s/%s%s' % (node['device'], part, path)), + resp.status), + http_host=node['ip'], http_port=node['port'], + http_device=node['device'], http_status=resp.status, + http_reason=resp.reason) + metadata = {} + for key, value in resp.getheaders(): + if key.lower().startswith('x-object-meta-'): + metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) + return resp.getheader('content-type'), \ + int(resp.getheader('content-length')), \ + resp.getheader('last-modified'), \ + resp.getheader('etag').strip('"'), \ + metadata + + +def direct_get_object(node, part, account, container, obj, conn_timeout=5, + response_timeout=15): + """ + Get object directly from the object server. + + :param node: node dictionary from the ring + :param part: partition the container is on + :param account: account name + :param container: container name + :param obj: object name + :param conn_timeout: timeout in seconds for establishing the connection + :param response_timeout: timeout in seconds for getting the response + :returns: object + """ + path = '/%s/%s/%s' % (account, container, obj) + with Timeout(conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], part, + 'GET', path) + with Timeout(response_timeout): + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + raise ClientException( + 'Object server %s:%s direct GET %s gave status %s' % + (node['ip'], node['port'], + repr('/%s/%s%s' % (node['device'], part, path)), + resp.status), + http_host=node['ip'], http_port=node['port'], + http_device=node['device'], http_status=resp.status, + http_reason=resp.reason) + metadata = {} + for key, value in resp.getheaders(): + if key.lower().startswith('x-object-meta-'): + metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) + return (resp.getheader('content-type'), + int(resp.getheader('content-length')), + resp.getheader('last-modified'), + resp.getheader('etag').strip('"'), + metadata, + resp.read()) + + +def direct_delete_object(node, part, account, container, obj, + conn_timeout=5, response_timeout=15, headers={}): + """ + Delete object directly from the object server. + + :param node: node dictionary from the ring + :param part: partition the container is on + :param account: account name + :param container: container name + :param obj: object name + :param conn_timeout: timeout in seconds for establishing the connection + :param response_timeout: timeout in seconds for getting the response + :returns: response from server + """ + path = '/%s/%s/%s' % (account, container, obj) + headers['X-Timestamp'] = normalize_timestamp(time()) + with Timeout(conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], part, + 'DELETE', path, headers) + with Timeout(response_timeout): + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + raise ClientException( + 'Object server %s:%s direct DELETE %s gave status %s' % + (node['ip'], node['port'], + repr('/%s/%s%s' % (node['device'], part, path)), + resp.status), + http_host=node['ip'], http_port=node['port'], + http_device=node['device'], http_status=resp.status, + http_reason=resp.reason) + return resp + + +def retry(func, *args, **kwargs): + """ + Helper function to retry a given function a number of times. + + :param func: callable to be called + :param retries: number of retries + :param error_log: logger for errors + :param args: arguments to send to func + :param kwargs: keyward arguments to send to func (if retries or + error_log are sent, they will be deleted from kwargs + before sending on to func) + :returns: restult of func + """ + retries = 5 + if 'retries' in kwargs: + retries = kwargs['retries'] + del kwargs['retries'] + error_log = None + if 'error_log' in kwargs: + error_log = kwargs['error_log'] + del kwargs['error_log'] + attempts = 0 + backoff = 1 + while attempts <= retries: + attempts += 1 + try: + return attempts, func(*args, **kwargs) + except (socket.error, HTTPException, Timeout), err: + if error_log: + error_log(err) + if attempts > retries: + raise + except ClientException, err: + if error_log: + error_log(err) + if attempts > retries or err.http_status < 500 or \ + err.http_status == 507 or err.http_status > 599: + raise + sleep(backoff) + backoff *= 2 + # Shouldn't actually get down here, but just in case. + if args and 'ip' in args[0]: + raise ClientException('Raise too many retries', + http_host=args[0]['ip'], http_port=args[0]['port'], + http_device=args[0]['device']) + else: + raise ClientException('Raise too many retries') diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py new file mode 100644 index 0000000000..bf1b07cae2 --- /dev/null +++ b/swift/common/exceptions.py @@ -0,0 +1,35 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from eventlet import TimeoutError + + +class MessageTimeout(TimeoutError): + + def __init__(self, seconds=None, msg=None): + TimeoutError.__init__(self, seconds=seconds) + self.msg = msg + + def __str__(self): + return '%s: %s' % (TimeoutError.__str__(self), self.msg) + + +class AuditException(Exception): pass +class AuthException(Exception): pass +class ChunkReadTimeout(TimeoutError): pass +class ChunkWriteTimeout(TimeoutError): pass +class ConnectionTimeout(TimeoutError): pass +class DriveNotMounted(Exception): pass +class LockTimeout(MessageTimeout): pass diff --git a/swift/common/healthcheck.py b/swift/common/healthcheck.py new file mode 100644 index 0000000000..a483eaffd9 --- /dev/null +++ b/swift/common/healthcheck.py @@ -0,0 +1,28 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from webob import Response + +class HealthCheckController(object): + """Basic controller used for monitoring.""" + + def __init__(self, *args, **kwargs): + pass + + @classmethod + def GET(self, req): + return Response(request=req, body="OK", content_type="text/plain") + +healthcheck = HealthCheckController.GET diff --git a/swift/common/memcached.py b/swift/common/memcached.py new file mode 100644 index 0000000000..e232e404f8 --- /dev/null +++ b/swift/common/memcached.py @@ -0,0 +1,272 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Lucid comes with memcached: v1.4.2. Protocol documentation for that +version is at: + +http://github.com/memcached/memcached/blob/1.4.2/doc/protocol.txt +""" + +import cPickle as pickle +import logging +import socket +import time +from bisect import bisect +from hashlib import md5 + + +CONN_TIMEOUT = 0.3 +IO_TIMEOUT = 2.0 +PICKLE_FLAG = 1 +NODE_WEIGHT = 50 +PICKLE_PROTOCOL = 2 +TRY_COUNT = 3 + +# if ERROR_LIMIT_COUNT errors occur in ERROR_LIMIT_TIME seconds, the server +# will be considered failed for ERROR_LIMIT_DURATION seconds. +ERROR_LIMIT_COUNT = 10 +ERROR_LIMIT_TIME = 60 +ERROR_LIMIT_DURATION = 300 + + +def md5hash(key): + return md5(key).hexdigest() + + +class MemcacheRing(object): + """ + Simple, consistent-hashed memcache client. + """ + + def __init__(self, servers, connect_timeout=CONN_TIMEOUT, + io_timeout=IO_TIMEOUT, tries=TRY_COUNT): + self._ring = {} + self._errors = dict(((serv, []) for serv in servers)) + self._error_limited = dict(((serv, 0) for serv in servers)) + for server in sorted(servers): + for i in xrange(NODE_WEIGHT): + self._ring[md5hash('%s-%s' % (server, i))] = server + self._tries = tries if tries <= len(servers) else len(servers) + self._sorted = sorted(self._ring.keys()) + self._client_cache = dict(((server, []) for server in servers)) + self._connect_timeout = connect_timeout + self._io_timeout = io_timeout + + def _exception_occurred(self, server, e, action='talking'): + if isinstance(e, socket.timeout): + logging.error("Timeout %s to memcached: %s" % (action, server)) + else: + logging.exception("Error %s to memcached: %s" % (action, server)) + now = time.time() + self._errors[server].append(time.time()) + if len(self._errors[server]) > ERROR_LIMIT_COUNT: + self._errors[server] = [err for err in self._errors[server] + if err > now - ERROR_LIMIT_TIME] + if len(self._errors[server]) > ERROR_LIMIT_COUNT: + self._error_limited[server] = now + ERROR_LIMIT_DURATION + logging.error('Error limiting server %s' % server) + + def _get_conns(self, key): + """ + Retrieves a server conn from the pool, or connects a new one. + Chooses the server based on a consistent hash of "key". + """ + pos = bisect(self._sorted, key) + served = [] + while len(served) < self._tries: + pos = (pos + 1) % len(self._sorted) + server = self._ring[self._sorted[pos]] + if server in served: + continue + served.append(server) + if self._error_limited[server] > time.time(): + continue + try: + fp, sock = self._client_cache[server].pop() + yield server, fp, sock + except IndexError: + try: + host, port = server.split(':') + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.settimeout(self._connect_timeout) + sock.connect((host, int(port))) + sock.settimeout(self._io_timeout) + yield server, sock.makefile(), sock + except Exception, e: + self._exception_occurred(server, e, 'connecting') + + def _return_conn(self, server, fp, sock): + """ Returns a server connection to the pool """ + self._client_cache[server].append((fp, sock)) + + def set(self, key, value, serialize=True, timeout=0): + """ + Set a key/value pair in memcache + + :param key: key + :param value: value + :param serialize: if True, value is pickled before sending to memcache + :param timeout: ttl in memcache + """ + key = md5hash(key) + if timeout > 0: + timeout += time.time() + flags = 0 + if serialize: + value = pickle.dumps(value, PICKLE_PROTOCOL) + flags |= PICKLE_FLAG + for (server, fp, sock) in self._get_conns(key): + try: + sock.sendall('set %s %d %d %s noreply\r\n%s\r\n' % \ + (key, flags, timeout, len(value), value)) + self._return_conn(server, fp, sock) + return + except Exception, e: + self._exception_occurred(server, e) + + def get(self, key): + """ + Gets the object specified by key. It will also unpickle the object + before returning if it is pickled in memcache. + + :param key: key + :returns: value of the key in memcache + """ + key = md5hash(key) + value = None + for (server, fp, sock) in self._get_conns(key): + try: + sock.sendall('get %s\r\n' % key) + line = fp.readline().strip().split() + while line[0].upper() != 'END': + if line[0].upper() == 'VALUE' and line[1] == key: + size = int(line[3]) + value = fp.read(size) + if int(line[2]) & PICKLE_FLAG: + value = pickle.loads(value) + fp.readline() + line = fp.readline().strip().split() + self._return_conn(server, fp, sock) + return value + except Exception, e: + self._exception_occurred(server, e) + + def incr(self, key, delta=1, timeout=0): + """ + Increments a key which has a numeric value by delta. + If the key can't be found, it's added as delta. + + :param key: key + :param delta: amount to add to the value of key (or set as the value + if the key is not found) + :param timeout: ttl in memcache + """ + key = md5hash(key) + for (server, fp, sock) in self._get_conns(key): + try: + sock.sendall('incr %s %s\r\n' % (key, delta)) + line = fp.readline().strip().split() + if line[0].upper() == 'NOT_FOUND': + line[0] = str(delta) + sock.sendall('add %s %d %d %s noreply\r\n%s\r\n' % \ + (key, 0, timeout, len(line[0]), line[0])) + ret = int(line[0].strip()) + self._return_conn(server, fp, sock) + return ret + except Exception, e: + self._exception_occurred(server, e) + + def delete(self, key): + """ + Deletes a key/value pair from memcache. + + :param key: key to be deleted + """ + key = md5hash(key) + for (server, fp, sock) in self._get_conns(key): + try: + sock.sendall('delete %s noreply\r\n' % key) + self._return_conn(server, fp, sock) + return + except Exception, e: + self._exception_occurred(server, e) + + def set_multi(self, mapping, server_key, serialize=True, timeout=0): + """ + Sets multiple key/value pairs in memcache. + + :param mapping: dictonary of keys and values to be set in memcache + :param servery_key: key to use in determining which server in the ring + is used + :param serialize: if True, value is pickled before sending to memcache + :param timeout: ttl for memcache + """ + server_key = md5hash(server_key) + if timeout > 0: + timeout += time.time() + msg = '' + for key, value in mapping.iteritems(): + key = md5hash(key) + flags = 0 + if serialize: + value = pickle.dumps(value, PICKLE_PROTOCOL) + flags |= PICKLE_FLAG + msg += ('set %s %d %d %s noreply\r\n%s\r\n' % + (key, flags, timeout, len(value), value)) + for (server, fp, sock) in self._get_conns(server_key): + try: + sock.sendall(msg) + self._return_conn(server, fp, sock) + return + except Exception, e: + self._exception_occurred(server, e) + + def get_multi(self, keys, server_key): + """ + Gets multiple values from memcache for the given keys. + + :param keys: keys for values to be retrieved from memcache + :param servery_key: key to use in determining which server in the ring + is used + :returns: list of values + """ + server_key = md5hash(server_key) + keys = [md5hash(key) for key in keys] + for (server, fp, sock) in self._get_conns(server_key): + try: + sock.sendall('get %s\r\n' % ' '.join(keys)) + line = fp.readline().strip().split() + responses = {} + while line[0].upper() != 'END': + if line[0].upper() == 'VALUE': + size = int(line[3]) + value = fp.read(size) + if int(line[2]) & PICKLE_FLAG: + value = pickle.loads(value) + responses[line[1]] = value + fp.readline() + line = fp.readline().strip().split() + values = [] + for key in keys: + if key in responses: + values.append(responses[key]) + else: + values.append(None) + self._return_conn(server, fp, sock) + return values + except Exception, e: + self._exception_occurred(server, e) diff --git a/swift/common/ring/__init__.py b/swift/common/ring/__init__.py new file mode 100644 index 0000000000..6040b860e3 --- /dev/null +++ b/swift/common/ring/__init__.py @@ -0,0 +1,2 @@ +from ring import RingData, Ring +from builder import RingBuilder diff --git a/swift/common/ring/builder.py b/swift/common/ring/builder.py new file mode 100644 index 0000000000..5826c112df --- /dev/null +++ b/swift/common/ring/builder.py @@ -0,0 +1,460 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from array import array +from bisect import bisect +from random import randint +from time import time + +from swift.common.ring import RingData + + +class RingBuilder(object): + """ + Used to build swift.common.RingData instances to be written to disk and + used with swift.common.ring.Ring instances. See bin/ring-builder.py for + example usage. + + The instance variable devs_changed indicates if the device information has + changed since the last balancing. This can be used by tools to know whether + a rebalance request is an isolated request or due to added, changed, or + removed devices. + + :param part_power: number of partitions = 2**part_power + :param replicas: number of replicas for each partition + :param min_part_hours: minimum number of hours between partition changes + """ + + def __init__(self, part_power, replicas, min_part_hours): + self.part_power = part_power + self.replicas = replicas + self.min_part_hours = min_part_hours + self.parts = 2 ** self.part_power + self.devs = [] + self.devs_changed = False + self.version = 0 + + # _replica2part2dev maps from replica number to partition number to + # device id. So, for a three replica, 2**23 ring, it's an array of + # three 2**23 arrays of device ids (unsigned shorts). This can work a + # bit faster than the 2**23 array of triplet arrays of device ids in + # many circumstances. Making one big 2**23 * 3 array didn't seem to + # have any speed change; though you're welcome to try it again (it was + # a while ago, code-wise, when I last tried it). + self._replica2part2dev = None + + # _last_part_moves is a 2**23 array of unsigned bytes representing the + # number of hours since a given partition was last moved. This is used + # to guarantee we don't move a partition twice within a given number of + # hours (24 is my usual test). Removing a device or setting it's weight + # to 0 overrides this behavior as it's assumed those actions are done + # because of device failure. + # _last_part_moves_epoch indicates the time the offsets in + # _last_part_moves is based on. + self._last_part_moves_epoch = None + self._last_part_moves = None + + self._last_part_gather_start = 0 + self._remove_devs = [] + self._ring = None + + def change_min_part_hours(self, min_part_hours): + """ + Changes the value used to decide if a given partition can be moved + again. This restriction is to give the overall system enough time to + settle a partition to its new location before moving it to yet another + location. While no data would be lost if a partition is moved several + times quickly, it could make that data unreachable for a short period + of time. + + This should be set to at least the average full partition replication + time. Starting it at 24 hours and then lowering it to what the + replicator reports as the longest partition cycle is best. + + :param min_part_hours: new value for min_part_hours + """ + self.min_part_hours = min_part_hours + + def get_ring(self): + """ + Get the ring, or more specifically, the swift.common.ring.RingData. + This ring data is the minimum required for use of the ring. The ring + builder itself keeps additional data such as when partitions were last + moved. + """ + if not self._ring: + devs = [None] * len(self.devs) + for dev in self.devs: + if dev is None: + continue + devs[dev['id']] = dict((k, v) for k, v in dev.items() + if k not in ('parts', 'parts_wanted')) + self._ring = \ + RingData([array('H', p2d) for p2d in self._replica2part2dev], + devs, 32 - self.part_power) + return self._ring + + def add_dev(self, dev): + """ + Add a device to the ring. This device dict should have a minimum of the + following keys: + + ====== =============================================================== + id unique integer identifier amongst devices + weight a float of the relative weight of this device as compared to + others; this indicates how many partitions the builder will try + to assign to this device + zone integer indicating which zone the device is in; a given + partition will not be assigned to multiple devices within the + same zone ip the ip address of the device + port the tcp port of the device + device the device's name on disk (sdb1, for example) + meta general use 'extra' field; for example: the online date, the + hardware description + ====== =============================================================== + + .. note:: + This will not rebalance the ring immediately as you may want to + make multiple changes for a single rebalance. + + :param dev: device dict + """ + if dev['id'] < len(self.devs) and self.devs[dev['id']] is not None: + raise Exception('Duplicate device id: %d' % dev['id']) + while dev['id'] >= len(self.devs): + self.devs.append(None) + dev['weight'] = float(dev['weight']) + dev['parts'] = 0 + self.devs[dev['id']] = dev + self._set_parts_wanted() + self.devs_changed = True + self.version += 1 + + def set_dev_weight(self, dev_id, weight): + """ + Set the weight of a device. This should be called rather than just + altering the weight key in the device dict directly, as the builder + will need to rebuild some internal state to reflect the change. + + .. note:: + This will not rebalance the ring immediately as you may want to + make multiple changes for a single rebalance. + + :param dev_id: device id + :param weight: new weight for device + """ + self.devs[dev_id]['weight'] = weight + self._set_parts_wanted() + self.devs_changed = True + self.version += 1 + + def remove_dev(self, dev_id): + """ + Remove a device from the ring. + + .. note:: + This will not rebalance the ring immediately as you may want to + make multiple changes for a single rebalance. + + :param dev_id: device id + """ + dev = self.devs[dev_id] + dev['weight'] = 0 + self._remove_devs.append(dev) + self._set_parts_wanted() + self.devs_changed = True + self.version += 1 + + def rebalance(self): + """ + Rebalance the ring. + + This is the main work function of the builder, as it will assign and + reassign partitions to devices in the ring based on weights, distinct + zones, recent reassignments, etc. + + The process doesn't always perfectly assign partitions (that'd take a + lot more analysis and therefore a lot more time -- I had code that did + that before). Because of this, it keeps rebalancing until the device + skew (number of partitions a device wants compared to what it has) gets + below 1% or doesn't change by more than 1% (only happens with ring that + can't be balanced no matter what -- like with 3 zones of differing + weights with replicas set to 3). + """ + self._ring = None + if self._last_part_moves_epoch is None: + self._initial_balance() + self.devs_changed = False + return self.parts, self.get_balance() + retval = 0 + self._update_last_part_moves() + last_balance = 0 + while True: + reassign_parts = self._gather_reassign_parts() + self._reassign_parts(reassign_parts) + retval += len(reassign_parts) + while self._remove_devs: + self.devs[self._remove_devs.pop()['id']] = None + balance = self.get_balance() + if balance < 1 or abs(last_balance - balance) < 1 or \ + retval == self.parts: + break + last_balance = balance + self.devs_changed = False + self.version += 1 + return retval, balance + + def validate(self, stats=False): + """ + Validate the ring. + + This is a safety function to try to catch any bugs in the building + process. It ensures partitions have been assigned to distinct zones, + aren't doubly assigned, etc. It can also optionally check the even + distribution of partitions across devices. + + :param stats: if True, check distribution of partitions across devices + :returns: if stats is True, a tuple of (device usage, worst stat), else + (None, None) + :raises Exception: problem was found with the ring. + """ + if sum(d['parts'] for d in self.devs if d is not None) != \ + self.parts * self.replicas: + raise Exception( + 'All partitions are not double accounted for: %d != %d' % + (sum(d['parts'] for d in self.devs if d is not None), + self.parts * self.replicas)) + if stats: + dev_usage = array('I', (0 for _ in xrange(len(self.devs)))) + for part in xrange(self.parts): + zones = {} + for replica in xrange(self.replicas): + dev_id = self._replica2part2dev[replica][part] + if stats: + dev_usage[dev_id] += 1 + zone = self.devs[dev_id]['zone'] + if zone in zones: + raise Exception( + 'Partition %d not in %d distinct zones. ' \ + 'Zones were: %s' % + (part, self.replicas, + [self.devs[self._replica2part2dev[r][part]]['zone'] + for r in xrange(self.replicas)])) + zones[zone] = True + if stats: + weighted_parts = self.parts * self.replicas / \ + sum(d['weight'] for d in self.devs if d is not None) + worst = 0 + for dev in self.devs: + if dev is None: + continue + if not dev['weight']: + if dev_usage[dev['id']]: + worst = 999.99 + break + continue + skew = abs(100.0 * dev_usage[dev['id']] / + (dev['weight'] * weighted_parts) - 100.0) + if skew > worst: + worst = skew + return dev_usage, worst + return None, None + + def get_balance(self): + """ + Get the balance of the ring. The balance value is the highest + percentage off the desired amount of partitions a given device wants. + For instance, if the "worst" device wants (based on its relative weight + and its zone's relative weight) 123 partitions and it has 124 + partitions, the balance value would be 0.83 (1 extra / 123 wanted * 100 + for percentage). + + :returns: balance of the ring + """ + weighted_parts = self.parts * self.replicas / \ + sum(d['weight'] for d in self.devs if d is not None) + balance = 0 + for dev in self.devs: + if dev is None: + continue + if not dev['weight']: + if dev['parts']: + balance = 999.99 + break + continue + dev_balance = abs(100.0 * dev['parts'] / + (dev['weight'] * weighted_parts) - 100.0) + if dev_balance > balance: + balance = dev_balance + return balance + + def pretend_min_part_hours_passed(self): + """ + Override min_part_hours by marking all partitions as having been moved + 255 hours ago. This can be used to force a full rebalance on the next + call to rebalance. + """ + for part in xrange(self.parts): + self._last_part_moves[part] = 0xff + + def _set_parts_wanted(self): + """ + Sets the parts_wanted key for each of the devices to the number of + partitions the device wants based on its relative weight. This key is + used to sort the devices according to "most wanted" during rebalancing + to best distribute partitions. + """ + weighted_parts = self.parts * self.replicas / \ + sum(d['weight'] for d in self.devs if d is not None) + for dev in self.devs: + if dev is not None: + if not dev['weight']: + dev['parts_wanted'] = self.parts * -2 + else: + dev['parts_wanted'] = \ + int(weighted_parts * dev['weight']) - dev['parts'] + + def _initial_balance(self): + """ + Initial partition assignment is treated separately from rebalancing an + existing ring. Initial assignment is performed by ordering all the + devices by how many partitions they still want (and kept in order + during the process). The partitions are then iterated through, + assigning them to the next "most wanted" device, with distinct zone + restrictions. + """ + for dev in self.devs: + dev['sort_key'] = \ + '%08x.%04x' % (dev['parts_wanted'], randint(0, 0xffff)) + available_devs = sorted((d for d in self.devs if d is not None), + key=lambda x: x['sort_key']) + self._replica2part2dev = [array('H') for _ in xrange(self.replicas)] + for _ in xrange(self.parts): + other_zones = array('H') + for replica in xrange(self.replicas): + index = len(available_devs) - 1 + while available_devs[index]['zone'] in other_zones: + index -= 1 + dev = available_devs.pop(index) + self._replica2part2dev[replica].append(dev['id']) + dev['parts_wanted'] -= 1 + dev['parts'] += 1 + dev['sort_key'] = \ + '%08x.%04x' % (dev['parts_wanted'], randint(0, 0xffff)) + index = 0 + end = len(available_devs) + while index < end: + mid = (index + end) // 2 + if dev['sort_key'] < available_devs[mid]['sort_key']: + end = mid + else: + index = mid + 1 + available_devs.insert(index, dev) + other_zones.append(dev['zone']) + self._last_part_moves = array('B', (0 for _ in xrange(self.parts))) + self._last_part_moves_epoch = int(time()) + for dev in self.devs: + del dev['sort_key'] + + def _update_last_part_moves(self): + """ + Updates how many hours ago each partition was moved based on the + current time. The builder won't move a partition that has been moved + more recently than min_part_hours. + """ + elapsed_hours = int(time() - self._last_part_moves_epoch) / 3600 + for part in xrange(self.parts): + self._last_part_moves[part] = \ + min(self._last_part_moves[part] + elapsed_hours, 0xff) + self._last_part_moves_epoch = int(time()) + + def _gather_reassign_parts(self): + """ + Returns an array('I') of partitions to be reassigned by gathering them + from removed devices and overweight devices. + """ + reassign_parts = array('I') + if self._remove_devs: + dev_ids = [d['id'] for d in self._remove_devs if d['parts']] + if dev_ids: + for replica in xrange(self.replicas): + part2dev = self._replica2part2dev[replica] + for part in xrange(self.parts): + if part2dev[part] in dev_ids: + part2dev[part] = 0xffff + self._last_part_moves[part] = 0 + reassign_parts.append(part) + start = self._last_part_gather_start / 4 + randint(0, self.parts / 2) + self._last_part_gather_start = start + for replica in xrange(self.replicas): + part2dev = self._replica2part2dev[replica] + for half in (xrange(start, self.parts), xrange(0, start)): + for part in half: + if self._last_part_moves[part] < self.min_part_hours: + continue + dev = self.devs[part2dev[part]] + if dev['parts_wanted'] < 0: + part2dev[part] = 0xffff + self._last_part_moves[part] = 0 + dev['parts_wanted'] += 1 + dev['parts'] -= 1 + reassign_parts.append(part) + return reassign_parts + + def _reassign_parts(self, reassign_parts): + """ + For an existing ring data set, partitions are reassigned similarly to + the initial assignment. The devices are ordered by how many partitions + they still want and kept in that order throughout the process. The + gathered partitions are iterated through, assigning them to devices + according to the "most wanted" and distinct zone restrictions. + """ + for dev in self.devs: + if dev is not None: + dev['sort_key'] = '%08x.%04x' % (self.parts + + dev['parts_wanted'], randint(0, 0xffff)) + available_devs = \ + sorted((d for d in self.devs if d is not None and d['weight']), + key=lambda x: x['sort_key']) + for part in reassign_parts: + other_zones = array('H') + replace = None + for replica in xrange(self.replicas): + if self._replica2part2dev[replica][part] == 0xffff: + replace = replica + else: + other_zones.append(self.devs[ + self._replica2part2dev[replica][part]]['zone']) + index = len(available_devs) - 1 + while available_devs[index]['zone'] in other_zones: + index -= 1 + dev = available_devs.pop(index) + self._replica2part2dev[replace][part] = dev['id'] + dev['parts_wanted'] -= 1 + dev['parts'] += 1 + dev['sort_key'] = '%08x.%04x' % (self.parts + dev['parts_wanted'], + randint(0, 0xffff)) + index = 0 + end = len(available_devs) + while index < end: + mid = (index + end) // 2 + if dev['sort_key'] < available_devs[mid]['sort_key']: + end = mid + else: + index = mid + 1 + available_devs.insert(index, dev) + for dev in self.devs: + if dev is not None: + del dev['sort_key'] diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py new file mode 100644 index 0000000000..0b5130a6f8 --- /dev/null +++ b/swift/common/ring/ring.py @@ -0,0 +1,141 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cPickle as pickle +from gzip import GzipFile +from hashlib import md5 +from os.path import getmtime +from struct import unpack_from +from time import time +from swift.common.utils import hash_path + + +class RingData(object): + """Partitioned consistent hashing ring data (used for serialization).""" + def __init__(self, replica2part2dev_id, devs, part_shift): + self.devs = devs + self._replica2part2dev_id = replica2part2dev_id + self._part_shift = part_shift + + +class Ring(object): + """ + Partitioned consistent hashing ring. + + :param pickle_gz_path: path to ring file + :param reload_time: time interval in seconds to check for a ring change + """ + def __init__(self, pickle_gz_path, reload_time=15): + self.pickle_gz_path = pickle_gz_path + self.reload_time = reload_time + self._reload(force=True) + + def _reload(self, force=False): + self._rtime = time() + self.reload_time + if force or self.has_changed(): + ring_data = pickle.load(GzipFile(self.pickle_gz_path, 'rb')) + self._mtime = getmtime(self.pickle_gz_path) + self.devs = ring_data.devs + self.zone2devs = {} + for dev in self.devs: + if not dev: + continue + if dev['zone'] in self.zone2devs: + self.zone2devs[dev['zone']].append(dev) + else: + self.zone2devs[dev['zone']] = [dev] + self._replica2part2dev_id = ring_data._replica2part2dev_id + self._part_shift = ring_data._part_shift + + @property + def replica_count(self): + """Number of replicas used in the ring.""" + return len(self._replica2part2dev_id) + + @property + def partition_count(self): + """Number of partitions in the ring.""" + return len(self._replica2part2dev_id[0]) + + def has_changed(self): + """ + Check to see if the ring on disk is different than the current one in + memory. + + :returns: True if the ring on disk has changed, False otherwise + """ + return getmtime(self.pickle_gz_path) != self._mtime + + def get_part_nodes(self, part): + """ + Get the nodes that are responsible for the partition. + + :param part: partition to get nodes for + :returns: list of node dicts + + See :func:`get_nodes` for a description of the node dicts. + """ + if time() > self._rtime: + self._reload() + return [self.devs[r[part]] for r in self._replica2part2dev_id] + + def get_nodes(self, account, container=None, obj=None): + """ + Get the partition and nodes for an account/container/object. + + :param account: account name + :param container: container name + :param obj: object name + :returns: a tuple of (partition, list of node dicts) + + Each node dict will have at least the following keys: + + ====== =============================================================== + id unique integer identifier amongst devices + weight a float of the relative weight of this device as compared to + others; this indicates how many partitions the builder will try + to assign to this device + zone integer indicating which zone the device is in; a given + partition will not be assigned to multiple devices within the + same zone ip the ip address of the device + port the tcp port of the device + device the device's name on disk (sdb1, for example) + meta general use 'extra' field; for example: the online date, the + hardware description + ====== =============================================================== + """ + key = hash_path(account, container, obj, raw_digest=True) + if time() > self._rtime: + self._reload() + part = unpack_from('>I', key)[0] >> self._part_shift + return part, [self.devs[r[part]] for r in self._replica2part2dev_id] + + def get_more_nodes(self, part): + """ + Generator to get extra nodes for a partition for hinted handoff. + + :param part: partition to get handoff nodes for + :returns: generator of node dicts + + See :func:`get_nodes` for a description of the node dicts. + """ + if time() > self._rtime: + self._reload() + zones = sorted(self.zone2devs.keys()) + for part2dev_id in self._replica2part2dev_id: + zones.remove(self.devs[part2dev_id[part]]['zone']) + while zones: + zone = zones.pop(part % len(zones)) + yield self.zone2devs[zone][part % len(self.zone2devs[zone])] diff --git a/swift/common/utils.py b/swift/common/utils.py new file mode 100644 index 0000000000..c2c091ce76 --- /dev/null +++ b/swift/common/utils.py @@ -0,0 +1,490 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Miscellaneous utility functions for use with Swift.""" + +import errno +import fcntl +import os +import pwd +import signal +import sys +import time +import mimetools +from hashlib import md5 +from random import shuffle +from urllib import quote +from contextlib import contextmanager +import ctypes +import ctypes.util +import fcntl +import struct + +import eventlet +from eventlet import greenio, GreenPool, sleep, Timeout, listen +from eventlet.green import socket, subprocess, ssl, thread, threading + +from swift.common.exceptions import LockTimeout, MessageTimeout + +# logging doesn't import patched as cleanly as one would like +from logging.handlers import SysLogHandler +import logging +logging.thread = eventlet.green.thread +logging.threading = eventlet.green.threading +logging._lock = logging.threading.RLock() + + +libc = ctypes.CDLL(ctypes.util.find_library('c')) +sys_fallocate = libc.fallocate +posix_fadvise = libc.posix_fadvise + + +# Used by hash_path to offer a bit more security when generating hashes for +# paths. It simply appends this value to all paths; guessing the hash a path +# will end up with would also require knowing this suffix. +HASH_PATH_SUFFIX = os.environ.get('SWIFT_HASH_PATH_SUFFIX', 'endcap') + + +def get_param(req, name, default=None): + """ + Get parameters from an HTTP request ensuring proper handling UTF-8 + encoding. + + :param req: Webob request object + :param name: parameter name + :param default: result to return if the parameter is not found + :returns: HTTP request parameter value + """ + value = req.str_params.get(name, default) + if value: + value.decode('utf8') # Ensure UTF8ness + return value + + +def fallocate(fd, size): + """ + Pre-allocate disk space for a file file. + + :param fd: file descriptor + :param size: size to allocate (in bytes) + """ + if size > 0: + # 1 means "FALLOC_FL_KEEP_SIZE", which means it pre-allocates invisibly + ret = sys_fallocate(fd, 1, 0, ctypes.c_uint64(size)) + # XXX: in (not very thorough) testing, errno always seems to be 0? + err = ctypes.get_errno() + if ret and err not in (0, errno.ENOSYS): + raise OSError(err, 'Unable to fallocate(%s)' % size) + + +def drop_buffer_cache(fd, offset, length): + """ + Drop 'buffer' cache for the given range of the given file. + + :param fd: file descriptor + :param offset: start offset + :param length: length + """ + # 4 means "POSIX_FADV_DONTNEED" + ret = posix_fadvise(fd, ctypes.c_uint64(offset), ctypes.c_uint64(length), 4) + if ret != 0: + print "posix_fadvise(%s, %s, %s, 4) -> %s" % (fd, offset, length, ret) + + +def normalize_timestamp(timestamp): + """ + Format a timestamp (string or numeric) into a standardized + xxxxxxxxxx.xxxxx format. + + :param timestamp: unix timestamp + :returns: normalized timestamp as a string + """ + return "%016.05f" % (float(timestamp)) + + +def mkdirs(path): + """ + Ensures the path is a directory or makes it if not. Errors if the path + exists but is a file or on permissions failure. + + :param path: path to create + """ + if not os.path.isdir(path): + try: + os.makedirs(path) + except OSError, err: + if err.errno != errno.EEXIST or not os.path.isdir(path): + raise + + +def renamer(old, new): # pragma: no cover + """ + Attempt to fix^H^H^Hhide race conditions like empty object directories + being removed by backend processes during uploads, by retrying. + + :param old: old path to be renamed + :param new: new path to be renamed to + """ + try: + mkdirs(os.path.dirname(new)) + os.rename(old, new) + except OSError: + mkdirs(os.path.dirname(new)) + os.rename(old, new) + + +def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): + """ + Validate and split the given HTTP request path. + + **Examples**:: + + ['a'] = split_path('/a') + ['a', None] = split_path('/a', 1, 2) + ['a', 'c'] = split_path('/a/c', 1, 2) + ['a', 'c', 'o/r'] = split_path('/a/c/o/r', 1, 3, True) + + :param path: HTTP Request path to be split + :param minsegs: Minimum number of segments to be extracted + :param maxsegs: Maximum number of segments to be extracted + :param rest_with_last: If True, trailing data will be returned as part + of last segment. If False, and there is + trailing data, raises ValueError. + :returns: list of segments with a length of maxsegs (non-existant + segments will return as None) + """ + if not maxsegs: + maxsegs = minsegs + if minsegs > maxsegs: + raise ValueError('minsegs > maxsegs: %d > %d' % (minsegs, maxsegs)) + if rest_with_last: + segs = path.split('/', maxsegs) + minsegs += 1 + maxsegs += 1 + count = len(segs) + if segs[0] or count < minsegs or count > maxsegs or \ + '' in segs[1:minsegs]: + raise ValueError('Invalid path: %s' % quote(path)) + else: + minsegs += 1 + maxsegs += 1 + segs = path.split('/', maxsegs) + count = len(segs) + if segs[0] or count < minsegs or count > maxsegs + 1 or \ + '' in segs[1:minsegs] or (count == maxsegs + 1 and segs[maxsegs]): + raise ValueError('Invalid path: %s' % quote(path)) + segs = segs[1:maxsegs] + segs.extend([None] * (maxsegs - 1 - len(segs))) + return segs + + +class NullLogger(): + """A no-op logger for eventlet wsgi.""" + + def write(self, *args): + #"Logs" the args to nowhere + pass + + +class LoggerFileObject(object): + + def __init__(self, logger): + self.logger = logger + + def write(self, value): + value = value.strip() + if value: + if 'Connection reset by peer' in value: + self.logger.error('STDOUT: Connection reset by peer') + else: + self.logger.error('STDOUT: %s' % value) + + def writelines(self, values): + self.logger.error('STDOUT: %s' % '#012'.join(values)) + + def close(self): + pass + + def flush(self): + pass + + def __iter__(self): + return self + + def next(self): + raise IOError(errno.EBADF, 'Bad file descriptor') + + def read(self, size=-1): + raise IOError(errno.EBADF, 'Bad file descriptor') + + def readline(self, size=-1): + raise IOError(errno.EBADF, 'Bad file descriptor') + + def tell(self): + return 0 + + def xreadlines(self): + return self + + +def drop_privileges(user): + """ + Sets the userid of the current process + + :param user: User id to change privileges to + """ + user = pwd.getpwnam(user) + os.setgid(user[3]) + os.setuid(user[2]) + + +class NamedLogger(object): + """Cheesy version of the LoggerAdapter available in Python 3""" + + def __init__(self, logger, server): + self.logger = logger + self.server = server + for proxied_method in ('debug', 'info', 'log', 'warn', 'warning', + 'error', 'critical'): + setattr(self, proxied_method, + self._proxy(getattr(logger, proxied_method))) + + def _proxy(self, logger_meth): + def _inner_proxy(msg, *args, **kwargs): + msg = '%s %s' % (self.server, msg) + logger_meth(msg, *args, **kwargs) + return _inner_proxy + + def getEffectiveLevel(self): + return self.logger.getEffectiveLevel() + + def exception(self, msg, *args): + _, exc, _ = sys.exc_info() + call = self.logger.error + emsg = '' + if isinstance(exc, OSError): + if exc.errno in (errno.EIO, errno.ENOSPC): + emsg = str(exc) + else: + call = self.logger.exception + elif isinstance(exc, socket.error): + if exc.errno == errno.ECONNREFUSED: + emsg = 'Connection refused' + elif exc.errno == errno.EHOSTUNREACH: + emsg = 'Host unreachable' + else: + call = self.logger.exception + elif isinstance(exc, eventlet.Timeout): + emsg = exc.__class__.__name__ + if hasattr(exc, 'seconds'): + emsg += ' (%ss)' % exc.seconds + if isinstance(exc, MessageTimeout): + if exc.msg: + emsg += ' %s' % exc.msg + else: + call = self.logger.exception + call('%s %s: %s' % (self.server, msg, emsg), *args) + + +def get_logger(conf, name): + """ + Get the current system logger using config settings. + + **Log config and defaults**:: + + log_facility = LOG_LOCAL0 + log_level = INFO + + :param conf: Configuration dict to read settings from + :param name: Name of the logger + """ + root_logger = logging.getLogger() + if hasattr(get_logger, 'handler') and get_logger.handler: + root_logger.removeHandler(get_logger.handler) + get_logger.handler = None + if conf is None: + root_logger.setLevel(logging.INFO) + return NamedLogger(root_logger, name) + get_logger.handler = SysLogHandler(address='/dev/log', + facility=getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), + SysLogHandler.LOG_LOCAL0)) + root_logger.addHandler(get_logger.handler) + root_logger.setLevel( + getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO)) + return NamedLogger(root_logger, name) + + +def whataremyips(): + """ + Get the machine's ip addresses using ifconfig + + :returns: list of Strings of IPv4 ip addresses + """ + proc = subprocess.Popen(['/sbin/ifconfig'], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + ret_val = proc.wait() + results = proc.stdout.read().split('\n') + return [x.split(':')[1].split()[0] for x in results if 'inet addr' in x] + + +def storage_directory(datadir, partition, hash): + """ + Get the storage directory + + :param datadir: Base data directory + :param partition: Partition + :param hash: Account, container or object hash + :returns: Storage directory + """ + return os.path.join(datadir, partition, hash[-3:], hash) + + +def hash_path(account, container=None, object=None, raw_digest=False): + """ + Get the connonical hash for an account/container/object + + :param account: Account + :param container: Container + :param object: Object + :param raw_digest: If True, return the raw version rather than a hex digest + :returns: hash string + """ + if object and not container: + raise ValueError('container is required if object is provided') + paths = [account] + if container: + paths.append(container) + if object: + paths.append(object) + if raw_digest: + return md5('/' + '/'.join(paths) + HASH_PATH_SUFFIX).digest() + else: + return md5('/' + '/'.join(paths) + HASH_PATH_SUFFIX).hexdigest() + + +@contextmanager +def lock_path(directory, timeout=10): + """ + Context manager that acquires a lock on a directory. This will block until + the lock can be acquired, or the timeout time has expired (whichever occurs + first). + + :param directory: directory to be locked + :param timeout: timeout (in seconds) + """ + mkdirs(directory) + fd = os.open(directory, os.O_RDONLY) + try: + with LockTimeout(timeout, directory): + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except IOError, err: + if err.errno != errno.EAGAIN: + raise + sleep(0.01) + yield True + finally: + os.close(fd) + + +def lock_parent_directory(filename, timeout=10): + """ + Context manager that acquires a lock on the parent directory of the given + file path. This will block until the lock can be acquired, or the timeout + time has expired (whichever occurs first). + + :param filename: file path of the parent directory to be locked + :param timeout: timeout (in seconds) + """ + return lock_path(os.path.dirname(filename)) + + +def get_time_units(time_amount): + """ + Get a nomralized length of time in the largest unit of time (hours, + minutes, or seconds.) + + :param time_amount: length of time in seconds + :returns: A touple of (length of time, unit of time) where unit of time is + one of ('h', 'm', 's') + """ + time_unit = 's' + if time_amount > 60: + time_amount /= 60 + time_unit = 'm' + if time_amount > 60: + time_amount /= 60 + time_unit = 'h' + return time_amount, time_unit + + +def compute_eta(start_time, current_value, final_value): + """ + Compute an ETA. Now only if we could also have a progress bar... + + :param start_time: Unix timestamp when the operation began + :param current_value: Current value + :param final_value: Final value + :returns: ETA as a tuple of (length of time, unit of time) where unit of + time is one of ('h', 'm', 's') + """ + elapsed = time.time() - start_time + completion = (float(current_value) / final_value) or 0.00001 + return get_time_units(1.0 / completion * elapsed - elapsed) + + +def iter_devices_partitions(devices_dir, item_type): + """ + Iterate over partitions accross all devices. + + :param devices_dir: Path to devices + :param item_type: One of 'accounts', 'containers', or 'objects' + :returns: Each iteration returns a tuple of (device, partition) + """ + devices = os.listdir(devices_dir) + shuffle(devices) + devices_partitions = [] + for device in devices: + partitions = os.listdir(os.path.join(devices_dir, device, item_type)) + shuffle(partitions) + devices_partitions.append((device, iter(partitions))) + yielded = True + while yielded: + yielded = False + for device, partitions in devices_partitions: + try: + yield device, partitions.next() + yielded = True + except StopIteration: + pass + + +def unlink_older_than(path, mtime): + """ + Remove any file in a given path that that was last modified before mtime. + + :param path: Path to remove file from + :mtime: Timestamp of oldest file to keep + """ + if os.path.exists(path): + for fname in os.listdir(path): + fpath = os.path.join(path, fname) + try: + if os.path.getmtime(fpath) < mtime: + os.unlink(fpath) + except OSError: + pass diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py new file mode 100644 index 0000000000..d025896d3f --- /dev/null +++ b/swift/common/wsgi.py @@ -0,0 +1,164 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""WSGI tools for use with swift.""" + +import errno +import os +import signal +import sys +import time +import mimetools + +import eventlet +from eventlet import greenio, GreenPool, sleep, wsgi, listen + +# Hook to ensure connection resets don't blow up our servers. +# Remove with next release of Eventlet that has it in the set already. +from errno import ECONNRESET +wsgi.ACCEPT_ERRNO.add(ECONNRESET) + +from eventlet.green import socket, ssl + +from swift.common.utils import get_logger, drop_privileges, \ + LoggerFileObject, NullLogger + +def monkey_patch_mimetools(): + """ + mimetools.Message defaults content-type to "text/plain" + This changes it to default to None, so we can detect missing headers. + """ + + orig_parsetype = mimetools.Message.parsetype + + def parsetype(self): + if not self.typeheader: + self.type = None + self.maintype = None + self.subtype = None + self.plisttext = '' + else: + orig_parsetype(self) + + mimetools.Message.parsetype = parsetype + +# We might be able to pull pieces of this out to test, but right now it seems +# like more work than it's worth. +def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover + """ + Loads common settings from conf, then instantiates app and runs + the server using the specified number of workers. + + :param app: WSGI callable + :param conf: Configuration dictionary + """ + if 'logger' in kwargs: + logger = kwargs['logger'] + else: + logger = get_logger(conf, app.log_name) + # log uncaught exceptions + sys.excepthook = lambda *exc_info: \ + logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) + sys.stdout = sys.stderr = LoggerFileObject(logger) + + try: + os.setsid() + except OSError: + no_cover = True # pass + bind_addr = (conf.get('bind_ip', '0.0.0.0'), + int(conf.get('bind_port', kwargs.get('default_port', 8080)))) + sock = None + retry_until = time.time() + 30 + while not sock and time.time() < retry_until: + try: + sock = listen(bind_addr) + if 'cert_file' in conf: + sock = ssl.wrap_socket(sock, certfile=conf['cert_file'], + keyfile=conf['key_file']) + except socket.error, err: + if err.args[0] != errno.EADDRINUSE: + raise + sleep(0.1) + if not sock: + raise Exception('Could not bind to %s:%s after trying for 30 seconds' % + bind_addr) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # in my experience, sockets can hang around forever without keepalive + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) + worker_count = int(conf.get('workers', '1')) + drop_privileges(conf.get('user', 'swift')) + if isinstance(app, type): + # Instantiate app if it hasn't been already + app = app(conf, *args) + + def run_server(): + wsgi.HttpProtocol.default_request_version = "HTTP/1.0" + eventlet.hubs.use_hub('poll') + eventlet.patcher.monkey_patch(all=False, socket=True) + monkey_patch_mimetools() + pool = GreenPool(size=1024) + try: + wsgi.server(sock, app, NullLogger(), custom_pool=pool) + except socket.error, err: + if err[0] != errno.EINVAL: + raise + pool.waitall() + + # Useful for profiling [no forks]. + if worker_count == 0: + run_server() + return + + def kill_children(*args): + """Kills the entire process group.""" + logger.error('SIGTERM received') + signal.signal(signal.SIGTERM, signal.SIG_IGN) + running[0] = False + os.killpg(0, signal.SIGTERM) + + def hup(*args): + """Shuts down the server, but allows running requests to complete""" + logger.error('SIGHUP received') + signal.signal(signal.SIGHUP, signal.SIG_IGN) + running[0] = False + + running = [True] + signal.signal(signal.SIGTERM, kill_children) + signal.signal(signal.SIGHUP, hup) + children = [] + while running[0]: + while len(children) < worker_count: + pid = os.fork() + if pid == 0: + signal.signal(signal.SIGHUP, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + run_server() + logger.info('Child %d exiting normally' % os.getpid()) + return + else: + logger.info('Started child %s' % pid) + children.append(pid) + try: + pid, status = os.wait() + if os.WIFEXITED(status) or os.WIFSIGNALED(status): + logger.error('Removing dead child %s' % pid) + children.remove(pid) + except OSError, err: + if err.errno not in (errno.EINTR, errno.ECHILD): + raise + greenio.shutdown_safe(sock) + sock.close() + logger.info('Exited') diff --git a/swift/container/__init__.py b/swift/container/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/swift/container/auditor.py b/swift/container/auditor.py new file mode 100644 index 0000000000..6947db7e5c --- /dev/null +++ b/swift/container/auditor.py @@ -0,0 +1,271 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import socket +import time +from random import choice, random +from urllib import quote + +from eventlet import Timeout + +from swift.container import server as container_server +from swift.common.db import ContainerBroker +from swift.common.bufferedhttp import http_connect +from swift.common.exceptions import ConnectionTimeout +from swift.common.ring import Ring +from swift.common.utils import get_logger + + +class AuditException(Exception): + pass + + +class ContainerAuditor(object): + """Audit containers.""" + + def __init__(self, server_conf, auditor_conf): + self.logger = get_logger(auditor_conf, 'container-auditor') + self.devices = server_conf.get('devices', '/srv/node') + self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.interval = int(auditor_conf.get('interval', 1800)) + swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') + self.account_ring = None + self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') + self.object_ring = None + self.node_timeout = int(auditor_conf.get('node_timeout', 10)) + self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) + self.max_object_count = int(auditor_conf.get('max_object_count', 100)) + self.account_passes = 0 + self.account_failures = 0 + self.account_errors = 0 + self.object_passes = 0 + self.object_failures = 0 + self.object_errors = 0 + + def get_account_ring(self): + """ + Get the account ring. Loads the ring if neccesary. + + :returns: account ring + """ + if not self.account_ring: + self.logger.debug( + 'Loading account ring from %s' % self.account_ring_path) + self.account_ring = Ring(self.account_ring_path) + return self.account_ring + + def get_object_ring(self): + """ + Get the object ring. Loads the ring if neccesary. + + :returns: object ring + """ + if not self.object_ring: + self.logger.debug( + 'Loading object ring from %s' % self.object_ring_path) + self.object_ring = Ring(self.object_ring_path) + return self.object_ring + + def audit_forever(self): # pragma: no cover + """Run the container audit until stopped.""" + reported = time.time() + time.sleep(random() * self.interval) + while True: + begin = time.time() + pids = [] + for device in os.listdir(self.devices): + if self.mount_check and not\ + os.path.ismount(os.path.join(self.devices, device)): + self.logger.debug( + 'Skipping %s as it is not mounted' % device) + continue + self.container_audit(device) + if time.time() - reported >= 3600: # once an hour + self.logger.info( + 'Since %s: Remote audits with accounts: %s passed audit, ' + '%s failed audit, %s errors Remote audits with objects: ' + '%s passed audit, %s failed audit, %s errors' % + (time.ctime(reported), self.account_passes, + self.account_failures, self.account_errors, + self.object_passes, self.object_failures, + self.object_errors)) + reported = time.time() + self.account_passes = 0 + self.account_failures = 0 + self.account_errors = 0 + self.object_passes = 0 + self.object_failures = 0 + self.object_errors = 0 + elapsed = time.time() - begin + if elapsed < self.interval: + time.sleep(self.interval - elapsed) + + def audit_once(self): + """Run the container audit once.""" + self.logger.info('Begin container audit "once" mode') + begin = time.time() + for device in os.listdir(self.devices): + if self.mount_check and \ + not os.path.ismount(os.path.join(self.devices, device)): + self.logger.debug( + 'Skipping %s as it is not mounted' % device) + continue + self.container_audit(device) + elapsed = time.time() - begin + self.logger.info( + 'Container audit "once" mode completed: %.02fs' % elapsed) + + def container_audit(self, device): + """ + Audit any containers found on the device + + :param device: device to audit + """ + datadir = os.path.join(self.devices, device, container_server.DATADIR) + if not os.path.exists(datadir): + return + broker = None + partition = None + attempts = 100 + while not broker and attempts: + attempts -= 1 + try: + partition = choice(os.listdir(datadir)) + fpath = os.path.join(datadir, partition) + if not os.path.isdir(fpath): + continue + suffix = choice(os.listdir(fpath)) + fpath = os.path.join(fpath, suffix) + if not os.path.isdir(fpath): + continue + hsh = choice(os.listdir(fpath)) + fpath = os.path.join(fpath, hsh) + if not os.path.isdir(fpath): + continue + except IndexError: + continue + for fname in sorted(os.listdir(fpath), reverse=True): + if fname.endswith('.db'): + broker = ContainerBroker(os.path.join(fpath, fname)) + if broker.is_deleted(): + broker = None + break + if not broker: + return + info = broker.get_info() + found = False + good_response = False + results = [] + part, nodes = self.get_account_ring().get_nodes(info['account']) + for node in nodes: + try: + with ConnectionTimeout(self.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], part, 'GET', + '/%s' % info['account'], + query_string='prefix=%s' % + quote(info['container'])) + with Timeout(self.node_timeout): + resp = conn.getresponse() + body = resp.read() + if 200 <= resp.status <= 299: + good_reponse = True + for cname in body.split('\n'): + if cname == info['container']: + found = True + break + if found: + break + else: + results.append('%s:%s/%s %s %s = %s' % (node['ip'], + node['port'], node['device'], resp.status, + resp.reason, repr(body))) + else: + results.append('%s:%s/%s %s %s' % + (node['ip'], node['port'], node['device'], + resp.status, resp.reason)) + except socket.error, err: + results.append('%s:%s/%s Socket Error: %s' % (node['ip'], + node['port'], node['device'], err)) + except ConnectionTimeout: + results.append('%(ip)s:%(port)s/%(device)s ConnectionTimeout' % + node) + except Timeout: + results.append('%(ip)s:%(port)s/%(device)s Timeout' % node) + except Exception, err: + self.logger.exception('ERROR With remote server ' + '%(ip)s:%(port)s/%(device)s' % node) + results.append('%s:%s/%s Exception: %s' % (node['ip'], + node['port'], node['device'], err)) + if found: + self.account_passes += 1 + self.logger.debug('Audit passed for /%s/%s %s' % (info['account'], + info['container'], broker.db_file)) + else: + if good_response: + self.account_failures += 1 + else: + self.account_errors += 1 + self.logger.error('ERROR Could not find container /%s/%s %s on ' + 'any of the primary account servers it should be on: %s' % + (info['account'], info['container'], broker.db_file, results)) + for obj in broker.get_random_objects(max_count=self.max_object_count): + found = False + results = [] + part, nodes = self.get_object_ring().get_nodes(info['account'], + info['container'], obj) + for node in nodes: + try: + with ConnectionTimeout(self.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], part, 'HEAD', + '/%s/%s/%s' % + (info['account'], info['container'], obj)) + with Timeout(self.node_timeout): + resp = conn.getresponse() + body = resp.read() + if 200 <= resp.status <= 299: + found = True + break + else: + results.append('%s:%s/%s %s %s' % (node['ip'], + node['port'], node['device'], resp.status, + resp.reason)) + except socket.error, err: + results.append('%s:%s/%s Socket Error: %s' % (node['ip'], + node['port'], node['device'], err)) + except ConnectionTimeout: + results.append( + '%(ip)s:%(port)s/%(device)s ConnectionTimeout' % node) + except Timeout: + results.append('%(ip)s:%(port)s/%(device)s Timeout' % node) + except Exception, err: + self.logger.exception('ERROR With remote server ' + '%(ip)s:%(port)s/%(device)s' % node) + results.append('%s:%s/%s Exception: %s' % (node['ip'], + node['port'], node['device'], err)) + if found: + self.object_passes += 1 + self.logger.debug('Audit passed for /%s/%s %s object %s' % + (info['account'], info['container'], broker.db_file, obj)) + else: + self.object_errors += 1 + self.logger.error('ERROR Could not find object /%s/%s/%s ' + 'referenced by %s on any of the primary object ' + 'servers it should be on: %s' % (info['account'], + info['container'], obj, broker.db_file, results)) diff --git a/swift/container/server.py b/swift/container/server.py new file mode 100644 index 0000000000..ea8d2d8b22 --- /dev/null +++ b/swift/container/server.py @@ -0,0 +1,383 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import errno +import os +import socket +import time +import traceback +from urllib import unquote +from xml.sax import saxutils +from datetime import datetime + +import simplejson +from eventlet.timeout import Timeout +from eventlet import TimeoutError +from webob import Request, Response +from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ + HTTPCreated, HTTPException, HTTPInternalServerError, HTTPNoContent, \ + HTTPNotFound, HTTPPreconditionFailed, HTTPMethodNotAllowed + +from swift.common import CONTAINER_LISTING_LIMIT +from swift.common.db import ContainerBroker +from swift.common.utils import get_logger, get_param, hash_path, \ + storage_directory, split_path, mkdirs +from swift.common.constraints import check_mount, check_float, \ + check_xml_encodable +from swift.common.bufferedhttp import http_connect +from swift.common.healthcheck import healthcheck +from swift.common.exceptions import ConnectionTimeout, MessageTimeout +from swift.common.db_replicator import ReplicatorRpc + +DATADIR = 'containers' + + +class ContainerController(object): + """WSGI Controller for the container server.""" + + log_name = 'container' + + def __init__(self, conf): + self.logger = get_logger(conf, self.log_name) + self.root = conf.get('devices', '/srv/node/') + self.mount_check = conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.node_timeout = int(conf.get('node_timeout', 3)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, + ContainerBroker, self.mount_check) + + def _get_container_broker(self, drive, part, account, container): + """ + Get a DB broker for the container. + + :param drive: drive that holds the container + :param part: partition the container is in + :param account: account name + :param container: container name + :returns: ContainerBroker object + """ + hsh = hash_path(account, container) + db_dir = storage_directory(DATADIR, part, hsh) + db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') + return ContainerBroker(db_path, account=account, container=container, + logger=self.logger) + + def account_update(self, req, account, container, broker): + """ + Update the account server with latest container info. + + :param req: webob.Request object + :param account: account name + :param container: container name + :param borker: container DB broker object + :returns: if the account request returns a 404 error code, + HTTPNotFound response object, otherwise None. + """ + account_host = req.headers.get('X-Account-Host') + account_partition = req.headers.get('X-Account-Partition') + account_device = req.headers.get('X-Account-Device') + if all([account_host, account_partition, account_device]): + account_ip, account_port = account_host.split(':') + new_path = '/' + '/'.join([account, container]) + info = broker.get_info() + account_headers = {'x-put-timestamp': info['put_timestamp'], + 'x-delete-timestamp': info['delete_timestamp'], + 'x-object-count': info['object_count'], + 'x-bytes-used': info['bytes_used'], + 'x-cf-trans-id': req.headers.get('X-Cf-Trans-Id', '-')} + if req.headers.get('x-account-override-deleted', 'no').lower() == \ + 'yes': + account_headers['x-account-override-deleted'] = 'yes' + try: + with ConnectionTimeout(self.conn_timeout): + conn = http_connect(account_ip, account_port, + account_device, account_partition, 'PUT', new_path, + account_headers) + with Timeout(self.node_timeout): + account_response = conn.getresponse() + account_response.read() + if account_response.status == 404: + return HTTPNotFound(request=req) + elif account_response.status < 200 or \ + account_response.status > 299: + self.logger.error('ERROR Account update failed ' + 'with %s:%s/%s transaction %s (will retry ' + 'later): Response %s %s' % (account_ip, + account_port, account_device, + req.headers.get('x-cf-trans-id'), + account_response.status, + account_response.reason)) + except: + self.logger.exception('ERROR account update failed with ' + '%s:%s/%s transaction %s (will retry later)' % + (account_ip, account_port, account_device, + req.headers.get('x-cf-trans-id', '-'))) + return None + + def DELETE(self, req): + """Handle HTTP DELETE request.""" + try: + drive, part, account, container, obj = split_path( + unquote(req.path), 4, 5, True) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + if 'x-timestamp' not in req.headers or \ + not check_float(req.headers['x-timestamp']): + return HTTPBadRequest(body='Missing timestamp', request=req, + content_type='text/plain') + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + broker = self._get_container_broker(drive, part, account, container) + if not os.path.exists(broker.db_file): + return HTTPNotFound() + if obj: # delete object + broker.delete_object(obj, req.headers.get('x-timestamp')) + return HTTPNoContent(request=req) + else: + # delete container + if not broker.empty(): + return HTTPConflict(request=req) + existed = float(broker.get_info()['put_timestamp']) and \ + not broker.is_deleted() + broker.delete_db(req.headers['X-Timestamp']) + if not broker.is_deleted(): + return HTTPConflict(request=req) + resp = self.account_update(req, account, container, broker) + if resp: + return resp + if existed: + return HTTPNoContent(request=req) + return HTTPAccepted(request=req) + + def PUT(self, req): + """Handle HTTP PUT request.""" + try: + drive, part, account, container, obj = split_path( + unquote(req.path), 4, 5, True) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + if 'x-timestamp' not in req.headers or \ + not check_float(req.headers['x-timestamp']): + return HTTPBadRequest(body='Missing timestamp', request=req, + content_type='text/plain') + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + broker = self._get_container_broker(drive, part, account, container) + if obj: # put container object + if not os.path.exists(broker.db_file): + return HTTPNotFound() + broker.put_object(obj, req.headers['x-timestamp'], + int(req.headers['x-size']), req.headers['x-content-type'], + req.headers['x-etag']) + return HTTPCreated(request=req) + else: # put container + if not os.path.exists(broker.db_file): + broker.initialize(req.headers['x-timestamp']) + created = True + else: + created = broker.is_deleted() + broker.update_put_timestamp(req.headers['x-timestamp']) + if broker.is_deleted(): + return HTTPConflict(request=req) + resp = self.account_update(req, account, container, broker) + if resp: + return resp + if created: + return HTTPCreated(request=req) + else: + return HTTPAccepted(request=req) + + def HEAD(self, req): + """Handle HTTP HEAD request.""" + try: + drive, part, account, container, obj = split_path( + unquote(req.path), 4, 5, True) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + broker = self._get_container_broker(drive, part, account, container) + broker.pending_timeout = 0.1 + broker.stale_reads_ok = True + if broker.is_deleted(): + return HTTPNotFound(request=req) + info = broker.get_info() + headers = { + 'X-Container-Object-Count': info['object_count'], + 'X-Container-Bytes-Used': info['bytes_used'], + 'X-Timestamp': info['created_at'], + 'X-PUT-Timestamp': info['put_timestamp'], + } + return HTTPNoContent(request=req, headers=headers) + + def GET(self, req): + """Handle HTTP GET request.""" + try: + drive, part, account, container, obj = split_path( + unquote(req.path), 4, 5, True) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + broker = self._get_container_broker(drive, part, account, container) + broker.pending_timeout = 0.1 + broker.stale_reads_ok = True + if broker.is_deleted(): + return HTTPNotFound(request=req) + info = broker.get_info() + resp_headers = { + 'X-Container-Object-Count': info['object_count'], + 'X-Container-Bytes-Used': info['bytes_used'], + 'X-Timestamp': info['created_at'], + 'X-PUT-Timestamp': info['put_timestamp'], + } + try: + path = get_param(req, 'path') + prefix = get_param(req, 'prefix') + delimiter = get_param(req, 'delimiter') + if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): + # delimiters can be made more flexible later + return HTTPPreconditionFailed(body='Bad delimiter') + marker = get_param(req, 'marker', '') + limit = CONTAINER_LISTING_LIMIT + given_limit = get_param(req, 'limit') + if given_limit and given_limit.isdigit(): + limit = int(given_limit) + if limit > CONTAINER_LISTING_LIMIT: + return HTTPPreconditionFailed(request=req, + body='Maximum limit is %d' % CONTAINER_LISTING_LIMIT) + query_format = get_param(req, 'format') + except UnicodeDecodeError, err: + return HTTPBadRequest(body='parameters not utf8', + content_type='text/plain', request=req) + header_format = req.accept.first_match(['text/plain', + 'application/json', + 'application/xml']) + format = query_format if query_format else header_format + if format.startswith('application/'): + format = format[12:] + container_list = broker.list_objects_iter(limit, marker, prefix, + delimiter, path) + if format == 'json': + out_content_type = 'application/json' + json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s', + '"content_type":%s, "last_modified":"%s"'] + json_pattern = '{' + ','.join(json_pattern) + '}' + json_out = [] + for (name, created_at, size, content_type, etag) in container_list: + # escape name and format date here + name = simplejson.dumps(name) + created_at = datetime.utcfromtimestamp( + float(created_at)).isoformat() + if content_type is None: + json_out.append('{"subdir":%s}' % name) + else: + content_type = simplejson.dumps(content_type) + json_out.append(json_pattern % (name, + etag, + size, + content_type, + created_at)) + container_list = '[' + ','.join(json_out) + ']' + elif format == 'xml': + out_content_type = 'application/xml' + xml_output = [] + for (name, created_at, size, content_type, etag) in container_list: + # escape name and format date here + name = saxutils.escape(name) + created_at = datetime.utcfromtimestamp( + float(created_at)).isoformat() + if content_type is None: + xml_output.append('' % name) + else: + content_type = saxutils.escape(content_type) + xml_output.append('%s%s'\ + '%d%s'\ + '%s' % \ + (name, etag, size, content_type, created_at)) + container_list = ''.join([ + '\n', + '' % saxutils.quoteattr(container), + ''.join(xml_output), '']) + else: + if not container_list: + return HTTPNoContent(request=req, headers=resp_headers) + out_content_type = 'text/plain' + container_list = '\n'.join(r[0] for r in container_list) + '\n' + ret = Response(body=container_list, request=req, headers=resp_headers) + ret.content_type = out_content_type + ret.charset = 'utf8' + return ret + + def POST(self, req): + """ + Handle HTTP POST request (json-encoded RPC calls for replication.) + """ + try: + post_args = split_path(unquote(req.path), 3) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain', + request=req) + drive, partition, hash = post_args + if self.mount_check and not check_mount(self.root, drive): + return Response(status='507 %s is not mounted' % drive) + try: + args = simplejson.load(req.body_file) + except ValueError, err: + return HTTPBadRequest(body=str(err), content_type='text/plain') + ret = self.replicator_rpc.dispatch(post_args, args) + ret.request = req + return ret + + def __call__(self, env, start_response): + start_time = time.time() + req = Request(env) + if req.path_info == '/healthcheck': + return healthcheck(req)(env, start_response) + elif not check_xml_encodable(req.path_info): + res = HTTPPreconditionFailed(body='Invalid UTF8') + else: + try: + if hasattr(self, req.method): + res = getattr(self, req.method)(req) + else: + res = HTTPMethodNotAllowed() + except: + self.logger.exception('ERROR __call__ error with %s %s ' + 'transaction %s' % (env.get('REQUEST_METHOD', '-'), + env.get('PATH_INFO', '-'), env.get('HTTP_X_CF_TRANS_ID', + '-'))) + res = HTTPInternalServerError(body=traceback.format_exc()) + trans_time = '%.4f' % (time.time() - start_time) + log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % ( + req.remote_addr, + time.strftime('%d/%b/%Y:%H:%M:%S +0000', + time.gmtime()), + req.method, req.path, + res.status.split()[0], res.content_length or '-', + req.headers.get('x-cf-trans-id', '-'), + req.referer or '-', req.user_agent or '-', + trans_time) + if req.method.upper() == 'POST': + self.logger.debug(log_message) + else: + self.logger.info(log_message) + return res(env, start_response) diff --git a/swift/container/updater.py b/swift/container/updater.py new file mode 100644 index 0000000000..486cb45959 --- /dev/null +++ b/swift/container/updater.py @@ -0,0 +1,232 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import logging +import os +import signal +import socket +import sys +import time +from random import random, shuffle + +from eventlet import spawn, patcher, Timeout + +from swift.container.server import DATADIR +from swift.common.bufferedhttp import http_connect +from swift.common.db import ContainerBroker +from swift.common.exceptions import ConnectionTimeout +from swift.common.ring import Ring +from swift.common.utils import get_logger, whataremyips + + +class ContainerUpdater(object): + """Update container information in account listings.""" + + def __init__(self, server_conf, updater_conf): + self.logger = get_logger(updater_conf, 'container-updater') + self.devices = server_conf.get('devices', '/srv/node') + self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(updater_conf.get('interval', 300)) + self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') + self.account_ring = None + self.concurrency = int(updater_conf.get('concurrency', 4)) + self.slowdown = float(updater_conf.get('slowdown', 0.01)) + self.node_timeout = int(updater_conf.get('node_timeout', 3)) + self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5)) + self.no_changes = 0 + self.successes = 0 + self.failures = 0 + + def get_account_ring(self): + """Get the account ring. Load it if it hasn't been yet.""" + if not self.account_ring: + self.logger.debug( + 'Loading account ring from %s' % self.account_ring_path) + self.account_ring = Ring(self.account_ring_path) + return self.account_ring + + def get_paths(self): + """ + Get paths to all of the partitions on each drive to be processed. + + :returns: a list of paths + """ + paths = [] + ips = whataremyips() + for device in os.listdir(self.devices): + dev_path = os.path.join(self.devices, device) + if self.mount_check and not os.path.ismount(dev_path): + self.logger.warn('%s is not mounted' % device) + continue + con_path = os.path.join(dev_path, DATADIR) + if not os.path.exists(con_path): + continue + for partition in os.listdir(con_path): + paths.append(os.path.join(con_path, partition)) + shuffle(paths) + return paths + + def update_forever(self): # pragma: no cover + """ + Run the updator continuously. + """ + time.sleep(random() * self.interval) + while True: + self.logger.info('Begin container update sweep') + begin = time.time() + pids = [] + # read from account ring to ensure it's fresh + self.get_account_ring().get_nodes('') + for path in self.get_paths(): + while len(pids) >= self.concurrency: + pids.remove(os.wait()[0]) + pid = os.fork() + if pid: + pids.append(pid) + else: + signal.signal(signal.SIGTERM, signal.SIG_DFL) + patcher.monkey_patch(all=False, socket=True) + self.no_changes = 0 + self.successes = 0 + self.failures = 0 + forkbegin = time.time() + self.container_sweep(path) + elapsed = time.time() - forkbegin + self.logger.debug( + 'Container update sweep of %s completed: ' + '%.02fs, %s successes, %s failures, %s with no changes' + % (path, elapsed, self.successes, self.failures, + self.no_changes)) + sys.exit() + while pids: + pids.remove(os.wait()[0]) + elapsed = time.time() - begin + self.logger.info('Container update sweep completed: %.02fs' % + elapsed) + if elapsed < self.interval: + time.sleep(self.interval - elapsed) + + def update_once_single_threaded(self): + """ + Run the updater once. + """ + patcher.monkey_patch(all=False, socket=True) + self.logger.info('Begin container update single threaded sweep') + begin = time.time() + self.no_changes = 0 + self.successes = 0 + self.failures = 0 + for path in self.get_paths(): + self.container_sweep(path) + elapsed = time.time() - begin + self.logger.info('Container update single threaded sweep completed: ' + '%.02fs, %s successes, %s failures, %s with no changes' % + (elapsed, self.successes, self.failures, self.no_changes)) + + def container_sweep(self, path): + """ + Walk the path looking for container DBs and process them. + + :param path: path to walk + """ + for root, dirs, files in os.walk(path): + for file in files: + if file.endswith('.db'): + self.process_container(os.path.join(root, file)) + time.sleep(self.slowdown) + + def process_container(self, dbfile): + """ + Process a container, and update the information in the account. + + :param dbfile: container DB to process + """ + broker = ContainerBroker(dbfile, logger=self.logger) + info = broker.get_info() + # Don't send updates if the container was auto-created since it + # definitely doesn't have up to date statistics. + if float(info['put_timestamp']) <= 0: + return + if info['put_timestamp'] > info['reported_put_timestamp'] or \ + info['delete_timestamp'] > info['reported_delete_timestamp'] \ + or info['object_count'] != info['reported_object_count'] or \ + info['bytes_used'] != info['reported_bytes_used']: + container = '/%s/%s' % (info['account'], info['container']) + part, nodes = self.get_account_ring().get_nodes(info['account']) + events = [spawn(self.container_report, node, part, container, + info['put_timestamp'], info['delete_timestamp'], + info['object_count'], info['bytes_used']) + for node in nodes] + successes = 0 + failures = 0 + for event in events: + if 200 <= event.wait() < 300: + successes += 1 + else: + failures += 1 + if successes > failures: + self.successes += 1 + self.logger.debug( + 'Update report sent for %s %s' % (container, dbfile)) + broker.reported(info['put_timestamp'], + info['delete_timestamp'], info['object_count'], + info['bytes_used']) + else: + self.failures += 1 + self.logger.debug( + 'Update report failed for %s %s' % (container, dbfile)) + else: + self.no_changes += 1 + + def container_report(self, node, part, container, put_timestamp, + delete_timestamp, count, bytes): + """ + Report container info to an account server. + + :param node: node dictionary from the account ring + :param part: partition the account is on + :param container: container name + :param put_timestamp: put timestamp + :param delete_timestamp: delete timestamp + :param count: object count in the container + :param bytes: bytes used in the container + """ + with ConnectionTimeout(self.conn_timeout): + try: + conn = http_connect( + node['ip'], node['port'], node['device'], part, + 'PUT', container, + headers={'X-Put-Timestamp': put_timestamp, + 'X-Delete-Timestamp': delete_timestamp, + 'X-Object-Count': count, + 'X-Bytes-Used': bytes, + 'X-Account-Override-Deleted': 'yes'}) + except: + self.logger.exception('ERROR account update failed with ' + '%(ip)s:%(port)s/%(device)s (will retry later): ' % node) + return 500 + with Timeout(self.node_timeout): + try: + resp = conn.getresponse() + resp.read() + return resp.status + except: + if self.logger.getEffectiveLevel() <= logging.DEBUG: + self.logger.exception( + 'Exception with %(ip)s:%(port)s/%(device)s' % node) + return 500 diff --git a/swift/obj/__init__.py b/swift/obj/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py new file mode 100644 index 0000000000..91079ebade --- /dev/null +++ b/swift/obj/auditor.py @@ -0,0 +1,233 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cPickle as pickle +import os +import socket +import sys +import time +from hashlib import md5 +from random import choice, random +from urllib import quote + +from eventlet import Timeout + +from swift.obj import server as object_server +from swift.common.bufferedhttp import http_connect +from swift.common.exceptions import ConnectionTimeout +from swift.common.ring import Ring +from swift.common.utils import get_logger, renamer +from swift.common.exceptions import AuditException + + +class ObjectAuditor(object): + """Audit objects.""" + + def __init__(self, server_conf, auditor_conf): + self.logger = get_logger(auditor_conf, 'object-auditor') + self.devices = server_conf.get('devices', '/srv/node') + self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.interval = int(auditor_conf.get('interval', 1800)) + swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') + self.container_ring = None + self.node_timeout = int(auditor_conf.get('node_timeout', 10)) + self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) + self.passes = 0 + self.quarantines = 0 + self.errors = 0 + self.container_passes = 0 + self.container_failures = 0 + self.container_errors = 0 + + def get_container_ring(self): + """ + Get the container ring, loading it if neccesary. + + :returns: container ring + """ + if not self.container_ring: + self.logger.debug( + 'Loading container ring from %s' % self.container_ring_path) + self.container_ring = Ring(self.container_ring_path) + return self.container_ring + + def audit_forever(self): # pragma: no cover + """Run the object audit until stopped.""" + reported = time.time() + time.sleep(random() * self.interval) + while True: + begin = time.time() + pids = [] + # read from container ring to ensure it's fresh + self.get_container_ring().get_nodes('') + for device in os.listdir(self.devices): + if self.mount_check and not \ + os.path.ismount(os.path.join(self.devices, device)): + self.logger.debug( + 'Skipping %s as it is not mounted' % device) + continue + self.object_audit(device) + if time.time() - reported >= 3600: # once an hour + self.logger.info( + 'Since %s: Locally: %d passed audit, %d quarantined, %d ' + 'errors Remote audits with containers: %s passed audit, ' + '%s failed audit, %s errors' % + (time.ctime(reported), self.passes, self.quarantines, + self.errors, self.container_passes, + self.container_failures, self.container_errors)) + reported = time.time() + self.passes = 0 + self.quarantines = 0 + self.errors = 0 + self.container_passes = 0 + self.container_failures = 0 + self.container_errors = 0 + elapsed = time.time() - begin + if elapsed < self.interval: + time.sleep(self.interval - elapsed) + + def audit_once(self): + """Run the object audit once.""" + self.logger.info('Begin object audit "once" mode') + begin = time.time() + for device in os.listdir(self.devices): + if self.mount_check and \ + not os.path.ismount(os.path.join(self.devices, device)): + self.logger.debug( + 'Skipping %s as it is not mounted' % device) + continue + self.object_audit(device) + elapsed = time.time() - begin + self.logger.info( + 'Object audit "once" mode completed: %.02fs' % elapsed) + + def object_audit(self, device): + """Walk the device, and audit any objects found.""" + datadir = os.path.join(self.devices, device, object_server.DATADIR) + if not os.path.exists(datadir): + return + name = None + partition = None + attempts = 100 + while not name and attempts: + attempts -= 1 + try: + partition = choice(os.listdir(datadir)) + fpath = os.path.join(datadir, partition) + if not os.path.isdir(fpath): + continue + suffix = choice(os.listdir(fpath)) + fpath = os.path.join(fpath, suffix) + if not os.path.isdir(fpath): + continue + hsh = choice(os.listdir(fpath)) + fpath = os.path.join(fpath, hsh) + if not os.path.isdir(fpath): + continue + except IndexError: + continue + for fname in sorted(os.listdir(fpath), reverse=True): + if fname.endswith('.ts'): + break + if fname.endswith('.data'): + name = object_server.read_metadata( + os.path.join(fpath, fname))['name'] + break + if not name: + return + _, account, container, obj = name.split('/', 3) + df = object_server.DiskFile(self.devices, device, partition, account, + container, obj, keep_data_fp=True) + try: + if os.path.getsize(df.data_file) != \ + int(df.metadata['Content-Length']): + raise AuditException('Content-Length of %s does not match ' + 'file size of %s' % (int(df.metadata['Content-Length']), + os.path.getsize(df.data_file))) + etag = md5() + for chunk in df: + etag.update(chunk) + etag = etag.hexdigest() + if etag != df.metadata['ETag']: + raise AuditException("ETag of %s does not match file's md5 of " + "%s" % (df.metadata['ETag'], etag)) + except AuditException, err: + self.quarantines += 1 + self.logger.error('ERROR Object %s failed audit and will be ' + 'quarantined: %s' % (df.datadir, err)) + renamer(df.datadir, os.path.join(self.devices, device, + 'quarantined', 'objects', os.path.basename(df.datadir))) + return + except: + self.errors += 1 + self.logger.exception('ERROR Trying to audit %s' % df.datadir) + return + self.passes += 1 + found = False + good_response = False + results = [] + part, nodes = self.get_container_ring().get_nodes(account, container) + for node in nodes: + try: + with ConnectionTimeout(self.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], part, 'GET', + '/%s/%s' % (account, container), + query_string='prefix=%s' % quote(obj)) + with Timeout(self.node_timeout): + resp = conn.getresponse() + body = resp.read() + if 200 <= resp.status <= 299: + good_reponse = True + for oname in body.split('\n'): + if oname == obj: + found = True + break + if found: + break + else: + results.append('%s:%s/%s %s %s = %s' % (node['ip'], + node['port'], node['device'], resp.status, + resp.reason, repr(body))) + else: + results.append('%s:%s/%s %s %s' % + (node['ip'], node['port'], node['device'], + resp.status, resp.reason)) + except socket.error, err: + results.append('%s:%s/%s Socket Error: %s' % (node['ip'], + node['port'], node['device'], err)) + except ConnectionTimeout: + results.append('%(ip)s:%(port)s/%(device)s ConnectionTimeout' % + node) + except Timeout: + results.append('%(ip)s:%(port)s/%(device)s Timeout' % node) + except Exception, err: + self.logger.exception('ERROR With remote server ' + '%(ip)s:%(port)s/%(device)s' % node) + results.append('%s:%s/%s Exception: %s' % (node['ip'], + node['port'], node['device'], err)) + if found: + self.container_passes += 1 + self.logger.debug('Audit passed for %s %s' % (name, df.datadir)) + else: + if good_response: + self.container_failures += 1 + else: + self.container_errors += 1 + self.logger.error('ERROR Could not find object %s %s on any of ' + 'the primary container servers it should be on: %s' % (name, + df.datadir, results)) diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py new file mode 100644 index 0000000000..c9a83805ac --- /dev/null +++ b/swift/obj/replicator.py @@ -0,0 +1,501 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, sys +from os.path import isdir, join +from ConfigParser import ConfigParser +import random +import shutil +import time +import logging +import hashlib +import itertools +import cPickle as pickle + +import eventlet +from eventlet import GreenPool, tpool, Timeout, sleep +from eventlet.green import subprocess +from eventlet.support.greenlets import GreenletExit + +from swift.common.ring import Ring +from swift.common.utils import whataremyips, unlink_older_than, lock_path, \ + renamer, compute_eta +from swift.common.bufferedhttp import http_connect + + +REPLICAS = 3 +MAX_HANDOFFS = 5 +PICKLE_PROTOCOL = 2 +ONE_WEEK = 604800 +HASH_FILE = 'hashes.pkl' + +def hash_suffix(path, reclaim_age): + """ + Performs reclamation and returns an md5 of all (remaining) files. + + :param reclaim_age: age in seconds at which to remove tombstones + """ + md5 = hashlib.md5() + for hsh in sorted(os.listdir(path)): + hsh_path = join(path, hsh) + files = os.listdir(hsh_path) + if len(files) == 1: + if files[0].endswith('.ts'): + # remove tombstones older than reclaim_age + ts = files[0].rsplit('.', 1)[0] + if (time.time() - float(ts)) > reclaim_age: + os.unlink(join(hsh_path, files[0])) + files.remove(files[0]) + elif files: + files.sort(reverse=True) + meta = data = tomb = None + for filename in files: + if not meta and filename.endswith('.meta'): + meta = filename + if not data and filename.endswith('.data'): + data = filename + if not tomb and filename.endswith('.ts'): + tomb = filename + if (filename < tomb or # any file older than tomb + filename < data or # any file older than data + (filename.endswith('.meta') and + filename < meta)): # old meta + os.unlink(join(hsh_path, filename)) + files.remove(filename) + if not files: + os.rmdir(hsh_path) + for filename in files: + md5.update(filename) + try: + os.rmdir(path) + except OSError: + pass + return md5.hexdigest() + + +def recalculate_hashes(partition_dir, suffixes, reclaim_age=ONE_WEEK): + """ + Recalculates hashes for the given suffixes in the partition and updates + them in the partition's hashes file. + + :param partition_dir: directory of the partition in which to recalculate + :param suffixes: list of suffixes to recalculate + :param reclaim_age: age in seconds at which tombstones should be removed + """ + def tpool_listdir(partition_dir): + return dict(((suff, None) for suff in os.listdir(partition_dir) + if len(suff) == 3 and isdir(join(partition_dir, suff)))) + hashes_file = join(partition_dir, HASH_FILE) + with lock_path(partition_dir): + try: + with open(hashes_file, 'rb') as fp: + hashes = pickle.load(fp) + except Exception: + hashes = tpool.execute(tpool_listdir, partition_dir) + for suffix in suffixes: + suffix_dir = join(partition_dir, suffix) + if os.path.exists(suffix_dir): + hashes[suffix] = hash_suffix(suffix_dir, reclaim_age) + elif suffix in hashes: + del hashes[suffix] + with open(hashes_file + '.tmp', 'wb') as fp: + pickle.dump(hashes, fp, PICKLE_PROTOCOL) + renamer(hashes_file + '.tmp', hashes_file) + + +def invalidate_hash(suffix_dir): + """ + Invalidates the hash for a suffix_dir in the partition's hashes file. + + :param suffix_dir: absolute path to suffix dir whose hash needs invalidating + """ + suffix = os.path.basename(suffix_dir) + partition_dir = os.path.dirname(suffix_dir) + hashes_file = join(partition_dir, HASH_FILE) + with lock_path(partition_dir): + try: + with open(hashes_file, 'rb') as fp: + hashes = pickle.load(fp) + if suffix in hashes and not hashes[suffix]: + return + except Exception: + return + hashes[suffix] = None + with open(hashes_file + '.tmp', 'wb') as fp: + pickle.dump(hashes, fp, PICKLE_PROTOCOL) + renamer(hashes_file + '.tmp', hashes_file) + + +def get_hashes(partition_dir, do_listdir=True, reclaim_age=ONE_WEEK): + """ + Get a list of hashes for the suffix dir. do_listdir causes it to mistrust + the hash cache for suffix existence at the (unexpectedly high) cost of a + listdir. reclaim_age is just passed on to hash_suffix. + + :param partition_dir: absolute path of partition to get hashes for + :param do_listdir: force existence check for all hashes in the partition + :param reclaim_age: age at which to remove tombstones + + :returns: tuple of (number of suffix dirs hashed, dictionary of hashes) + """ + def tpool_listdir(hashes, partition_dir): + return dict(((suff, hashes.get(suff, None)) + for suff in os.listdir(partition_dir) + if len(suff) == 3 and isdir(join(partition_dir, suff)))) + hashed = 0 + hashes_file = join(partition_dir, HASH_FILE) + with lock_path(partition_dir): + modified = False + hashes = {} + try: + with open(hashes_file, 'rb') as fp: + hashes = pickle.load(fp) + except Exception: + do_listdir = True + if do_listdir: + hashes = tpool.execute(tpool_listdir, hashes, partition_dir) + modified = True + for suffix, hash_ in hashes.items(): + if not hash_: + suffix_dir = join(partition_dir, suffix) + if os.path.exists(suffix_dir): + try: + hashes[suffix] = hash_suffix(suffix_dir, reclaim_age) + hashed += 1 + except OSError: + logging.exception('Error hashing suffix') + hashes[suffix] = None + else: + del hashes[suffix] + modified = True + sleep() + if modified: + with open(hashes_file + '.tmp', 'wb') as fp: + pickle.dump(hashes, fp, PICKLE_PROTOCOL) + renamer(hashes_file + '.tmp', hashes_file) + return hashed, hashes + + +class ObjectReplicator(object): + """ + Replicate objects. + + Encapsulates most logic and data needed by the object replication process. + Each call to .run() performs one replication pass. It's up to the caller + to do this in a loop. + """ + + def __init__(self, conf, logger): + """ + :param conf: configuration object obtained from ConfigParser + :param logger: logging object + """ + self.conf = conf + self.logger = logger + self.devices_dir = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.vm_test_mode = conf.get( + 'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1') + self.swift_dir = conf.get('swift_dir', '/etc/swift') + self.port = int(conf.get('bind_port', 6000)) + self.concurrency = int(conf.get('replication_concurrency', 1)) + self.timeout = conf['timeout'] + self.stats_interval = int(conf['stats_interval']) + self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz')) + self.ring_check_interval = int(conf.get('ring_check_interval', 15)) + self.next_check = time.time() + self.ring_check_interval + self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) + self.partition_times = [] + + def _rsync(self, args): + """ + Execute the rsync binary to replicate a partition. + + :returns: a tuple of (rsync exit code, rsync standard output) + """ + start_time = time.time() + ret_val = None + try: + with Timeout(120): + proc = subprocess.Popen(args, stdout = subprocess.PIPE, + stderr = subprocess.STDOUT) + results = proc.stdout.read() + ret_val = proc.wait() + finally: + if ret_val is None: + proc.kill() + total_time = time.time() - start_time + if results: + for result in results.split('\n'): + if result == '': + continue + if result.startswith('cd+'): + continue + self.logger.info(result) + self.logger.info( + "Sync of %s at %s complete (%.03f) [%d]" % ( + args[-2], args[-1], total_time, ret_val)) + else: + self.logger.debug( + "Sync of %s at %s complete (%.03f) [%d]" % ( + args[-2], args[-1], total_time, ret_val)) + if ret_val: + self.logger.error('Bad rsync return code: %d' % ret_val) + return ret_val, results + + def rsync(self, node, job, suffixes): + """ + Synchronize local suffix directories from a partition with a remote + node. + + :param node: the "dev" entry for the remote node to sync with + :param job: information about the partition being synced + :param suffixes: a list of suffixes which need to be pushed + + :returns: boolean indicating success or failure + """ + if not os.path.exists(job['path']): + return False + args = [ + 'rsync', + '--recursive', + '--whole-file', + '--human-readable', + '--xattrs', + '--itemize-changes', + '--ignore-existing', + '--timeout=%s' % self.timeout, + '--contimeout=%s' % self.timeout, + ] + if self.vm_test_mode: + rsync_module = '%s::object%s' % (node['ip'], node['port']) + else: + rsync_module = '%s::object' % node['ip'] + had_any = False + for suffix in suffixes: + spath = join(job['path'], suffix) + if os.path.exists(spath): + args.append(spath) + had_any = True + if not had_any: + return False + args.append(join(rsync_module, node['device'], + 'objects', job['partition'])) + ret_val, results = self._rsync(args) + return ret_val == 0 + + def check_ring(self): + """ + Check to see if the ring has been updated + + :returns: boolean indicating whether or not the ring has changed + """ + if time.time() > self.next_check: + self.next_check = time.time() + self.ring_check_interval + if self.object_ring.has_changed(): + return False + return True + + def update_deleted(self, job): + """ + High-level method that replicates a single partition that doesn't belong + on this node. + + :param job: a dict containing info about the partition to be replicated + """ + def tpool_get_suffixes(path): + return [suff for suff in os.listdir(path) + if len(suff) == 3 and isdir(join(path, suff))] + self.replication_count += 1 + begin = time.time() + try: + responses = [] + suffixes = tpool.execute(tpool_get_suffixes, job['path']) + if suffixes: + for node in job['nodes']: + success = self.rsync(node, job, suffixes) + if success: + with Timeout(60): + http_connect(node['ip'], node['port'], + node['device'], job['partition'], 'REPLICATE', + '/' + '-'.join(suffixes), + headers={'Content-Length': '0'} + ).getresponse().read() + responses.append(success) + if not suffixes or (len(responses) == REPLICAS and all(responses)): + self.logger.info("Removing partition: %s" % job['path']) + tpool.execute(shutil.rmtree, job['path'], ignore_errors=True) + except (Exception, Timeout): + self.logger.exception("Error syncing handoff partition") + finally: + self.partition_times.append(time.time() - begin) + + def update(self, job): + """ + High-level method that replicates a single partition. + + :param job: a dict containing info about the partition to be replicated + """ + self.replication_count += 1 + begin = time.time() + try: + hashed, local_hash = get_hashes(job['path'], + do_listdir=(self.replication_count % 10) == 0, + reclaim_age=self.reclaim_age) + self.suffix_hash += hashed + successes = 0 + nodes = itertools.chain(job['nodes'], + self.object_ring.get_more_nodes(int(job['partition']))) + while successes < (REPLICAS - 1): + node = next(nodes) + try: + with Timeout(60): + resp = http_connect(node['ip'], node['port'], + node['device'], job['partition'], 'REPLICATE', + '', headers={'Content-Length': '0'} + ).getresponse() + if resp.status != 200: + self.logger.error("Invalid response %s from %s" % + (resp.status, node['ip'])) + continue + remote_hash = pickle.loads(resp.read()) + del resp + successes += 1 + suffixes = [suffix for suffix in local_hash + if local_hash[suffix] != remote_hash.get(suffix, -1)] + if not suffixes: + continue + success = self.rsync(node, job, suffixes) + recalculate_hashes(job['path'], suffixes, + reclaim_age=self.reclaim_age) + with Timeout(60): + http_connect(node['ip'], node['port'], + node['device'], job['partition'], 'REPLICATE', + '/' + '-'.join(suffixes), + headers={'Content-Length': '0'} + ).getresponse().read() + self.suffix_sync += len(suffixes) + except (Exception, Timeout): + logging.exception("Error syncing with node: %s" % node) + self.suffix_count += len(local_hash) + except (Exception, Timeout): + self.logger.exception("Error syncing partition") + finally: + self.partition_times.append(time.time() - begin) + + def stats_line(self): + """ + Logs various stats for the currently running replication pass. + """ + if self.replication_count: + rate = self.replication_count / (time.time() - self.start) + left = int((self.job_count - self.replication_count) / rate) + self.logger.info("%d/%d (%.2f%%) partitions replicated in %.2f seconds (%.2f/sec, %s remaining)" + % (self.replication_count, self.job_count, + self.replication_count * 100.0 / self.job_count, + time.time() - self.start, rate, + '%d%s' % compute_eta(self.start, self.replication_count, self.job_count))) + if self.suffix_count: + self.logger.info("%d suffixes checked - %.2f%% hashed, %.2f%% synced" % + (self.suffix_count, + (self.suffix_hash * 100.0) / self.suffix_count, + (self.suffix_sync * 100.0) / self.suffix_count)) + self.partition_times.sort() + self.logger.info("Partition times: max %.4fs, min %.4fs, med %.4fs" + % (self.partition_times[-1], self.partition_times[0], + self.partition_times[len(self.partition_times) // 2])) + else: + self.logger.info("Nothing replicated for %s seconds." % (time.time() - self.start)) + + def kill_coros(self): + """Utility function that kills all coroutines currently running.""" + for coro in list(self.run_pool.coroutines_running): + try: + coro.kill(GreenletExit) + except GreenletExit: + pass + + def heartbeat(self): + """ + Loop that runs in the background during replication. It periodically + logs progress and attempts to detect lockups, killing any running + coroutines if the replicator hasn't made progress since last hearbeat. + """ + while True: + if self.replication_count == self.last_replication_count: + self.logger.error("Lockup detected.. killing live coros.") + self.kill_coros() + self.last_replication_count = self.replication_count + eventlet.sleep(300) + self.stats_line() + + def run(self): + """Run a replication pass""" + self.start = time.time() + self.suffix_count = 0 + self.suffix_sync = 0 + self.suffix_hash = 0 + self.replication_count = 0 + self.last_replication_count = -1 + self.partition_times = [] + jobs = [] + stats = eventlet.spawn(self.heartbeat) + try: + ips = whataremyips() + self.run_pool = GreenPool(size=self.concurrency) + for local_dev in [ + dev for dev in self.object_ring.devs + if dev and dev['ip'] in ips and dev['port'] == self.port + ]: + dev_path = join(self.devices_dir, local_dev['device']) + obj_path = join(dev_path, 'objects') + tmp_path = join(dev_path, 'tmp') + if self.mount_check and not os.path.ismount(dev_path): + self.logger.warn('%s is not mounted' % local_dev['device']) + continue + unlink_older_than(tmp_path, time.time() - self.reclaim_age) + if not os.path.exists(obj_path): + continue + for partition in os.listdir(obj_path): + try: + nodes = [node for node in + self.object_ring.get_part_nodes(int(partition)) + if node['id'] != local_dev['id']] + jobs.append(dict(path=join(obj_path, partition), + nodes=nodes, delete=len(nodes) > 2, + partition=partition)) + except ValueError: + continue + random.shuffle(jobs) + # Partititons that need to be deleted take priority + jobs.sort(key=lambda job: not job['delete']) + self.job_count = len(jobs) + for job in jobs: + if not self.check_ring(): + self.logger.info( + "Ring change detected. Aborting current replication pass.") + return + if job['delete']: + self.run_pool.spawn(self.update_deleted, job) + else: + self.run_pool.spawn(self.update, job) + with Timeout(120): + self.run_pool.waitall() + except (Exception, Timeout): + self.logger.exception("Exception while replicating") + self.kill_coros() + self.stats_line() + stats.kill() diff --git a/swift/obj/server.py b/swift/obj/server.py new file mode 100644 index 0000000000..85a965724f --- /dev/null +++ b/swift/obj/server.py @@ -0,0 +1,599 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Object Server for Swift """ + +from __future__ import with_statement +import cPickle as pickle +import errno +import os +import socket +import time +import traceback +from datetime import datetime +from hashlib import md5 +from tempfile import mkstemp +from urllib import unquote +from contextlib import contextmanager + +from webob import Request, Response, UTC +from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ + HTTPInternalServerError, HTTPLengthRequired, HTTPNoContent, HTTPNotFound, \ + HTTPNotImplemented, HTTPNotModified, HTTPPreconditionFailed, \ + HTTPRequestTimeout, HTTPUnprocessableEntity, HTTPMethodNotAllowed +from xattr import getxattr, setxattr +from eventlet import sleep, Timeout + +from swift.common.exceptions import MessageTimeout +from swift.common.utils import mkdirs, normalize_timestamp, \ + storage_directory, hash_path, get_logger, renamer, fallocate, \ + split_path, drop_buffer_cache +from swift.common.healthcheck import healthcheck +from swift.common.bufferedhttp import http_connect +from swift.common.constraints import check_object_creation, check_mount, \ + check_float, check_xml_encodable +from swift.common.exceptions import ConnectionTimeout +from swift.obj.replicator import get_hashes, invalidate_hash, \ + recalculate_hashes + + +DATADIR = 'objects' +ASYNCDIR = 'async_pending' +PICKLE_PROTOCOL = 2 +METADATA_KEY = 'user.swift.metadata' +MAX_OBJECT_NAME_LENGTH = 1024 + + +def read_metadata(fd): + """ + Helper function to read the pickled metadata from an object file. + + :param fd: file descriptor to load the metadata from + + :returns: dictionary of metadata + """ + metadata = '' + key = 0 + try: + while True: + metadata += getxattr(fd, '%s%s' % (METADATA_KEY, (key or ''))) + key += 1 + except IOError: + pass + return pickle.loads(metadata) + + +class DiskFile(object): + """ + Manage object files on disk. + + :param path: path to devices on the node + :param device: device name + :param partition: partition on the device the object lives in + :param account: account name for the object + :param container: container name for the object + :param obj: object name for the object + :param keep_data_fp: if True, don't close the fp, otherwise close it + :param disk_chunk_Size: size of chunks on file reads + """ + def __init__(self, path, device, partition, account, container, obj, + keep_data_fp=False, disk_chunk_size=65536): + self.disk_chunk_size = disk_chunk_size + self.name = '/' + '/'.join((account, container, obj)) + name_hash = hash_path(account, container, obj) + self.datadir = os.path.join(path, device, + storage_directory(DATADIR, partition, name_hash)) + self.tmpdir = os.path.join(path, device, 'tmp') + self.metadata = {} + self.meta_file = None + self.data_file = None + if not os.path.exists(self.datadir): + return + files = sorted(os.listdir(self.datadir), reverse=True) + for file in files: + if file.endswith('.ts'): + self.data_file = self.meta_file = None + self.metadata = {'deleted': True} + return + if file.endswith('.meta') and not self.meta_file: + self.meta_file = os.path.join(self.datadir, file) + if file.endswith('.data') and not self.data_file: + self.data_file = os.path.join(self.datadir, file) + break + if not self.data_file: + return + self.fp = open(self.data_file, 'rb') + self.metadata = read_metadata(self.fp) + if not keep_data_fp: + self.close() + if self.meta_file: + with open(self.meta_file) as mfp: + for key in self.metadata.keys(): + if key.lower() not in ('content-type', 'content-encoding', + 'deleted', 'content-length', 'etag'): + del self.metadata[key] + self.metadata.update(read_metadata(mfp)) + + def __iter__(self): + """Returns an iterator over the data file.""" + try: + dropped_cache = 0 + read = 0 + while True: + chunk = self.fp.read(self.disk_chunk_size) + if chunk: + read += len(chunk) + if read - dropped_cache > (1024 * 1024): + drop_buffer_cache(self.fp.fileno(), dropped_cache, + read - dropped_cache) + dropped_cache = read + yield chunk + else: + drop_buffer_cache(self.fp.fileno(), dropped_cache, + read - dropped_cache) + break + finally: + self.close() + + def app_iter_range(self, start, stop): + """Returns an iterator over the data file for range (start, stop)""" + if start: + self.fp.seek(start) + if stop is not None: + length = stop - start + else: + length = None + for chunk in self: + if length is not None: + length -= len(chunk) + if length < 0: + # Chop off the extra: + yield chunk[:length] + break + yield chunk + + def close(self): + """Close the file.""" + if self.fp: + self.fp.close() + self.fp = None + + def is_deleted(self): + """ + Check if the file is deleted. + + :returns: True if the file doesn't exist or has been flagged as + deleted. + """ + return not self.data_file or 'deleted' in self.metadata + + @contextmanager + def mkstemp(self): + """Contextmanager to make a temporary file.""" + if not os.path.exists(self.tmpdir): + mkdirs(self.tmpdir) + fd, tmppath = mkstemp(dir=self.tmpdir) + try: + yield fd, tmppath + finally: + try: + os.close(fd) + except OSError: + pass + try: + os.unlink(tmppath) + except OSError: + pass + + def put(self, fd, tmppath, metadata, extension='.data'): + """ + Finalize writing the file on disk, and renames it from the temp file to + the real location. This should be called after the data has been + written to the temp file. + + :params fd: file descriptor of the temp file + :param tmppath: path to the temporary file being used + :param metadata: dictionary of metada to be written + :param extention: extension to be used when making the file + """ + metadata['name'] = self.name + timestamp = normalize_timestamp(metadata['X-Timestamp']) + metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) + key = 0 + while metastr: + setxattr(fd, '%s%s' % (METADATA_KEY, key or ''), metastr[:254]) + metastr = metastr[254:] + key += 1 + if 'Content-Length' in metadata: + drop_buffer_cache(fd, 0, int(metadata['Content-Length'])) + os.fsync(fd) + invalidate_hash(os.path.dirname(self.datadir)) + renamer(tmppath, os.path.join(self.datadir, timestamp + extension)) + self.metadata = metadata + + def unlinkold(self, timestamp): + """ + Remove any older versions of the object file. Any file that has an + older timestamp than timestamp will be deleted. + + :param timestamp: timestamp to compare with each file + """ + timestamp = normalize_timestamp(timestamp) + for fname in os.listdir(self.datadir): + if fname < timestamp: + try: + os.unlink(os.path.join(self.datadir, fname)) + except OSError, err: # pragma: no cover + if err.errno != errno.ENOENT: + raise + + +class ObjectController(object): + """Implements the WSGI application for the Swift Object Server.""" + + log_name = 'object' + + def __init__(self, conf): + """ + Creates a new WSGI application for the Swift Object Server. An + example configuration is given at + /etc/object-server.conf-sample or + /etc/swift/object-server.conf-sample. + """ + self.logger = get_logger(conf, self.log_name) + self.devices = conf.get('devices', '/srv/node/') + self.mount_check = conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + self.node_timeout = int(conf.get('node_timeout', 3)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) + self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) + self.log_requests = conf.get('log_requests', 't')[:1].lower() == 't' + self.max_upload_time = int(conf.get('max_upload_time', 86400)) + self.slow = int(conf.get('slow', 0)) + self.chunks_per_sync = int(conf.get('chunks_per_sync', 8000)) + + def container_update(self, op, account, container, obj, headers_in, + headers_out, objdevice): + """ + Update the container when objects are updated. + + :param op: operation performed (ex: 'PUT', or 'DELETE') + :param account: account name for the object + :param container: container name for the object + :param obj: object name + :param headers_in: dictionary of headers from the original request + :param headers_out: dictionary of headers to send in the container + request + :param objdevice: device name that the object is in + """ + host = headers_in.get('X-Container-Host', None) + partition = headers_in.get('X-Container-Partition', None) + contdevice = headers_in.get('X-Container-Device', None) + if not all([host, partition, contdevice]): + return + full_path = '/%s/%s/%s' % (account, container, obj) + try: + with ConnectionTimeout(self.conn_timeout): + ip, port = host.split(':') + conn = http_connect(ip, port, contdevice, partition, op, + full_path, headers_out) + with Timeout(self.node_timeout): + response = conn.getresponse() + response.read() + if 200 <= response.status < 300: + return + else: + self.logger.error('ERROR Container update failed (saving ' + 'for async update later): %d response from %s:%s/%s' % + (response.status, ip, port, contdevice)) + except: + self.logger.exception('ERROR container update failed with ' + '%s:%s/%s transaction %s (saving for async update later)' % + (ip, port, contdevice, headers_in.get('x-cf-trans-id', '-'))) + async_dir = os.path.join(self.devices, objdevice, ASYNCDIR) + fd, tmppath = mkstemp(dir=os.path.join(self.devices, objdevice, 'tmp')) + with os.fdopen(fd, 'wb') as fo: + pickle.dump({'op': op, 'account': account, 'container': container, + 'obj': obj, 'headers': headers_out}, fo) + fo.flush() + os.fsync(fd) + ohash = hash_path(account, container, obj) + renamer(tmppath, os.path.join(async_dir, ohash[-3:], ohash + '-' + + normalize_timestamp(headers_out['x-timestamp']))) + + def POST(self, request): + """Handle HTTP POST requests for the Swift Object Server.""" + try: + device, partition, account, container, obj = \ + split_path(unquote(request.path), 5, 5, True) + except ValueError, err: + return HTTPBadRequest(body=str(err), request=request, + content_type='text/plain') + if 'x-timestamp' not in request.headers or \ + not check_float(request.headers['x-timestamp']): + return HTTPBadRequest(body='Missing timestamp', request=request, + content_type='text/plain') + if self.mount_check and not check_mount(self.devices, device): + return Response(status='507 %s is not mounted' % device) + file = DiskFile(self.devices, device, partition, account, container, + obj, disk_chunk_size=self.disk_chunk_size) + deleted = file.is_deleted() + if file.is_deleted(): + response_class = HTTPNotFound + else: + response_class = HTTPAccepted + old_metadata = file.metadata + metadata = {'X-Timestamp': request.headers['x-timestamp']} + metadata.update(val for val in request.headers.iteritems() + if val[0].lower().startswith('x-object-meta-')) + with file.mkstemp() as (fd, tmppath): + file.put(fd, tmppath, metadata, extension='.meta') + return response_class(request=request) + + def PUT(self, request): + """Handle HTTP PUT requests for the Swift Object Server.""" + try: + device, partition, account, container, obj = \ + split_path(unquote(request.path), 5, 5, True) + except ValueError, err: + return HTTPBadRequest(body=str(err), request=request, + content_type='text/plain') + if self.mount_check and not check_mount(self.devices, device): + return Response(status='507 %s is not mounted' % device) + if 'x-timestamp' not in request.headers or \ + not check_float(request.headers['x-timestamp']): + return HTTPBadRequest(body='Missing timestamp', request=request, + content_type='text/plain') + error_response = check_object_creation(request, obj) + if error_response: + return error_response + file = DiskFile(self.devices, device, partition, account, container, + obj, disk_chunk_size=self.disk_chunk_size) + upload_expiration = time.time() + self.max_upload_time + etag = md5() + upload_size = 0 + with file.mkstemp() as (fd, tmppath): + if 'content-length' in request.headers: + fallocate(fd, int(request.headers['content-length'])) + chunk_count = 0 + dropped_cache = 0 + for chunk in iter(lambda: request.body_file.read( + self.network_chunk_size), ''): + upload_size += len(chunk) + if time.time() > upload_expiration: + return HTTPRequestTimeout(request=request) + etag.update(chunk) + while chunk: + written = os.write(fd, chunk) + chunk = chunk[written:] + chunk_count += 1 + # For large files sync every 512MB (by default) written + if chunk_count % self.chunks_per_sync == 0: + os.fdatasync(fd) + drop_buffer_cache(fd, dropped_cache, + upload_size - dropped_cache) + dropped_cache = upload_size + + if 'content-length' in request.headers and \ + int(request.headers['content-length']) != upload_size: + return Response(status='499 Client Disconnect') + etag = etag.hexdigest() + if 'etag' in request.headers and \ + request.headers['etag'].lower() != etag: + return HTTPUnprocessableEntity(request=request) + metadata = { + 'X-Timestamp': request.headers['x-timestamp'], + 'Content-Type': request.headers['content-type'], + 'ETag': etag, + 'Content-Length': str(os.fstat(fd).st_size), + } + metadata.update(val for val in request.headers.iteritems() + if val[0].lower().startswith('x-object-meta-') and + len(val[0]) > 14) + if 'content-encoding' in request.headers: + metadata['Content-Encoding'] = \ + request.headers['Content-Encoding'] + file.put(fd, tmppath, metadata) + file.unlinkold(metadata['X-Timestamp']) + self.container_update('PUT', account, container, obj, request.headers, + {'x-size': file.metadata['Content-Length'], + 'x-content-type': file.metadata['Content-Type'], + 'x-timestamp': file.metadata['X-Timestamp'], + 'x-etag': file.metadata['ETag'], + 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, + device) + resp = HTTPCreated(request=request, etag=etag) + return resp + + def GET(self, request): + """Handle HTTP GET requests for the Swift Object Server.""" + try: + device, partition, account, container, obj = \ + split_path(unquote(request.path), 5, 5, True) + except ValueError, err: + return HTTPBadRequest(body=str(err), request=request, + content_type='text/plain') + if self.mount_check and not check_mount(self.devices, device): + return Response(status='507 %s is not mounted' % device) + file = DiskFile(self.devices, device, partition, account, container, + obj, keep_data_fp=True, disk_chunk_size=self.disk_chunk_size) + if file.is_deleted(): + if request.headers.get('if-match') == '*': + return HTTPPreconditionFailed(request=request) + else: + return HTTPNotFound(request=request) + if request.headers.get('if-match') not in (None, '*') and \ + file.metadata['ETag'] not in request.if_match: + file.close() + return HTTPPreconditionFailed(request=request) + if request.headers.get('if-none-match') != None: + if file.metadata['ETag'] in request.if_none_match: + resp = HTTPNotModified(request=request) + resp.etag = file.metadata['ETag'] + file.close() + return resp + try: + if_unmodified_since = request.if_unmodified_since + except (OverflowError, ValueError): + # catches timestamps before the epoch + return HTTPPreconditionFailed(request=request) + if if_unmodified_since and \ + datetime.fromtimestamp(float(file.metadata['X-Timestamp']), UTC) > \ + if_unmodified_since: + file.close() + return HTTPPreconditionFailed(request=request) + try: + if_modified_since = request.if_modified_since + except (OverflowError, ValueError): + # catches timestamps before the epoch + return HTTPPreconditionFailed(request=request) + if if_modified_since and \ + datetime.fromtimestamp(float(file.metadata['X-Timestamp']), UTC) < \ + if_modified_since: + file.close() + return HTTPNotModified(request=request) + response = Response(content_type=file.metadata.get('Content-Type', + 'application/octet-stream'), app_iter=file, + request=request, conditional_response=True) + for key, value in file.metadata.iteritems(): + if key.lower().startswith('x-object-meta-'): + response.headers[key] = value + response.etag = file.metadata['ETag'] + response.last_modified = float(file.metadata['X-Timestamp']) + response.content_length = int(file.metadata['Content-Length']) + if 'Content-Encoding' in file.metadata: + response.content_encoding = file.metadata['Content-Encoding'] + return request.get_response(response) + + def HEAD(self, request): + """Handle HTTP HEAD requests for the Swift Object Server.""" + try: + device, partition, account, container, obj = \ + split_path(unquote(request.path), 5, 5, True) + except ValueError, err: + resp = HTTPBadRequest(request=request) + resp.content_type = 'text/plain' + resp.body = str(err) + return resp + if self.mount_check and not check_mount(self.devices, device): + return Response(status='507 %s is not mounted' % device) + file = DiskFile(self.devices, device, partition, account, container, + obj, disk_chunk_size=self.disk_chunk_size) + if file.is_deleted(): + return HTTPNotFound(request=request) + response = Response(content_type=file.metadata['Content-Type'], + request=request, conditional_response=True) + for key, value in file.metadata.iteritems(): + if key.lower().startswith('x-object-meta-'): + response.headers[key] = value + response.etag = file.metadata['ETag'] + response.last_modified = float(file.metadata['X-Timestamp']) + response.content_length = int(file.metadata['Content-Length']) + if 'Content-Encoding' in file.metadata: + response.content_encoding = file.metadata['Content-Encoding'] + return response + + def DELETE(self, request): + """Handle HTTP DELETE requests for the Swift Object Server.""" + try: + device, partition, account, container, obj = \ + split_path(unquote(request.path), 5, 5, True) + except ValueError, e: + return HTTPBadRequest(body=str(e), request=request, + content_type='text/plain') + if 'x-timestamp' not in request.headers or \ + not check_float(request.headers['x-timestamp']): + return HTTPBadRequest(body='Missing timestamp', request=request, + content_type='text/plain') + if self.mount_check and not check_mount(self.devices, device): + return Response(status='507 %s is not mounted' % device) + response_class = HTTPNoContent + file = DiskFile(self.devices, device, partition, account, container, + obj, disk_chunk_size=self.disk_chunk_size) + if file.is_deleted(): + response_class = HTTPNotFound + metadata = { + 'X-Timestamp': request.headers['X-Timestamp'], 'deleted': True, + } + with file.mkstemp() as (fd, tmppath): + file.put(fd, tmppath, metadata, extension='.ts') + file.unlinkold(metadata['X-Timestamp']) + self.container_update('DELETE', account, container, obj, + request.headers, {'x-timestamp': metadata['X-Timestamp'], + 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, + device) + resp = response_class(request=request) + return resp + + def REPLICATE(self, request): + """ + Handle REPLICATE requests for the Swift Object Server. This is used + by the object replicator to get hashes for directories. + """ + device, partition, suffix = split_path( + unquote(request.path), 2, 3, True) + if self.mount_check and not check_mount(self.devices, device): + return Response(status='507 %s is not mounted' % device) + if suffix: + recalculate_hashes(os.path.join(self.devices, device, + DATADIR, partition), suffix.split('-')) + return Response() + path = os.path.join(self.devices, device, DATADIR, partition) + if not os.path.exists(path): + mkdirs(path) + _, hashes = get_hashes(path, do_listdir=False) + return Response(body=pickle.dumps(hashes)) + + def __call__(self, env, start_response): + """WSGI Application entry point for the Swift Object Server.""" + start_time = time.time() + req = Request(env) + if req.path_info == '/healthcheck': + return healthcheck(req)(env, start_response) + elif not check_xml_encodable(req.path_info): + res = HTTPPreconditionFailed(body='Invalid UTF8') + else: + try: + if hasattr(self, req.method): + res = getattr(self, req.method)(req) + else: + res = HTTPMethodNotAllowed() + except: + self.logger.exception('ERROR __call__ error with %s %s ' + 'transaction %s' % (env.get('REQUEST_METHOD', '-'), + env.get('PATH_INFO', '-'), env.get('HTTP_X_CF_TRANS_ID', + '-'))) + res = HTTPInternalServerError(body=traceback.format_exc()) + trans_time = time.time() - start_time + if self.log_requests: + log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( + req.remote_addr, + time.strftime('%d/%b/%Y:%H:%M:%S +0000', + time.gmtime()), + req.method, req.path, res.status.split()[0], + res.content_length or '-', req.referer or '-', + req.headers.get('x-cf-trans-id', '-'), + req.user_agent or '-', + trans_time) + if req.method == 'REPLICATE': + self.logger.debug(log_line) + else: + self.logger.info(log_line) + if req.method in ('PUT', 'DELETE'): + slow = self.slow - trans_time + if slow > 0: + sleep(slow) + return res(env, start_response) diff --git a/swift/obj/updater.py b/swift/obj/updater.py new file mode 100644 index 0000000000..3609c1ca95 --- /dev/null +++ b/swift/obj/updater.py @@ -0,0 +1,197 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cPickle as pickle +import errno +import logging +import os +import signal +import sys +import time +from random import random + +from eventlet import patcher, Timeout + +from swift.common.bufferedhttp import http_connect +from swift.common.exceptions import ConnectionTimeout +from swift.common.ring import Ring +from swift.common.utils import get_logger, renamer +from swift.common.db_replicator import ReplConnection +from swift.obj.server import ASYNCDIR + + +class ObjectUpdater(object): + """Update object information in container listings.""" + + def __init__(self, server_conf, updater_conf): + self.logger = get_logger(updater_conf, 'object-updater') + self.devices = server_conf.get('devices', '/srv/node') + self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + ('true', 't', '1', 'on', 'yes', 'y') + swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(updater_conf.get('interval', 300)) + self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') + self.container_ring = None + self.concurrency = int(updater_conf.get('concurrency', 1)) + self.slowdown = float(updater_conf.get('slowdown', 0.01)) + self.node_timeout = int(updater_conf.get('node_timeout', 10)) + self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5)) + self.successes = 0 + self.failures = 0 + + def get_container_ring(self): + """Get the container ring. Load it, if it hasn't been yet.""" + if not self.container_ring: + self.logger.debug( + 'Loading container ring from %s' % self.container_ring_path) + self.container_ring = Ring(self.container_ring_path) + return self.container_ring + + def update_forever(self): # pragma: no cover + """Run the updater continuously.""" + time.sleep(random() * self.interval) + while True: + self.logger.info('Begin object update sweep') + begin = time.time() + pids = [] + # read from container ring to ensure it's fresh + self.get_container_ring().get_nodes('') + for device in os.listdir(self.devices): + if self.mount_check and not \ + os.path.ismount(os.path.join(self.devices, device)): + self.logger.warn( + 'Skipping %s as it is not mounted' % device) + continue + while len(pids) >= self.concurrency: + pids.remove(os.wait()[0]) + pid = os.fork() + if pid: + pids.append(pid) + else: + signal.signal(signal.SIGTERM, signal.SIG_DFL) + patcher.monkey_patch(all=False, socket=True) + self.successes = 0 + self.failures = 0 + forkbegin = time.time() + self.object_sweep(os.path.join(self.devices, device)) + elapsed = time.time() - forkbegin + self.logger.info('Object update sweep of %s completed: ' + '%.02fs, %s successes, %s failures' % + (device, elapsed, self.successes, self.failures)) + sys.exit() + while pids: + pids.remove(os.wait()[0]) + elapsed = time.time() - begin + self.logger.info('Object update sweep completed: %.02fs' % elapsed) + if elapsed < self.interval: + time.sleep(self.interval - elapsed) + + def update_once_single_threaded(self): + """Run the updater once""" + self.logger.info('Begin object update single threaded sweep') + begin = time.time() + self.successes = 0 + self.failures = 0 + for device in os.listdir(self.devices): + if self.mount_check and \ + not os.path.ismount(os.path.join(self.devices, device)): + self.logger.warn( + 'Skipping %s as it is not mounted' % device) + continue + self.object_sweep(os.path.join(self.devices, device)) + elapsed = time.time() - begin + self.logger.info('Object update single threaded sweep completed: ' + '%.02fs, %s successes, %s failures' % + (elapsed, self.successes, self.failures)) + + def object_sweep(self, device): + """ + If there are async pendings on the device, walk each one and update. + + :param device: path to device + """ + async_pending = os.path.join(device, ASYNCDIR) + if not os.path.isdir(async_pending): + return + for prefix in os.listdir(async_pending): + prefix_path = os.path.join(async_pending, prefix) + if not os.path.isdir(prefix_path): + continue + for update in os.listdir(prefix_path): + update_path = os.path.join(prefix_path, update) + if not os.path.isfile(update_path): + continue + self.process_object_update(update_path, device) + time.sleep(self.slowdown) + try: + os.rmdir(prefix_path) + except OSError: + pass + + def process_object_update(self, update_path, device): + """ + Process the object information to be updated and update. + + :param update_path: path to pickled object update file + :param device: path to device + """ + try: + update = pickle.load(open(update_path, 'rb')) + except Exception, err: + self.logger.exception( + 'ERROR Pickle problem, quarantining %s' % update_path) + renamer(update_path, os.path.join(device, + 'quarantined', 'objects', os.path.basename(update_path))) + return + part, nodes = self.get_container_ring().get_nodes( + update['account'], update['container']) + obj = '/%s/%s/%s' % \ + (update['account'], update['container'], update['obj']) + success = True + for node in nodes: + status = self.object_update(node, part, update['op'], obj, + update['headers']) + if not (200 <= status < 300) and status != 404: + success = False + if success: + self.successes += 1 + self.logger.debug('Update sent for %s %s' % (obj, update_path)) + os.unlink(update_path) + else: + self.failures += 1 + self.logger.debug('Update failed for %s %s' % (obj, update_path)) + + def object_update(self, node, part, op, obj, headers): + """ + Perform the object update to the container + + :param node: node dictionary from the container ring + :param part: partition that holds the container + :param op: operation performed (ex: 'POST' or 'DELETE') + :param obj: object name being updated + :param headers: headers to send with the update + """ + try: + with ConnectionTimeout(self.conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], + part, op, obj, headers) + with Timeout(self.node_timeout): + resp = conn.getresponse() + resp.read() + return resp.status + except: + self.logger.exception('ERROR with remote server ' + '%(ip)s:%(port)s/%(device)s' % node) + return 500 diff --git a/swift/proxy/__init__.py b/swift/proxy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/swift/proxy/server.py b/swift/proxy/server.py new file mode 100644 index 0000000000..a0804826ad --- /dev/null +++ b/swift/proxy/server.py @@ -0,0 +1,1187 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import errno +import mimetypes +import os +import socket +import time +from ConfigParser import ConfigParser, NoOptionError +from urllib import unquote, quote +import uuid +import sys +import functools + +from eventlet.timeout import Timeout +from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ + HTTPCreated, HTTPLengthRequired, HTTPMethodNotAllowed, HTTPNoContent, \ + HTTPNotFound, HTTPNotModified, HTTPPreconditionFailed, \ + HTTPRequestTimeout, HTTPServiceUnavailable, HTTPUnauthorized, \ + HTTPUnprocessableEntity, HTTPRequestEntityTooLarge, status_map +from webob import Request, Response + +from swift.common.ring import Ring +from swift.common.utils import get_logger, normalize_timestamp, split_path +from swift.common.bufferedhttp import http_connect +from swift.common.healthcheck import HealthCheckController +from swift.common.constraints import check_object_creation, check_metadata, \ + MAX_FILE_SIZE, check_xml_encodable +from swift.common.exceptions import ChunkReadTimeout, \ + ChunkWriteTimeout, ConnectionTimeout + +MAX_CONTAINER_NAME_LENGTH = 256 + + +def update_headers(response, headers): + """ + Helper function to update headers in the response. + + :param response: webob.Response object + :param headers: dictionary headers + """ + if hasattr(headers, 'items'): + headers = headers.items() + for name, value in headers: + if name == 'etag': + response.headers[name] = value.replace('"', '') + elif name not in ('date', 'content-length', 'content-type', + 'connection', 'x-timestamp', 'x-put-timestamp'): + response.headers[name] = value + + +def public(func): + """ + Decorator to declare which methods are public accessible as HTTP requests + + :param func: function to make public + """ + func.publicly_accessible = True + + @functools.wraps(func) + def wrapped(*a, **kw): + return func(*a, **kw) + return wrapped + + +class Controller(object): + """Base WSGI controller class for the proxy""" + + def __init__(self, app): + self.account_name = None + self.app = app + self.trans_id = '-' + + def error_increment(self, node): + """ + Handles incrementing error counts when talking to nodes. + + :param node: dictionary of node to increment the error count for + """ + node['errors'] = node.get('errors', 0) + 1 + node['last_error'] = time.time() + + def error_occurred(self, node, msg): + """ + Handle logging, and handling of errors. + + :param node: dictionary of node to handle errors for + :param msg: error message + """ + self.error_increment(node) + self.app.logger.error( + '%s %s:%s' % (msg, node['ip'], node['port'])) + + def exception_occurred(self, node, typ, additional_info): + """ + Handle logging of generic exceptions. + + :param node: dictionary of node to log the error for + :param typ: server type + :param additional_info: additional information to log + """ + self.app.logger.exception( + 'ERROR with %s server %s:%s/%s transaction %s re: %s' % (typ, + node['ip'], node['port'], node['device'], self.trans_id, + additional_info)) + + def error_limited(self, node): + """ + Check if the node is currently error limited. + + :param node: dictionary of node to check + :returns: True if error limited, False otherwise + """ + now = time.time() + if not 'errors' in node: + return False + if 'last_error' in node and node['last_error'] < \ + now - self.app.error_suppression_interval: + del node['last_error'] + if 'errors' in node: + del node['errors'] + return False + limited = node['errors'] > self.app.error_suppression_limit + if limited: + self.app.logger.debug( + 'Node error limited %s:%s (%s)' % ( + node['ip'], node['port'], node['device'])) + return limited + + def error_limit(self, node): + """ + Mark a node as error limited. + + :param node: dictionary of node to error limit + """ + node['errors'] = self.app.error_suppression_limit + 1 + node['last_error'] = time.time() + + def account_info(self, account): + """ + Get account information, and also verify that the account exists. + + :param account: name of the account to get the info for + :returns: tuple of (account partition, account nodes) or (None, None) + if it does not exist + """ + partition, nodes = self.app.account_ring.get_nodes(account) + path = '/%s' % account + cache_key = 'account%s' % path + # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses + if self.app.memcache.get(cache_key): + return partition, nodes + result_code = 0 + attempts_left = self.app.account_ring.replica_count + headers = {'x-cf-trans-id': self.trans_id} + for node in self.iter_nodes(partition, nodes, self.app.account_ring): + if self.error_limited(node): + continue + try: + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], partition, 'HEAD', path, headers) + with Timeout(self.app.node_timeout): + resp = conn.getresponse() + body = resp.read() + if 200 <= resp.status <= 299: + result_code = 200 + break + elif resp.status == 404: + result_code = 404 if not result_code else -1 + elif resp.status == 507: + self.error_limit(node) + continue + else: + result_code = -1 + attempts_left -= 1 + if attempts_left <= 0: + break + except: + self.exception_occurred(node, 'Account', + 'Trying to get account info for %s' % path) + if result_code == 200: + cache_timeout = self.app.recheck_account_existence + else: + cache_timeout = self.app.recheck_account_existence * 0.1 + self.app.memcache.set(cache_key, result_code, timeout=cache_timeout) + if result_code == 200: + return partition, nodes + return (None, None) + + def container_info(self, account, container): + """ + Get container information and thusly verify container existance. + This will also make a call to account_info to verify that the + account exists. + + :param account: account name for the container + :param container: container name to look up + :returns: tuple of (container partition, container nodes) or + (None, None) if the container does not exist + """ + partition, nodes = self.app.container_ring.get_nodes( + account, container) + path = '/%s/%s' % (account, container) + cache_key = 'container%s' % path + # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses + if self.app.memcache.get(cache_key) == 200: + return partition, nodes + if not self.account_info(account)[1]: + return (None, None) + result_code = 0 + attempts_left = self.app.container_ring.replica_count + headers = {'x-cf-trans-id': self.trans_id} + for node in self.iter_nodes(partition, nodes, self.app.container_ring): + if self.error_limited(node): + continue + try: + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], partition, 'HEAD', path, headers) + with Timeout(self.app.node_timeout): + resp = conn.getresponse() + body = resp.read() + if 200 <= resp.status <= 299: + result_code = 200 + break + elif resp.status == 404: + result_code = 404 if not result_code else -1 + elif resp.status == 507: + self.error_limit(node) + continue + else: + result_code = -1 + attempts_left -= 1 + if attempts_left <= 0: + break + except: + self.exception_occurred(node, 'Container', + 'Trying to get container info for %s' % path) + if result_code == 200: + cache_timeout = self.app.recheck_container_existence + else: + cache_timeout = self.app.recheck_container_existence * 0.1 + self.app.memcache.set(cache_key, result_code, timeout=cache_timeout) + if result_code == 200: + return partition, nodes + return (None, None) + + def iter_nodes(self, partition, nodes, ring): + """ + Node iterator that will first iterate over the normal nodes for a + partition and then the handoff partitions for the node. + + :param partition: partition to iterate nodes for + :param nodes: list of node dicts from the ring + :param ring: ring to get handoff nodes from + """ + for node in nodes: + yield node + for node in ring.get_more_nodes(partition): + yield node + + def get_update_nodes(self, partition, nodes, ring): + """ Returns ring.replica_count nodes; the nodes will not be error + limited, if possible. """ + """ + Attempt to get a non error limited list of nodes. + + :param partition: partition for the nodes + :param nodes: list of node dicts for the partition + :param ring: ring to get handoff nodes from + :returns: list of node dicts that are not error limited (if possible) + """ + + # make a copy so we don't modify caller's list + nodes = list(nodes) + update_nodes = [] + for node in self.iter_nodes(partition, nodes, ring): + if self.error_limited(node): + continue + update_nodes.append(node) + if len(update_nodes) >= ring.replica_count: + break + while len(update_nodes) < ring.replica_count: + node = nodes.pop() + if node not in update_nodes: + update_nodes.append(node) + return update_nodes + + def best_response(self, req, statuses, reasons, bodies, server_type, + etag=None): + """ + Given a list of responses from several servers, choose the best to + return to the API. + + :param req: webob.Request object + :param statuses: list of statuses returned + :param reasons: list of reasons for each status + :param bodies: bodies of each response + :param server_type: type of server the responses came from + :param etag: etag + :returns: webob.Response object with the correct status, body, etc. set + """ + resp = Response(request=req) + if len(statuses): + for hundred in (200, 300, 400): + hstatuses = \ + [s for s in statuses if hundred <= s < hundred + 100] + if len(hstatuses) > len(statuses) / 2: + status = max(hstatuses) + status_index = statuses.index(status) + resp.status = '%s %s' % (status, reasons[status_index]) + resp.body = bodies[status_index] + resp.content_type = 'text/plain' + if etag: + resp.headers['etag'] = etag.strip('"') + return resp + self.app.logger.error('%s returning 503 for %s, transaction %s' % + (server_type, statuses, self.trans_id)) + resp.status = '503 Internal Server Error' + return resp + + @public + def GET(self, req): + """Handler for HTTP GET requests.""" + return self.GETorHEAD(req) + + @public + def HEAD(self, req): + """Handler for HTTP HEAD requests.""" + return self.GETorHEAD(req) + + def GETorHEAD_base(self, req, server_type, partition, nodes, path, + attempts): + """ + Base handler for HTTP GET or HEAD requests. + + :param req: webob.Request object + :param server_type: server type + :param partition: partition + :param nodes: nodes + :param path: path for the request + :param attempts: number of attempts to try + :returns: webob.Response object + """ + statuses = [] + reasons = [] + bodies = [] + for node in nodes: + if len(statuses) >= attempts: + break + if self.error_limited(node): + continue + try: + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], partition, req.method, path, + headers=req.headers, + query_string=req.query_string) + with Timeout(self.app.node_timeout): + source = conn.getresponse() + except: + self.exception_occurred(node, server_type, + 'Trying to %s %s' % (req.method, req.path)) + continue + if source.status == 507: + self.error_limit(node) + continue + if 200 <= source.status <= 399: + # 404 if we know we don't have a synced copy + if not float(source.getheader('X-PUT-Timestamp', '1')): + statuses.append(404) + reasons.append('') + bodies.append('') + source.read() + continue + if req.method == 'GET' and source.status in (200, 206): + + def file_iter(): + try: + while True: + with ChunkReadTimeout(self.app.node_timeout): + chunk = source.read(self.app.object_chunk_size) + if not chunk: + break + yield chunk + req.sent_size += len(chunk) + except GeneratorExit: + req.client_disconnect = True + self.app.logger.info( + 'Client disconnected on read transaction %s' % + self.trans_id) + except: + self.exception_occurred(node, 'Object', + 'Trying to read during GET of %s' % req.path) + raise + + res = Response(app_iter=file_iter(), request=req, + conditional_response=True) + update_headers(res, source.getheaders()) + res.status = source.status + res.content_length = source.getheader('Content-Length') + if source.getheader('Content-Type'): + res.charset = None + res.content_type = source.getheader('Content-Type') + return res + elif 200 <= source.status <= 399: + res = status_map[source.status](request=req) + update_headers(res, source.getheaders()) + if req.method == 'HEAD': + res.content_length = source.getheader('Content-Length') + if source.getheader('Content-Type'): + res.charset = None + res.content_type = source.getheader('Content-Type') + return res + statuses.append(source.status) + reasons.append(source.reason) + bodies.append(source.read()) + if source.status >= 500: + self.error_occurred(node, 'ERROR %d %s From %s Server' % + (source.status, bodies[-1][:1024], server_type)) + return self.best_response(req, statuses, reasons, bodies, + '%s %s' % (server_type, req.method)) + + +class ObjectController(Controller): + """WSGI controller for object requests.""" + + def __init__(self, app, account_name, container_name, object_name, + **kwargs): + Controller.__init__(self, app) + self.account_name = unquote(account_name) + self.container_name = unquote(container_name) + self.object_name = unquote(object_name) + + def node_post_or_delete(self, req, partition, node, path): + """ + Handle common POST/DELETE functionality + + :param req: webob.Request object + :param partition: partition for the object + :param node: node dictionary for the object + :param path: path to send for the request + """ + if self.error_limited(node): + return 500, '', '' + try: + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], + partition, req.method, path, req.headers) + with Timeout(self.app.node_timeout): + response = conn.getresponse() + body = response.read() + if response.status == 507: + self.error_limit(node) + elif response.status >= 500: + self.error_occurred(node, + 'ERROR %d %s From Object Server' % + (response.status, body[:1024])) + return response.status, response.reason, body + except: + self.exception_occurred(node, 'Object', + 'Trying to %s %s' % (req.method, req.path)) + return 500, '', '' + + def GETorHEAD(self, req): + """Handle HTTP GET or HEAD requests.""" + partition, nodes = self.app.object_ring.get_nodes( + self.account_name, self.container_name, self.object_name) + return self.GETorHEAD_base(req, 'Object', partition, + self.iter_nodes(partition, nodes, self.app.object_ring), + req.path_info, self.app.object_ring.replica_count) + + @public + def POST(self, req): + """HTTP POST request handler.""" + error_response = check_metadata(req) + if error_response: + return error_response + container_partition, containers = \ + self.container_info(self.account_name, self.container_name) + if not containers: + return HTTPNotFound(request=req) + containers = self.get_update_nodes(container_partition, containers, + self.app.container_ring) + partition, nodes = self.app.object_ring.get_nodes( + self.account_name, self.container_name, self.object_name) + req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + statuses = [] + reasons = [] + bodies = [] + for node in self.iter_nodes(partition, nodes, self.app.object_ring): + container = containers.pop() + req.headers['X-Container-Host'] = '%(ip)s:%(port)s' % container + req.headers['X-Container-Partition'] = container_partition + req.headers['X-Container-Device'] = container['device'] + status, reason, body = \ + self.node_post_or_delete(req, partition, node, req.path_info) + if 200 <= status < 300 or 400 <= status < 500: + statuses.append(status) + reasons.append(reason) + bodies.append(body) + else: + containers.insert(0, container) + if not containers: + break + while len(statuses) < len(nodes): + statuses.append(503) + reasons.append('') + bodies.append('') + return self.best_response(req, statuses, reasons, + bodies, 'Object POST') + + @public + def PUT(self, req): + """HTTP PUT request handler.""" + container_partition, containers = \ + self.container_info(self.account_name, self.container_name) + if not containers: + return HTTPNotFound(request=req) + containers = self.get_update_nodes(container_partition, containers, + self.app.container_ring) + partition, nodes = self.app.object_ring.get_nodes( + self.account_name, self.container_name, self.object_name) + req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + # this is a temporary hook for migrations to set PUT timestamps + if '!Migration-Timestamp!' in req.headers: + req.headers['X-Timestamp'] = \ + normalize_timestamp(req.headers['!Migration-Timestamp!']) + # Sometimes the 'content-type' header exists, but is set to None. + if not req.headers.get('content-type'): + guessed_type, _ = mimetypes.guess_type(req.path_info) + if not guessed_type: + req.headers['Content-Type'] = 'application/octet-stream' + else: + req.headers['Content-Type'] = guessed_type + error_response = check_object_creation(req, self.object_name) + if error_response: + return error_response + conns = [] + data_source = \ + iter(lambda: req.body_file.read(self.app.client_chunk_size), '') + source_header = req.headers.get('X-Copy-From') + if source_header: + source_header = unquote(source_header) + acct = req.path_info.split('/', 2)[1] + if not source_header.startswith('/'): + source_header = '/' + source_header + source_header = '/' + acct + source_header + try: + src_container_name, src_obj_name = \ + source_header.split('/',3)[2:] + except ValueError: + return HTTPPreconditionFailed(request=req, + body='X-Copy-From header must be of the form' + 'container/object') + source_req = Request.blank(source_header) + orig_obj_name = self.object_name + orig_container_name = self.container_name + self.object_name = src_obj_name + self.container_name = src_container_name + source_resp = self.GET(source_req) + if source_resp.status_int >= 300: + return source_resp + self.object_name = orig_obj_name + self.container_name = orig_container_name + data_source = source_resp.app_iter + new_req = Request.blank(req.path_info, + environ=req.environ, headers=req.headers) + new_req.content_length = source_resp.content_length + new_req.etag = source_resp.etag + new_req.headers['X-Copy-From'] = source_header.split('/', 2)[2] + for k, v in source_resp.headers.items(): + if k.lower().startswith('x-object-meta-'): + new_req.headers[k] = v + for k, v in req.headers.items(): + if k.lower().startswith('x-object-meta-'): + new_req.headers[k] = v + req = new_req + for node in self.iter_nodes(partition, nodes, self.app.object_ring): + container = containers.pop() + req.headers['X-Container-Host'] = '%(ip)s:%(port)s' % container + req.headers['X-Container-Partition'] = container_partition + req.headers['X-Container-Device'] = container['device'] + req.headers['Expect'] = '100-continue' + resp = conn = None + if not self.error_limited(node): + try: + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], partition, 'PUT', + req.path_info, req.headers) + conn.node = node + with Timeout(self.app.node_timeout): + resp = conn.getexpect() + except: + self.exception_occurred(node, 'Object', + 'Expect: 100-continue on %s' % req.path) + if conn and resp: + if resp.status == 100: + conns.append(conn) + if not containers: + break + continue + elif resp.status == 507: + self.error_limit(node) + containers.insert(0, container) + if len(conns) <= len(nodes) / 2: + self.app.logger.error( + 'Object PUT returning 503, %s/%s required connections, ' + 'transaction %s' % + (len(conns), len(nodes) / 2 + 1, self.trans_id)) + return HTTPServiceUnavailable(request=req) + try: + req.creation_size = 0 + while True: + with ChunkReadTimeout(self.app.client_timeout): + try: + chunk = data_source.next() + except StopIteration: + if req.headers.get('transfer-encoding'): + chunk = '' + else: + break + len_chunk = len(chunk) + req.creation_size += len_chunk + if req.creation_size > MAX_FILE_SIZE: + req.creation_size = 0 + return HTTPRequestEntityTooLarge(request=req) + for conn in conns: + try: + with ChunkWriteTimeout(self.app.node_timeout): + if req.headers.get('transfer-encoding'): + conn.send('%x\r\n%s\r\n' % (len_chunk, chunk)) + else: + conn.send(chunk) + except: + self.exception_occurred(conn.node, 'Object', + 'Trying to write to %s' % req.path) + conns.remove(conn) + if req.headers.get('transfer-encoding') and chunk == '': + break + except ChunkReadTimeout, err: + self.app.logger.info( + 'ERROR Client read timeout (%ss)' % err.seconds) + return HTTPRequestTimeout(request=req) + except: + self.app.logger.exception( + 'ERROR Exception causing client disconnect') + return Response(status='499 Client Disconnect') + if req.content_length and req.creation_size < req.content_length: + self.app.logger.info( + 'Client disconnected without sending enough data %s' % + self.trans_id) + return Response(status='499 Client Disconnect') + statuses = [] + reasons = [] + bodies = [] + etags = set() + for conn in conns: + try: + with Timeout(self.app.node_timeout): + response = conn.getresponse() + statuses.append(response.status) + reasons.append(response.reason) + bodies.append(response.read()) + if response.status >= 500: + self.error_occurred(conn.node, + 'ERROR %d %s From Object Server re: %s' % + (response.status, bodies[-1][:1024], req.path)) + elif 200 <= response.status < 300: + etags.add(response.getheader('etag').strip('"')) + except: + self.exception_occurred(conn.node, 'Object', + 'Trying to get final status of PUT to %s' % req.path) + if len(etags) > 1: + return HTTPUnprocessableEntity(request=req) + etag = len(etags) and etags.pop() or None + while len(statuses) < len(nodes): + statuses.append(503) + reasons.append('') + bodies.append('') + resp = self.best_response(req, statuses, reasons, bodies, 'Object PUT', + etag=etag) + if 'x-copy-from' in req.headers: + resp.headers['X-Copied-From'] = req.headers['x-copy-from'] + for k, v in req.headers.items(): + if k.lower().startswith('x-object-meta-'): + resp.headers[k] = v + resp.last_modified = float(req.headers['X-Timestamp']) + return resp + + @public + def DELETE(self, req): + """HTTP DELETE request handler.""" + container_partition, containers = \ + self.container_info(self.account_name, self.container_name) + if not containers: + return HTTPNotFound(request=req) + containers = self.get_update_nodes(container_partition, containers, + self.app.container_ring) + partition, nodes = self.app.object_ring.get_nodes( + self.account_name, self.container_name, self.object_name) + req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + statuses = [] + reasons = [] + bodies = [] + for node in self.iter_nodes(partition, nodes, self.app.object_ring): + container = containers.pop() + req.headers['X-Container-Host'] = '%(ip)s:%(port)s' % container + req.headers['X-Container-Partition'] = container_partition + req.headers['X-Container-Device'] = container['device'] + status, reason, body = \ + self.node_post_or_delete(req, partition, node, req.path_info) + if 200 <= status < 300 or 400 <= status < 500: + statuses.append(status) + reasons.append(reason) + bodies.append(body) + else: + containers.insert(0, container) + if not containers: + break + while len(statuses) < len(nodes): + statuses.append(503) + reasons.append('') + bodies.append('') + return self.best_response(req, statuses, reasons, bodies, + 'Object DELETE') + + @public + def COPY(self, req): + """HTTP COPY request handler.""" + dest = req.headers.get('Destination') + if not dest: + return HTTPPreconditionFailed(request=req, + body='Destination header required') + dest = unquote(dest) + if not dest.startswith('/'): + dest = '/' + dest + try: + _, dest_container, dest_object = dest.split('/', 3) + except ValueError: + return HTTPPreconditionFailed(request=req, + body='Destination header must be of the form container/object') + new_source = '/' + self.container_name + '/' + self.object_name + self.container_name = dest_container + self.object_name = dest_object + new_headers = {} + for k, v in req.headers.items(): + new_headers[k] = v + new_headers['X-Copy-From'] = new_source + new_headers['Content-Length'] = 0 + del new_headers['Destination'] + new_path = '/' + self.account_name + dest + new_req = Request.blank(new_path, + environ={'REQUEST_METHOD': 'PUT'}, headers=new_headers) + return self.PUT(new_req) + + +class ContainerController(Controller): + """WSGI controller for container requests""" + + def __init__(self, app, account_name, container_name, **kwargs): + Controller.__init__(self, app) + self.account_name = unquote(account_name) + self.container_name = unquote(container_name) + + def GETorHEAD(self, req): + """Handler for HTTP GET/HEAD requests.""" + if not self.account_info(self.account_name)[1]: + return HTTPNotFound(request=req) + part, nodes = self.app.container_ring.get_nodes( + self.account_name, self.container_name) + resp = self.GETorHEAD_base(req, 'Container', part, nodes, + req.path_info, self.app.container_ring.replica_count) + return resp + + @public + def PUT(self, req): + """HTTP PUT request handler.""" + if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH: + resp = HTTPBadRequest(request=req) + resp.body = 'Container name length of %d longer than %d' % \ + (len(self.container_name), MAX_CONTAINER_NAME_LENGTH) + return resp + account_partition, accounts = self.account_info(self.account_name) + if not accounts: + return HTTPNotFound(request=req) + accounts = self.get_update_nodes(account_partition, accounts, + self.app.account_ring) + container_partition, containers = self.app.container_ring.get_nodes( + self.account_name, self.container_name) + headers = {'X-Timestamp': normalize_timestamp(time.time()), + 'x-cf-trans-id': self.trans_id} + statuses = [] + reasons = [] + bodies = [] + for node in self.iter_nodes(container_partition, containers, + self.app.container_ring): + if self.error_limited(node): + continue + try: + account = accounts.pop() + headers['X-Account-Host'] = '%(ip)s:%(port)s' % account + headers['X-Account-Partition'] = account_partition + headers['X-Account-Device'] = account['device'] + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], container_partition, 'PUT', + req.path_info, headers) + with Timeout(self.app.node_timeout): + source = conn.getresponse() + body = source.read() + if 200 <= source.status < 300 \ + or 400 <= source.status < 500: + statuses.append(source.status) + reasons.append(source.reason) + bodies.append(body) + else: + if source.status == 507: + self.error_limit(node) + accounts.insert(0, account) + except: + accounts.insert(0, account) + self.exception_occurred(node, 'Container', + 'Trying to PUT to %s' % req.path) + if not accounts: + break + while len(statuses) < len(containers): + statuses.append(503) + reasons.append('') + bodies.append('') + self.app.memcache.delete('container%s' % req.path_info.rstrip('/')) + return self.best_response(req, statuses, reasons, bodies, + 'Container PUT') + + @public + def DELETE(self, req): + """HTTP DELETE request handler.""" + account_partition, accounts = self.account_info(self.account_name) + if not accounts: + return HTTPNotFound(request=req) + accounts = self.get_update_nodes(account_partition, accounts, + self.app.account_ring) + container_partition, containers = self.app.container_ring.get_nodes( + self.account_name, self.container_name) + headers = {'X-Timestamp': normalize_timestamp(time.time()), + 'x-cf-trans-id': self.trans_id} + statuses = [] + reasons = [] + bodies = [] + for node in self.iter_nodes(container_partition, containers, + self.app.container_ring): + if self.error_limited(node): + continue + try: + account = accounts.pop() + headers['X-Account-Host'] = '%(ip)s:%(port)s' % account + headers['X-Account-Partition'] = account_partition + headers['X-Account-Device'] = account['device'] + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], container_partition, 'DELETE', + req.path_info, headers) + with Timeout(self.app.node_timeout): + source = conn.getresponse() + body = source.read() + if 200 <= source.status < 300 \ + or 400 <= source.status < 500: + statuses.append(source.status) + reasons.append(source.reason) + bodies.append(body) + else: + if source.status == 507: + self.error_limit(node) + accounts.insert(0, account) + except: + accounts.insert(0, account) + self.exception_occurred(node, 'Container', + 'Trying to DELETE %s' % req.path) + if not accounts: + break + while len(statuses) < len(containers): + statuses.append(503) + reasons.append('') + bodies.append('') + self.app.memcache.delete('container%s' % req.path_info.rstrip('/')) + resp = self.best_response(req, statuses, reasons, bodies, + 'Container DELETE') + if 200 <= resp.status_int <= 299: + for status in statuses: + if status < 200 or status > 299: + # If even one node doesn't do the delete, we can't be sure + # what the outcome will be once everything is in sync; so + # we 503. + self.app.logger.error('Returning 503 because not all ' + 'container nodes confirmed DELETE, transaction %s' % + self.trans_id) + return HTTPServiceUnavailable(request=req) + if resp.status_int == 202: # Indicates no server had the container + return HTTPNotFound(request=req) + return resp + + +class AccountController(Controller): + """WSGI controller for account requests""" + + def __init__(self, app, account_name, **kwargs): + Controller.__init__(self, app) + self.account_name = unquote(account_name) + + def GETorHEAD(self, req): + """Handler for HTTP GET/HEAD requests.""" + partition, nodes = self.app.account_ring.get_nodes(self.account_name) + return self.GETorHEAD_base(req, 'Account', partition, nodes, + req.path_info.rstrip('/'), self.app.account_ring.replica_count) + + +class BaseApplication(object): + """Base WSGI application for the proxy server""" + + log_name = 'base_application' + + def __init__(self, conf, memcache, logger=None): + if logger: + self.logger = logger + else: + self.logger = get_logger(conf, self.log_name) + if conf is None: + conf = {} + swift_dir = conf.get('swift_dir', '/etc/swift') + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.client_timeout = int(conf.get('client_timeout', 60)) + self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) + self.container_chunk_size = \ + int(conf.get('container_chunk_size', 65536)) + self.account_chunk_size = int(conf.get('account_chunk_size', 65536)) + self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) + self.log_headers = conf.get('log_headers') == 'True' + self.error_suppression_interval = \ + int(conf.get('error_suppression_interval', 60)) + self.error_suppression_limit = \ + int(conf.get('error_suppression_limit', 10)) + self.recheck_container_existence = \ + int(conf.get('recheck_container_existence', 60)) + self.recheck_account_existence = \ + int(conf.get('recheck_account_existence', 60)) + self.resellers_conf = ConfigParser() + self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) + self.object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz')) + self.container_ring = \ + Ring(os.path.join(swift_dir, 'container.ring.gz')) + self.account_ring = Ring(os.path.join(swift_dir, 'account.ring.gz')) + self.memcache = memcache + self.rate_limit = float(conf.get('rate_limit', 20000.0)) + self.account_rate_limit = float(conf.get('account_rate_limit', 200.0)) + self.rate_limit_whitelist = [x.strip() for x in + conf.get('rate_limit_account_whitelist', '').split(',') + if x.strip()] + self.rate_limit_blacklist = [x.strip() for x in + conf.get('rate_limit_account_blacklist', '').split(',') + if x.strip()] + self.container_put_lock_timeout = \ + int(conf.get('container_put_lock_timeout', 5)) + + def get_controller(self, path): + """ + Get the controller to handle a request. + + :param path: path from request + :returns: tuple of (controller class, path dictionary) + """ + version, account, container, obj = split_path(path, 1, 4, True) + d = dict(version=version, + account_name=account, + container_name=container, + object_name=obj) + if obj and container and account: + return ObjectController, d + elif container and account: + return ContainerController, d + elif account and not container and not obj: + return AccountController, d + elif version and version == 'healthcheck': + return HealthCheckController, d + return None, d + + def __call__(self, env, start_response): + """ + WSGI entry point. + Wraps env in webob.Request object and passes it down. + + :param env: WSGI environment dictionary + :param start_response: WSGI callable + """ + try: + req = self.update_request(Request(env)) + if 'eventlet.posthooks' in env: + env['eventlet.posthooks'].append( + (self.posthooklogger, (req,), {})) + return self.handle_request(req)(env, start_response) + else: + # Lack of posthook support means that we have to log on the + # start of the response, rather than after all the data has + # been sent. This prevents logging client disconnects + # differently than full transmissions. + response = self.handle_request(req)(env, start_response) + self.posthooklogger(env, req) + return response + except: + print "EXCEPTION IN __call__: %s" % env + start_response('500 Server Error', + [('Content-Type', 'text/plain')]) + return ['Internal server error.\n'] + + def posthooklogger(self, env, req): + pass + + def update_request(self, req): + req.creation_size = '-' + req.sent_size = 0 + req.client_disconnect = False + req.headers['x-cf-trans-id'] = 'tx' + str(uuid.uuid4()) + if 'x-storage-token' in req.headers and \ + 'x-auth-token' not in req.headers: + req.headers['x-auth-token'] = req.headers['x-storage-token'] + return req + + def handle_request(self, req): + """ + Entry point for proxy server. + Should return a WSGI-style callable (such as webob.Response). + + :param req: webob.Request object + """ + try: + try: + controller, path_parts = self.get_controller(req.path) + except ValueError: + return HTTPNotFound(request=req) + if controller == HealthCheckController: + controller = controller(self, **path_parts) + controller.trans_id = req.headers.get('x-cf-trans-id', '-') + if req.method == 'GET': + return controller.GET(req) + return HTTPMethodNotAllowed(request=req) + + if not check_xml_encodable(req.path_info): + return HTTPPreconditionFailed(request=req, body='Invalid UTF8') + if not controller: + return HTTPPreconditionFailed(request=req, body='Bad URL') + rate_limit_allowed_err_resp = \ + self.check_rate_limit(req, path_parts) + if rate_limit_allowed_err_resp is not None: + return rate_limit_allowed_err_resp + + controller = controller(self, **path_parts) + controller.trans_id = req.headers.get('x-cf-trans-id', '-') + try: + handler = getattr(controller, req.method) + if getattr(handler, 'publicly_accessible'): + if path_parts['version']: + req.path_info_pop() + return handler(req) + except AttributeError: + return HTTPMethodNotAllowed(request=req) + except: + self.logger.exception('ERROR Unhandled exception in request') + return HTTPServiceUnavailable(request=req) + + def check_rate_limit(self, req, path_parts): + """Check for rate limiting.""" + return None + + +class Application(BaseApplication): + """WSGI application for the proxy server.""" + + log_name = 'proxy' + + def handle_request(self, req): + """ + Wraps the BaseApplication.handle_request and logs the request. + """ + req.start_time = time.time() + req.response = super(Application, self).handle_request(req) + return req.response + + def posthooklogger(self, env, req): + response = req.response + trans_time = '%.4f' % (time.time() - req.start_time) + if not response.content_length and response.app_iter and \ + hasattr(response.app_iter, '__len__'): + response.content_length = sum(map(len, response.app_iter)) + the_request = quote(unquote(req.path)) + if req.query_string: + the_request = the_request + '?' + req.query_string + # remote user for zeus + client = req.headers.get('x-cluster-client-ip') + if not client and 'x-forwarded-for' in req.headers: + # remote user for other lbs + client = req.headers['x-forwarded-for'].split(',')[0].strip() + raw_in = req.content_length or 0 + if req.creation_size != '-': + raw_in = req.creation_size + raw_out = 0 + if req.method != 'HEAD': + if response.content_length: + raw_out = response.content_length + if req.sent_size or req.client_disconnect: + raw_out = req.sent_size + logged_headers = None + if self.log_headers: + logged_headers = '\n'.join('%s: %s' % (k, v) + for k, v in req.headers.items()) + status_int = req.client_disconnect and 499 or response.status_int + self.logger.info(' '.join(quote(str(x)) for x in ( + client or '-', + req.remote_addr or '-', + time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), + req.method, + the_request, + req.environ['SERVER_PROTOCOL'], + status_int, + req.referer or '-', + req.user_agent or '-', + req.headers.get('x-auth-token', '-'), + raw_in or '-', + raw_out or '-', + req.headers.get('etag', '-'), + req.headers.get('x-cf-trans-id', '-'), + logged_headers or '-', + trans_time, + ))) + + def check_rate_limit(self, req, path_parts): + """ + Check for rate limiting. + + :param req: webob.Request object + :param path_parts: parsed path dictionary + """ + if path_parts['account_name'] in self.rate_limit_blacklist: + self.logger.error('Returning 497 because of blacklisting') + return Response(status='497 Blacklisted', + body='Your account has been blacklisted', request=req) + if path_parts['account_name'] not in self.rate_limit_whitelist: + current_second = time.strftime('%x%H%M%S') + general_rate_limit_key = '%s%s' % (path_parts['account_name'], + current_second) + ops_count = self.memcache.incr(general_rate_limit_key, timeout=2) + if ops_count > self.rate_limit: + self.logger.error( + 'Returning 498 because of ops rate limiting') + return Response(status='498 Rate Limited', + body='Slow down', request=req) + elif (path_parts['container_name'] + and not path_parts['object_name']) \ + or \ + (path_parts['account_name'] + and not path_parts['container_name']): + # further limit operations on a single account or container + rate_limit_key = '%s%s%s' % (path_parts['account_name'], + path_parts['container_name'] or '-', + current_second) + ops_count = self.memcache.incr(rate_limit_key, timeout=2) + if ops_count > self.account_rate_limit: + self.logger.error( + 'Returning 498 because of account and container' + ' rate limiting') + return Response(status='498 Rate Limited', + body='Slow down', request=req) + return None diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/functional/sample.conf b/test/functional/sample.conf new file mode 100644 index 0000000000..dbcd969e74 --- /dev/null +++ b/test/functional/sample.conf @@ -0,0 +1,10 @@ +# Sample functional test configuration file +auth_host = 127.0.0.1 +auth_port = 80 +auth_ssl = no + +account = test_account +username = test_user +password = test_password + +collate = C diff --git a/test/functional/swift.py b/test/functional/swift.py new file mode 100644 index 0000000000..872a09085b --- /dev/null +++ b/test/functional/swift.py @@ -0,0 +1,731 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import httplib +import os +import random +import socket +import StringIO +import time +import urllib + +import simplejson as json + +from xml.dom import minidom + +class AuthenticationFailed(Exception): + pass + +class RequestError(Exception): + pass + +class ResponseError(Exception): + def __init__(self, response): + self.status = response.status + self.reason = response.reason + Exception.__init__(self) + + def __str__(self): + return '%d: %s' % (self.status, self.reason) + + def __repr__(self): + return '%d: %s' % (self.status, self.reason) + +def listing_empty(method): + for i in xrange(0, 6): + if len(method()) == 0: + return True + + time.sleep(2**i) + + return False + +def listing_items(method): + marker = None + once = True + items = [] + + while once or items: + for i in items: + yield i + + if once or marker: + if marker: + items = method(parms={'marker':marker}) + else: + items = method() + + if len(items) == 10000: + marker = items[-1] + else: + marker = None + + once = False + else: + items = [] + +class Connection(object): + def __init__(self, config): + self.auth_host = config['auth_host'] + self.auth_port = int(config['auth_port']) + self.auth_ssl = config['auth_ssl'] in ('on', 'true', 'yes', '1') + + self.account = config['account'] + self.username = config['username'] + self.password = config['password'] + + self.storage_host = None + self.storage_port = None + + self.conn_class = None + + def get_account(self): + return Account(self, self.account) + + def authenticate(self, clone_conn=None): + if clone_conn: + self.conn_class = clone_conn.conn_class + self.storage_host = clone_conn.storage_host + self.storage_url = clone_conn.storage_url + self.storage_port = clone_conn.storage_port + self.storage_token = clone_conn.storage_token + return + + headers = { + 'x-storage-user': self.username, + 'x-storage-pass': self.password, + } + + path = '/v1/%s/auth' % (self.account) + if self.auth_ssl: + connection = httplib.HTTPSConnection(self.auth_host, + port=self.auth_port) + else: + connection = httplib.HTTPConnection(self.auth_host, + port=self.auth_port) + #connection.set_debuglevel(3) + connection.request('GET', path, '', headers) + response = connection.getresponse() + connection.close() + + if response.status == 401: + raise AuthenticationFailed() + + if response.status != 204: + raise ResponseError(response) + + for hdr in response.getheaders(): + if hdr[0].lower() == "x-storage-url": + storage_url = hdr[1] + elif hdr[0].lower() == "x-storage-token": + storage_token = hdr[1] + + if not (storage_url and storage_token): + raise AuthenticationFailed() + + x = storage_url.split('/') + + if x[0] == 'http:': + self.conn_class = httplib.HTTPConnection + self.storage_port = 80 + elif x[0] == 'https:': + self.conn_class = httplib.HTTPSConnection + self.storage_port = 443 + else: + raise ValueError, 'unexpected protocol %s' % (x[0]) + + self.storage_host = x[2].split(':')[0] + if ':' in x[2]: + self.storage_port = int(x[2].split(':')[1]) + self.storage_url = '/%s/%s' % (x[3], x[4]) + + self.storage_token = storage_token + + self.http_connect() + return self.storage_url, self.storage_token + + def http_connect(self): + self.connection = self.conn_class(self.storage_host, + port=self.storage_port) + #self.connection.set_debuglevel(3) + + def make_path(self, path=[], cfg={}): + if cfg.get('version_only_path'): + return '/' + self.storage_url.split('/')[1] + + if path: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_path_quote'): + quote = lambda x: x + return '%s/%s' % (self.storage_url, '/'.join([quote(i) for i + in path])) + else: + return self.storage_url + + def make_headers(self, hdrs, cfg={}): + headers = {} + + if not cfg.get('no_auth_token'): + headers['X-Auth-Token'] = self.storage_token + + if isinstance(hdrs, dict): + headers.update(hdrs) + return headers + + def make_request(self, method, path=[], data='', hdrs={}, parms={}, + cfg={}): + + path = self.make_path(path, cfg=cfg) + headers = self.make_headers(hdrs, cfg=cfg) + + if isinstance(parms, dict) and parms: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_parms_quote'): + quote = lambda x: x + query_args = ['%s=%s' % (quote(x), quote(str(y))) for (x,y) in + parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = cfg.get('set_content_length') + else: + headers['Content-Length'] = len(data) + + def try_request(): + self.http_connect() + self.connection.request(method, path, data, headers) + return self.connection.getresponse() + + self.response = None + try_count = 0 + while try_count < 5: + try_count += 1 + + try: + self.response = try_request() + except httplib.HTTPException: + continue + + if self.response.status == 401: + self.authenticate() + continue + elif self.response.status == 503: + if try_count != 5: + time.sleep(5) + continue + + break + + if self.response: + return self.response.status + + raise RequestError('Unable to compelte http request') + + def put_start(self, path, hdrs={}, parms={}, cfg={}, chunked=False): + self.http_connect() + + path = self.make_path(path, cfg) + headers = self.make_headers(hdrs, cfg=cfg) + + if chunked: + headers['Transfer-Encoding'] = 'chunked' + headers.pop('Content-Length', None) + + if isinstance(parms, dict) and parms: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_parms_quote'): + quote = lambda x: x + query_args = ['%s=%s' % (quote(x), quote(str(y))) for (x,y) in + parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + + query_args = ['%s=%s' % (urllib.quote(x), + urllib.quote(str(y))) for (x,y) in parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + + self.connection = self.conn_class(self.storage_host, + port=self.storage_port) + #self.connection.set_debuglevel(3) + self.connection.putrequest('PUT', path) + for key, value in headers.iteritems(): + self.connection.putheader(key, value) + self.connection.endheaders() + + def put_data(self, data, chunked=False): + if chunked: + self.connection.send('%s\r\n%s\r\n' % (hex(len(data)), data)) + else: + self.connection.send(data) + + def put_end(self, chunked=False): + if chunked: + self.connection.send('0\r\n\r\n') + + self.response = self.connection.getresponse() + self.connection.close() + return self.response.status + +class Base: + def __str__(self): + return self.name + + def header_fields(self, fields): + headers = dict(self.conn.response.getheaders()) + + ret = {} + for field in fields: + if not headers.has_key(field[1]): + raise ValueError("%s was not found in response header" % + (field[1])) + + try: + ret[field[0]] = int(headers[field[1]]) + except ValueError: + ret[field[0]] = headers[field[1]] + return ret + +class Account(Base): + def __init__(self, conn, name): + self.conn = conn + self.name = str(name) + + def container(self, container_name): + return Container(self.conn, self.name, container_name) + + def containers(self, hdrs={}, parms={}, cfg={}): + format = parms.get('format', None) + if format not in [None, 'json', 'xml']: + raise RequestError('Invalid format: %s' % format) + if format == None and parms.has_key('format'): + del parms['format'] + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + if status == 200: + if format == 'json': + conts = json.loads(self.conn.response.read()) + for cont in conts: + cont['name'] = cont['name'].encode('utf-8') + return conts + elif format == 'xml': + conts = [] + tree = minidom.parseString(self.conn.response.read()) + for x in tree.getElementsByTagName('container'): + cont = {} + for key in ['name', 'count', 'bytes']: + cont[key] = x.getElementsByTagName(key)[0].\ + childNodes[0].nodeValue + conts.append(cont) + for cont in conts: + cont['name'] = cont['name'].encode('utf-8') + return conts + else: + lines = self.conn.response.read().split('\n') + if lines and not lines[-1]: + lines = lines[:-1] + return lines + elif status == 204: + return [] + + raise ResponseError(self.conn.response) + + def delete_containers(self): + for c in listing_items(self.containers): + cont = self.container(c) + if not cont.delete_recursive(): + return False + + return listing_empty(self.containers) + + def info(self, hdrs={}, parms={}, cfg={}): + if self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) != 204: + + raise ResponseError(self.conn.response) + + fields = [['object_count', 'x-account-object-count'], + ['container_count', 'x-account-container-count'], + ['bytes_used', 'x-account-bytes-used']] + + return self.header_fields(fields) + + @property + def path(self): + return [] + +class Container(Base): + def __init__(self, conn, account, name): + self.conn = conn + self.account = str(account) + self.name = str(name) + + def create(self, hdrs={}, parms={}, cfg={}): + return self.conn.make_request('PUT', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) in (201, 202) + + def delete(self, hdrs={}, parms={}): + return self.conn.make_request('DELETE', self.path, hdrs=hdrs, + parms=parms) == 204 + + def delete_files(self): + for f in listing_items(self.files): + file = self.file(f) + if not file.delete(): + return False + + return listing_empty(self.files) + + def delete_recursive(self): + return self.delete_files() and self.delete() + + def file(self, file_name): + return File(self.conn, self.account, self.name, file_name) + + def files(self, hdrs={}, parms={}, cfg={}): + format = parms.get('format', None) + if format not in [None, 'json', 'xml']: + raise RequestError('Invalid format: %s' % format) + if format == None and parms.has_key('format'): + del parms['format'] + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + if status == 200: + if format == 'json': + files = json.loads(self.conn.response.read()) + + for file in files: + file['name'] = file['name'].encode('utf-8') + file['content_type'] = file['content_type'].encode('utf-8') + return files + elif format == 'xml': + files = [] + tree = minidom.parseString(self.conn.response.read()) + for x in tree.getElementsByTagName('object'): + file = {} + for key in ['name', 'hash', 'bytes', 'content_type', + 'last_modified']: + + file[key] = x.getElementsByTagName(key)[0].\ + childNodes[0].nodeValue + files.append(file) + + for file in files: + file['name'] = file['name'].encode('utf-8') + file['content_type'] = file['content_type'].encode('utf-8') + return files + else: + content = self.conn.response.read() + if content: + lines = content.split('\n') + if lines and not lines[-1]: + lines = lines[:-1] + return lines + else: + return [] + elif status == 204: + return [] + + raise ResponseError(self.conn.response) + + def info(self, hdrs={}, parms={}, cfg={}): + status = self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + + if self.conn.response.status == 204: + fields = [['bytes_used', 'x-container-bytes-used'], + ['object_count', 'x-container-object-count']] + + return self.header_fields(fields) + + raise ResponseError(self.conn.response) + + @property + def path(self): + return [self.name] + +class File(Base): + def __init__(self, conn, account, container, name): + self.conn = conn + self.account = str(account) + self.container = str(container) + self.name = str(name) + + self.chunked_write_in_progress = False + self.content_type = None + self.size = None + self.metadata = {} + + def make_headers(self, cfg={}): + headers = {} + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = cfg.get('set_content_length') + elif self.size: + headers['Content-Length'] = self.size + else: + headers['Content-Length'] = 0 + + if cfg.get('no_content_type'): + pass + elif self.content_type: + headers['Content-Type'] = self.content_type + else: + headers['Content-Type'] = 'application/octet-stream' + + for key in self.metadata: + headers['X-Object-Meta-'+key] = self.metadata[key] + + return headers + + @classmethod + def compute_md5sum(cls, data): + block_size = 4096 + + if isinstance(data, str): + data = StringIO.StringIO(data) + + checksum = hashlib.md5() + buff = data.read(block_size) + while buff: + checksum.update(buff) + buff = data.read(block_size) + data.seek(0) + return checksum.hexdigest() + + def copy(self, dest_cont, dest_file, hdrs={}, parms={}, cfg={}): + if cfg.has_key('destination'): + headers = {'Destination': cfg['destination']} + elif cfg.get('no_destination'): + headers = {} + else: + headers = {'Destination': '%s/%s' % (dest_cont, dest_file)} + headers.update(hdrs) + + if headers.has_key('Destination'): + headers['Destination'] = urllib.quote(headers['Destination']) + + return self.conn.make_request('COPY', self.path, hdrs=headers, + parms=parms) == 201 + + def delete(self, hdrs={}, parms={}): + if self.conn.make_request('DELETE', self.path, hdrs=hdrs, + parms=parms) != 204: + + raise ResponseError(self.conn.response) + + return True + + def info(self, hdrs={}, parms={}, cfg={}): + if self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) != 200: + + raise ResponseError(self.conn.response) + + fields = [['content_length', 'content-length'], ['content_type', + 'content-type'], ['last_modified', 'last-modified'], ['etag', + 'etag']] + + header_fields = self.header_fields(fields) + header_fields['etag'] = header_fields['etag'].strip('"') + return header_fields + + def initialize(self, hdrs={}, parms={}): + if not self.name: + return False + + status = self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms) + if status == 404: + return False + elif (status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + for hdr in self.conn.response.getheaders(): + if hdr[0].lower() == 'content-type': + self.content_type = hdr[1] + if hdr[0].lower().startswith('x-object-meta-'): + self.metadata[hdr[0][14:]] = hdr[1] + if hdr[0].lower() == 'etag': + self.etag = hdr[1].strip('"') + if hdr[0].lower() == 'content-length': + self.size = int(hdr[1]) + if hdr[0].lower() == 'last-modified': + self.last_modified = hdr[1] + + return True + + def load_from_filename(self, filename, callback=None): + fobj = open(filename, 'rb') + self.write(fobj, callback=callback) + fobj.close() + + @property + def path(self): + return [self.container, self.name] + + @classmethod + def random_data(cls, size=None): + if size == None: + size = random.randint(1, 32768) + fd = open('/dev/urandom', 'r') + data = fd.read(size) + fd.close() + return data + + def read(self, size=-1, offset=0, hdrs=None, buffer=None, + callback=None, cfg={}): + + if size > 0: + range = 'bytes=%d-%d' % (offset, (offset + size) - 1) + if hdrs: + hdrs['Range'] = range + else: + hdrs = {'Range': range} + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + cfg=cfg) + + if(status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + for hdr in self.conn.response.getheaders(): + if hdr[0].lower() == 'content-type': + self.content_type = hdr[1] + + if hasattr(buffer, 'write'): + scratch = self.conn.response.read(8192) + transferred = 0 + + while len(scratch) > 0: + buffer.write(scratch) + transferred += len(scratch) + if callable(callback): + callback(transferred, self.size) + scratch = self.conn.response.read(8192) + return None + else: + return self.conn.response.read() + + def read_md5(self): + status = self.conn.make_request('GET', self.path) + + if(status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + checksum = hashlib.md5() + + scratch = self.conn.response.read(8192) + while len(scratch) > 0: + checksum.update(scratch) + scratch = self.conn.response.read(8192) + + return checksum.hexdigest() + + def save_to_filename(self, filename, callback=None): + try: + fobj = open(filename, 'wb') + self.read(buffer=fobj, callback=callback) + finally: + fobj.close() + + def sync_metadata(self, metadata={}, cfg={}): + self.metadata.update(metadata) + + if self.metadata: + headers = self.make_headers(cfg=cfg) + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = \ + cfg.get('set_content_length') + else: + headers['Content-Length'] = 0 + + self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) + + if self.conn.response.status != 202: + raise ResponseError(self.conn.response) + + return True + + def chunked_write(self, data=None, hdrs={}, parms={}, cfg={}): + if data != None and self.chunked_write_in_progress: + self.conn.put_data(data, True) + elif data != None: + self.chunked_write_in_progress = True + + headers = self.make_headers(cfg=cfg) + headers.update(hdrs) + + self.conn.put_start(self.path, hdrs=headers, parms=parms, + cfg=cfg, chunked=True) + + self.conn.put_data(data, True) + elif self.chunked_write_in_progress: + self.chunked_write_in_progress = False + return self.conn.put_end(True) == 201 + else: + raise RuntimeError + + def write(self, data='', hdrs={}, parms={}, callback=None, cfg={}): + block_size = 2**20 + + if isinstance(data, file): + try: + data.flush() + data.seek(0) + except IOError: + pass + self.size = int(os.fstat(data.fileno())[6]) + else: + data = StringIO.StringIO(data) + self.size = data.len + + headers = self.make_headers(cfg=cfg) + headers.update(hdrs) + + self.conn.put_start(self.path, hdrs=headers, parms=parms, cfg=cfg) + + transfered = 0 + buff = data.read(block_size) + try: + while len(buff) > 0: + self.conn.put_data(buff) + buff = data.read(block_size) + transfered += len(buff) + if callable(callback): + callback(transfered, self.size) + + self.conn.put_end() + except socket.timeout, err: + raise err + + if (self.conn.response.status < 200) or \ + (self.conn.response.status > 299): + + raise ResponseError(self.conn.response) + + self.md5 = self.compute_md5sum(data) + + return True + + def write_random(self, size=None, hdrs={}, parms={}, cfg={}): + data = self.random_data(size) + if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): + raise ResponseError(self.conn.response) + self.md5 = self.compute_md5sum(StringIO.StringIO(data)) + return data diff --git a/test/functional/tests.py b/test/functional/tests.py new file mode 100644 index 0000000000..75d6e17326 --- /dev/null +++ b/test/functional/tests.py @@ -0,0 +1,1498 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import configobj +from datetime import datetime +import locale +import os +import random +import StringIO +import time +import threading +import uuid +import unittest +import urllib + +from swift import Account, AuthenticationFailed, Connection, Container, \ + File, ResponseError + +config = configobj.ConfigObj(os.environ['SWIFT_TEST_CONFIG_FILE']) +locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C')) + +NoRun = object + +class Base: + pass + +def chunks(s, length=3): + i, j = 0, length + while i < len(s): + yield s[i:j] + i, j = j, j + length + +def timeout(seconds, method, *args, **kwargs): + class TimeoutThread(threading.Thread): + def __init__ (self, method, *args, **kwargs): + threading.Thread.__init__(self) + + self.method = method + self.args = args + self.kwargs = kwargs + self.exception = None + + def run(self): + try: + self.method(*self.args, **self.kwargs) + except Exception, e: + self.exception = e + + t = TimeoutThread(method, *args, **kwargs) + t.start() + t.join(seconds) + + if t.exception: + raise t.exception + + if t.isAlive(): + t._Thread__stop() + return True + return False + +class Utils: + @classmethod + def create_ascii_name(cls, length=None): + return uuid.uuid4().hex + + @classmethod + def create_utf8_name(cls, length=None): + if length == None: + length = 15 + else: + length = int(length) + + utf8_chars = u'\uF10F\uD20D\uB30B\u9409\u8508\u5605\u3703\u1801'\ + u'\u0900\uF110\uD20E\uB30C\u940A\u8509\u5606\u3704'\ + u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\ + u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\ + u'\u5608\u3706\u1804\u0903\u03A9\u2603' + return ''.join([random.choice(utf8_chars) for x in \ + xrange(length)]).encode('utf-8') + + create_name = create_ascii_name + +class Base(unittest.TestCase): + def setUp(self): + cls = type(self) + if not cls.set_up: + cls.env.setUp() + cls.set_up = True + + def assert_body(self, body): + response_body = self.env.conn.response.read() + self.assert_(response_body == body, + 'Body returned: %s' % (response_body)) + + def assert_status(self, status): + self.assert_(self.env.conn.response.status == status, + 'Status returned: %d' % (self.env.conn.response.status)) + +class Base2(object): + def setUp(self): + Utils.create_name = Utils.create_utf8_name + super(Base2, self).setUp() + + def tearDown(self): + Utils.create_name = Utils.create_ascii_name + +class TestAccountEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config['account']) + cls.account.delete_containers() + + cls.containers = [] + for i in range(10): + cont = cls.account.container(Utils.create_name()) + if not cont.create(): + raise ResponseError(cls.conn.response) + + cls.containers.append(cont) + +class TestAccountDev(Base): + env = TestAccountEnv + set_up = False + +class TestAccountDevUTF8(Base2, TestAccountDev): + set_up = False + +class TestAccount(Base): + env = TestAccountEnv + set_up = False + + def testNoAuthToken(self): + self.assertRaises(ResponseError, self.env.account.info, + cfg={'no_auth_token':True}) + self.assert_status(412) + + self.assertRaises(ResponseError, self.env.account.containers, + cfg={'no_auth_token':True}) + self.assert_status(412) + + def testInvalidUTF8Path(self): + invalid_utf8 = Utils.create_utf8_name()[::-1] + container = self.env.account.container(invalid_utf8) + self.assert_(not container.create(cfg={'no_path_quote':True})) + self.assert_status(412) + self.assert_body('Invalid UTF8') + + def testVersionOnlyPath(self): + self.env.account.conn.make_request('PUT', + cfg={'version_only_path':True}) + self.assert_status(412) + self.assert_body('Bad URL') + + def testPUT(self): + self.env.account.conn.make_request('PUT') + self.assert_status(405) + + def testAccountHead(self): + try_count = 0 + while try_count < 5: + try_count += 1 + + info = self.env.account.info() + for field in ['object_count', 'container_count', 'bytes_used']: + self.assert_(info[field] >= 0) + + if info['container_count'] == len(self.env.containers): + break + + if try_count < 5: + time.sleep(1) + + self.assertEquals(info['container_count'], len(self.env.containers)) + self.assert_status(204) + + def testContainerSerializedInfo(self): + container_info = {} + for container in self.env.containers: + info = {'bytes': 0} + info['count'] = random.randint(10, 30) + for i in range(info['count']): + file = container.file(Utils.create_name()) + bytes = random.randint(1, 32768) + file.write_random(bytes) + info['bytes'] += bytes + + container_info[container.name] = info + + for format in ['json', 'xml']: + for a in self.env.account.containers( + parms={'format':format}): + + self.assert_(a['count'] >= 0) + self.assert_(a['bytes'] >= 0) + + headers = dict(self.env.conn.response.getheaders()) + if format == 'json': + self.assertEquals(headers['content-type'], + 'application/json; charset=utf8') + elif format == 'xml': + self.assertEquals(headers['content-type'], + 'application/xml; charset=utf8') + + def testListingLimit(self): + limit = 10000 + + for l in (1, 100, limit/2, limit-1, limit, limit+1, limit*2): + p = {'limit':l} + + if l <= limit: + self.assert_(len(self.env.account.containers(parms=p)) <= l) + self.assert_status(200) + else: + self.assertRaises(ResponseError, + self.env.account.containers, parms=p) + self.assert_status(412) + + def testContainerListing(self): + a = sorted([c.name for c in self.env.containers]) + + for format in [None, 'json', 'xml']: + b = self.env.account.containers(parms={'format':format}) + + if isinstance(b[0], dict): + b = [x['name'] for x in b] + + self.assertEquals(a, b) + + def testInvalidAuthToken(self): + hdrs = {'X-Auth-Token': 'bogus_auth_token'} + self.assertRaises(ResponseError, self.env.account.info, hdrs=hdrs) + self.assert_status(401) + + def testLastContainerMarker(self): + for format in [None, 'json', 'xml']: + containers = self.env.account.containers({'format':format}) + self.assertEquals(len(containers), len(self.env.containers)) + self.assert_status(200) + + containers = self.env.account.containers( + parms={'format':format,'marker':containers[-1]}) + self.assertEquals(len(containers), 0) + if format == None: + self.assert_status(204) + else: + self.assert_status(200) + + def testMarkerLimitContainerList(self): + for format in [None, 'json', 'xml']: + for marker in ['0', 'A', 'I', 'R', 'Z', 'a', 'i', 'r', 'z', \ + 'abc123', 'mnop', 'xyz']: + + limit = random.randint(2, 9) + containers = self.env.account.containers( + parms={'format':format, 'marker':marker, 'limit':limit}) + self.assert_(len(containers) <= limit) + if containers: + if isinstance(containers[0], dict): + containers = [x['name'] for x in containers] + self.assert_(locale.strcoll(containers[0], marker) > 0) + + def testContainersOrderedByName(self): + for format in [None, 'json', 'xml']: + containers = self.env.account.containers( + parms={'format':format}) + if isinstance(containers[0], dict): + containers = [x['name'] for x in containers] + self.assertEquals(sorted(containers, cmp=locale.strcoll), + containers) + +class TestAccountUTF8(Base2, TestAccount): + set_up = False + +class TestAccountNoContainersEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config['account']) + cls.account.delete_containers() + +class TestAccountNoContainers(Base): + env = TestAccountNoContainersEnv + set_up = False + + def testGetRequest(self): + for format in [None, 'json', 'xml']: + self.assert_(not self.env.account.containers( + parms={'format':format})) + + if format == None: + self.assert_status(204) + else: + self.assert_status(200) + +class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers): + set_up = False + +class TestContainerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config['account']) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_count = 10 + cls.file_size = 128 + cls.files = list() + for x in range(cls.file_count): + file = cls.container.file(Utils.create_name()) + file.write_random(cls.file_size) + cls.files.append(file.name) + +class TestContainerDev(Base): + env = TestContainerEnv + set_up = False + +class TestContainerDevUTF8(Base2, TestContainerDev): + set_up = False + +class TestContainer(Base): + env = TestContainerEnv + set_up = False + + def testContainerNameLimit(self): + limit = 256 + + for l in (limit-100, limit-10, limit-1, limit, + limit+1, limit+10, limit+100): + + cont = self.env.account.container('a'*l) + if l <= limit: + self.assert_(cont.create()) + self.assert_status(201) + else: + self.assert_(not cont.create()) + self.assert_status(400) + + def testFileThenContainerDelete(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + self.assert_(file.write_random()) + + self.assert_(file.delete()) + self.assert_status(204) + self.assert_(file.name not in cont.files()) + + self.assert_(cont.delete()) + self.assert_status(204) + self.assert_(cont.name not in self.env.account.containers()) + + def testFileListingLimitMarkerPrefix(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + + files = sorted([Utils.create_name() for x in xrange(10)]) + for f in files: + file = cont.file(f) + self.assert_(file.write_random()) + + for i in xrange(len(files)): + f = files[i] + for j in xrange(1, len(files)-i): + self.assert_(cont.files(parms={'limit':j, 'marker':f}) == files[i+1:i+j+1]) + self.assert_(cont.files(parms={'marker':f}) == files[i+1:]) + self.assert_(cont.files(parms={'marker': f, 'prefix':f}) == []) + self.assert_(cont.files(parms={'prefix': f}) == [f]) + + def testPrefixAndLimit(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + + prefix_file_count = 10 + limit_count = 2 + prefixs = ['alpha/', 'beta/', 'kappa/'] + prefix_files = {} + + all_files = [] + for prefix in prefixs: + prefix_files[prefix] = [] + + for i in range(prefix_file_count): + file = cont.file(prefix + Utils.create_name()) + file.write() + prefix_files[prefix].append(file.name) + + for format in [None, 'json', 'xml']: + for prefix in prefixs: + files = cont.files(parms={'prefix':prefix}) + self.assertEquals(files, sorted(prefix_files[prefix])) + + for format in [None, 'json', 'xml']: + for prefix in prefixs: + files = cont.files(parms={'limit':limit_count, + 'prefix':prefix}) + self.assertEquals(len(files), limit_count) + + for file in files: + self.assert_(file.startswith(prefix)) + + def testCreate(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.name in self.env.account.containers()) + + def testContainerFileListOnContainerThatDoesNotExist(self): + for format in [None, 'json', 'xml']: + container = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, container.files, + parms={'format':format}) + self.assert_status(404) + + def testUtf8Container(self): + valid_utf8 = Utils.create_utf8_name() + invalid_utf8 = valid_utf8[::-1] + container = self.env.account.container(valid_utf8) + self.assert_(container.create(cfg={'no_path_quote':True})) + self.assert_(container.name in self.env.account.containers()) + self.assertEquals(container.files(), []) + self.assert_(container.delete()) + + container = self.env.account.container(invalid_utf8) + self.assert_(not container.create(cfg={'no_path_quote':True})) + self.assert_status(412) + self.assertRaises(ResponseError, container.files, + cfg={'no_path_quote':True}) + self.assert_status(412) + + def testCreateOnExisting(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.create()) + self.assert_status(202) + + def testSlashInName(self): + if Utils.create_name == Utils.create_utf8_name: + cont_name = list(unicode(Utils.create_name(), 'utf-8')) + else: + cont_name = list(Utils.create_name()) + + cont_name[random.randint(2, len(cont_name)-2)] = '/' + cont_name = ''.join(cont_name) + + if Utils.create_name == Utils.create_utf8_name: + cont_name = cont_name.encode('utf-8') + + cont = self.env.account.container(cont_name) + self.assert_(not cont.create(cfg={'no_path_quote':True}), + 'created container with name %s' % (cont_name)) + self.assert_status(404) + self.assert_(cont.name not in self.env.account.containers()) + + def testDelete(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.delete()) + self.assert_status(204) + self.assert_(cont.name not in self.env.account.containers()) + + def testDeleteOnContainerThatDoesNotExist(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(not cont.delete()) + self.assert_status(404) + + def testDeleteOnContainerWithFiles(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + file.write_random(self.env.file_size) + self.assert_(file.name in cont.files()) + self.assert_(not cont.delete()) + self.assert_status(409) + + def testFileCreateInContainerThatDoesNotExist(self): + file = File(self.env.conn, self.env.account, Utils.create_name(), + Utils.create_name()) + self.assertRaises(ResponseError, file.write) + self.assert_status(404) + + def testLastFileMarker(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files({'format':format}) + self.assertEquals(len(files), len(self.env.files)) + self.assert_status(200) + + files = self.env.container.files( + parms={'format':format,'marker':files[-1]}) + self.assertEquals(len(files), 0) + + if format == None: + self.assert_status(204) + else: + self.assert_status(200) + + def testContainerFileList(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format':format}) + self.assert_status(200) + if isinstance(files[0], dict): + files = [x['name'] for x in files] + + for file in self.env.files: + self.assert_(file in files) + + for file in files: + self.assert_(file in self.env.files) + + def testMarkerLimitFileList(self): + for format in [None, 'json', 'xml']: + for marker in ['0', 'A', 'I', 'R', 'Z', 'a', 'i', 'r', 'z', \ + 'abc123', 'mnop', 'xyz']: + + limit = random.randint(2, self.env.file_count-1) + files = self.env.container.files(parms={'format':format, \ + 'marker':marker, 'limit':limit}) + + if not files: + continue + + if isinstance(files[0], dict): + files = [x['name'] for x in files] + + self.assert_(len(files) <= limit) + if files: + if isinstance(files[0], dict): + files = [x['name'] for x in files] + self.assert_(locale.strcoll(files[0], marker) > 0) + + def testFileOrder(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format':format}) + if isinstance(files[0], dict): + files = [x['name'] for x in files] + self.assertEquals(sorted(files, cmp=locale.strcoll), files) + + def testContainerInfo(self): + info = self.env.container.info() + self.assert_status(204) + self.assertEquals(info['object_count'], self.env.file_count) + self.assertEquals(info['bytes_used'], + self.env.file_count*self.env.file_size) + + def testContainerInfoOnContainerThatDoesNotExist(self): + container = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, container.info) + self.assert_status(404) + + def testContainerFileListWithLimit(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format':format, + 'limit':2}) + self.assertEquals(len(files), 2) + + def testTooLongName(self): + cont = self.env.account.container('x'*257) + self.assert_(not cont.create(), 'created container with name %s' % \ + (cont.name)) + self.assert_status(400) + + def testYourmomContainer(self): + cont = self.env.account.container('yourmom') + self.assert_(cont.create()) + + info = cont.info() + self.assert_status(204) + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + def testContainerExistenceCachingProblem(self): + cont = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, cont.files) + self.assert_(cont.create()) + cont.files() + + cont = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, cont.files) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + file.write_random() + +class TestContainerUTF8(Base2, TestContainer): + set_up = False + +class TestContainerPathsEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config['account']) + cls.account.delete_containers() + + cls.file_size = 8 + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.files = [ + '/file1', + '/file A', + '/dir1/', + '/dir2/', + '/dir1/file2', + '/dir1/subdir1/', + '/dir1/subdir2/', + '/dir1/subdir1/file2', + '/dir1/subdir1/file3', + '/dir1/subdir1/file4', + '/dir1/subdir1/subsubdir1/', + '/dir1/subdir1/subsubdir1/file5', + '/dir1/subdir1/subsubdir1/file6', + '/dir1/subdir1/subsubdir1/file7', + '/dir1/subdir1/subsubdir1/file8', + '/dir1/subdir1/subsubdir2/', + '/dir1/subdir1/subsubdir2/file9', + '/dir1/subdir1/subsubdir2/file0', + 'file1', + 'dir1/', + 'dir2/', + 'dir1/file2', + 'dir1/subdir1/', + 'dir1/subdir2/', + 'dir1/subdir1/file2', + 'dir1/subdir1/file3', + 'dir1/subdir1/file4', + 'dir1/subdir1/subsubdir1/', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file6', + 'dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir2/', + 'dir1/subdir1/subsubdir2/file9', + 'dir1/subdir1/subsubdir2/file0', + 'dir1/subdir with spaces/', + 'dir1/subdir with spaces/file B', + 'dir1/subdir+with{whatever/', + 'dir1/subdir+with{whatever/file D', + ] + + for f in cls.files: + file = cls.container.file(f) + if f.endswith('/'): + file.write(hdrs={'content-type': 'application/directory'}) + else: + file.write_random(cls.file_size, hdrs={'content-type': \ + 'application/directory'}) + +class TestContainerPaths(Base): + env = TestContainerPathsEnv + set_up = False + + def testTraverseContainer(self): + found_files = [] + found_dirs = [] + def recurse_path(path, count=0): + if count > 10: + raise ValueError('too deep recursion') + + for file in self.env.container.files(parms={'path':path}): + self.assert_(file.startswith(path)) + if file.endswith('/'): + recurse_path(file, count + 1) + found_dirs.append(file) + else: + found_files.append(file) + recurse_path('') + for file in self.env.files: + if file.startswith('/'): + self.assert_(file not in found_dirs) + self.assert_(file not in found_files) + elif file.endswith('/'): + self.assert_(file in found_dirs) + self.assert_(file not in found_files) + else: + self.assert_(file in found_files) + self.assert_(file not in found_dirs) + found_files = [] + found_dirs = [] + recurse_path('/') + for file in self.env.files: + if not file.startswith('/'): + self.assert_(file not in found_dirs) + self.assert_(file not in found_files) + elif file.endswith('/'): + self.assert_(file in found_dirs) + self.assert_(file not in found_files) + else: + self.assert_(file in found_files) + self.assert_(file not in found_dirs) + + def testContainerListing(self): + for format in (None, 'json', 'xml'): + files = self.env.container.files(parms={'format':format}) + + if isinstance(files[0], dict): + files = [str(x['name']) for x in files] + + self.assertEquals(files, sorted(self.env.files)) + + for format in ('json', 'xml'): + for file in self.env.container.files(parms={'format':format}): + self.assert_(int(file['bytes']) >= 0) + self.assert_(file.has_key('last_modified')) + if file['name'].endswith('/'): + self.assertEquals(file['content_type'], + 'application/directory') + + def testStructure(self): + def assert_listing(path, list): + files = self.env.container.files(parms={'path':path}) + self.assertEquals(sorted(list, cmp=locale.strcoll), files) + + assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A']) + assert_listing('/dir1', + ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/']) + assert_listing('/dir1/', + ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/']) + assert_listing('/dir1/subdir1', + ['/dir1/subdir1/subsubdir2/', '/dir1/subdir1/file2', + '/dir1/subdir1/file3', '/dir1/subdir1/file4', + '/dir1/subdir1/subsubdir1/']) + assert_listing('/dir1/subdir2', []) + assert_listing('', ['file1', 'dir1/', 'dir2/']) + assert_listing('dir1', ['dir1/file2', 'dir1/subdir1/', + 'dir1/subdir2/', 'dir1/subdir with spaces/', + 'dir1/subdir+with{whatever/']) + assert_listing('dir1/subdir1', + ['dir1/subdir1/file4', 'dir1/subdir1/subsubdir2/', + 'dir1/subdir1/file2', 'dir1/subdir1/file3', + 'dir1/subdir1/subsubdir1/']) + assert_listing('dir1/subdir1/subsubdir1', + ['dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir1/file6']) + assert_listing('dir1/subdir1/subsubdir1/', + ['dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir1/file6']) + assert_listing('dir1/subdir with spaces/', + ['dir1/subdir with spaces/file B']) + +class TestFileEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config['account']) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_size = 128 + +class TestFileDev(Base): + env = TestFileEnv + set_up = False + +class TestFileDevUTF8(Base2, TestFileDev): + set_up = False + +class TestFile(Base): + env = TestFileEnv + set_up = False + + """ + def testCopy(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + + metadata = {} + for i in range(1): + metadata[Utils.create_name()] = Utils.create_name() + + data = file.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + # copy both from within and across containers + for cont in (self.env.container, dest_cont): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file = self.env.container.file(source_filename) + file.copy('%s%s' % (prefix, cont), dest_filename) + + self.assert_(dest_filename in cont.files()) + + file = cont.file(dest_filename) + + self.assert_(data == file.read()) + self.assert_(file.initialize()) + self.assert_(metadata == file.metadata) + + def testCopy404s(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + for prefix in ('', '/'): + # invalid source container + source_cont = self.env.account.container(Utils.create_name()) + file = source_cont.file(source_filename) + self.assert_(not file.copy('%s%s' % (prefix, self.env.container), + Utils.create_name())) + self.assert_status(404) + + self.assert_(not file.copy('%s%s' % (prefix, dest_cont), + Utils.create_name())) + self.assert_status(404) + + # invalid source object + file = self.env.container.file(Utils.create_name()) + self.assert_(not file.copy('%s%s' % (prefix, self.env.container), + Utils.create_name())) + self.assert_status(404) + + self.assert_(not file.copy('%s%s' % (prefix, dest_cont), + Utils.create_name())) + self.assert_status(404) + + # invalid destination container + file = self.env.container.file(source_filename) + self.assert_(not file.copy('%s%s' % (prefix, Utils.create_name()), + Utils.create_name())) + + def testCopyNoDestinationHeader(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + file = self.env.container.file(source_filename) + self.assert_(not file.copy(Utils.create_name(), Utils.create_name(), + cfg={'no_destination': True})) + self.assert_status(412) + + def testCopyDestinationSlashProblems(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + # no slash + self.assert_(not file.copy(Utils.create_name(), Utils.create_name(), + cfg={'destination': Utils.create_name()})) + self.assert_status(412) + + # extra slash + self.assert_(not file.copy(Utils.create_name(), Utils.create_name(), + cfg={'destination': '/%s/%s/%s' % (Utils.create_name(), + Utils.create_name(), Utils.create_name())})) + self.assert_status(412) + + def testCopyFromHeader(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + + metadata = {} + for i in range(1): + metadata[Utils.create_name()] = Utils.create_name() + file.metadata = metadata + + data = file.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + # copy both from within and across containers + for cont in (self.env.container, dest_cont): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file = cont.file(dest_filename) + file.write(hdrs={'X-Copy-From': '%s%s/%s' % (prefix, + self.env.container.name, source_filename)}) + + self.assert_(dest_filename in cont.files()) + + file = cont.file(dest_filename) + + self.assert_(data == file.read()) + self.assert_(file.initialize()) + self.assert_(metadata == file.metadata) + + def testCopyFromHeader404s(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + for prefix in ('', '/'): + # invalid source container + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % (prefix, + Utils.create_name(), source_filename)}) + self.assert_status(404) + + # invalid source object + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % (prefix, + self.env.container.name, Utils.create_name())}) + self.assert_status(404) + + # invalid destination container + dest_cont = self.env.account.container(Utils.create_name()) + file = dest_cont.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % (prefix, + self.env.container.name, source_filename)}) + self.assert_status(404) + """ + + def testNameLimit(self): + limit = 1024 + + for l in (1, 10, limit/2, limit-1, limit, limit+1, limit*2): + file = self.env.container.file('a'*l) + + if l <= limit: + self.assert_(file.write()) + self.assert_status(201) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + + def testQuestionMarkInName(self): + if Utils.create_name == Utils.create_ascii_name: + file_name = list(Utils.create_name()) + file_name[random.randint(2, len(file_name)-2)] = '?' + file_name = "".join(file_name) + else: + file_name = Utils.create_name(6) + '?' + Utils.create_name(6) + + file = self.env.container.file(file_name) + self.assert_(file.write(cfg={'no_path_quote':True})) + self.assert_(file_name not in self.env.container.files()) + self.assert_(file_name.split('?')[0] in self.env.container.files()) + + def testDeleteThen404s(self): + file = self.env.container.file(Utils.create_name()) + self.assert_(file.write_random()) + self.assert_status(201) + + self.assert_(file.delete()) + self.assert_status(204) + + file.metadata = {Utils.create_name(): Utils.create_name()} + + for method in (file.info, file.read, file.sync_metadata, \ + file.delete): + + self.assertRaises(ResponseError, method) + self.assert_status(404) + + def testBlankMetadataName(self): + file = self.env.container.file(Utils.create_name()) + file.metadata = {'': Utils.create_name()} + self.assertRaises(ResponseError, file.write_random) + self.assert_status(400) + + def testMetadataNumberLimit(self): + number_limit = 90 + + for i in (number_limit-10, number_limit-1, number_limit, + number_limit+1, number_limit+10, number_limit+100): + + size_limit = 4096 + + j = size_limit/(i * 2) + + size = 0 + metadata = {} + while len(metadata.keys()) < i: + key = Utils.create_name() + val = Utils.create_name() + + if len(key) > j: + key = key[:j] + val = val[:j] + + size += len(key) + len(val) + metadata[key] = val + + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + + if i <= number_limit: + self.assert_(file.write()) + self.assert_status(201) + self.assert_(file.sync_metadata()) + self.assert_status(202) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + file.metadata = {} + self.assert_(file.write()) + self.assert_status(201) + file.metadata = metadata + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(400) + + def testContentTypeGuessing(self): + file_types = {'wav': 'audio/x-wav', 'txt': 'text/plain', + 'zip': 'application/zip'} + + container = self.env.account.container(Utils.create_name()) + self.assert_(container.create()) + + for i in file_types.keys(): + file = container.file(Utils.create_name() + '.' + i) + file.write('', cfg={'no_content_type':True}) + + file_types_read = {} + for i in container.files(parms={'format': 'json'}): + file_types_read[i['name'].split('.')[1]] = i['content_type'] + + self.assertEquals(file_types, file_types_read) + + def testRangedGets(self): + file_length = 10000 + range_size = file_length/10 + file = self.env.container.file(Utils.create_name()) + data = file.write_random(file_length) + + for i in range(0, file_length, range_size): + range_string = 'bytes=%d-%d' % (i, i+range_size-1) + hdrs = {'Range': range_string} + self.assert_(data[i:i+range_size] == file.read(hdrs=hdrs), + range_string) + + range_string = 'bytes=-%d' % (i) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data[-i:], range_string) + + range_string = 'bytes=%d-' % (i) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data[i-file_length:], + range_string) + + range_string = 'bytes=%d-%d' % (file_length+1000, file_length+2000) + hdrs = {'Range': range_string} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(416) + + range_string = 'bytes=%d-%d' % (file_length-1000, file_length+2000) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data[-1000:], range_string) + + hdrs = {'Range': '0-4'} + self.assert_(file.read(hdrs=hdrs) == data, range_string) + + for r in ('BYTES=0-999', 'bytes = 0-999', 'BYTES = 0 - 999', + 'bytes = 0 - 999', 'bytes=0 - 999', 'bytes=0-999 '): + + self.assert_(file.read(hdrs={'Range': r}) == data[0:1000]) + + def testFileSizeLimit(self): + limit = 5*2**30 + 2 + tsecs = 3 + + for i in (limit-100, limit-10, limit-1, limit, limit+1, limit+10, + limit+100): + + file = self.env.container.file(Utils.create_name()) + + if i <= limit: + self.assert_(timeout(tsecs, file.write, + cfg={'set_content_length':i})) + else: + self.assertRaises(ResponseError, timeout, tsecs, + file.write, cfg={'set_content_length':i}) + + def testNoContentLengthForPut(self): + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, 'testing', + cfg={'no_content_length':True}) + self.assert_status(411) + + def testDelete(self): + file = self.env.container.file(Utils.create_name()) + file.write_random(self.env.file_size) + + self.assert_(file.name in self.env.container.files()) + self.assert_(file.delete()) + self.assert_(file.name not in self.env.container.files()) + + def testBadHeaders(self): + file_length = 100 + + # no content type on puts should be ok + file = self.env.container.file(Utils.create_name()) + file.write_random(file_length, cfg={'no_content_type':True}) + self.assert_status(201) + + # content length x + self.assertRaises(ResponseError, file.write_random, file_length, + hdrs={'Content-Length':'X'}, cfg={'no_content_length':True}) + self.assert_status(400) + + # bad request types + for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'): + self.env.account.conn.make_request(req) + self.assert_status(405) + + # bad range headers + self.assert_(len(file.read(hdrs={'Range':'parsecs=8-12'})) == \ + file_length) + self.assert_status(200) + + def testMetadataLengthLimits(self): + key_limit, value_limit = 128, 256 + lengths = [[key_limit, value_limit], [key_limit, value_limit+1], \ + [key_limit+1, value_limit], [key_limit, 0], \ + [key_limit, value_limit*10], [key_limit*10, value_limit]] + + for l in lengths: + metadata = {'a'*l[0]: 'b'*l[1]} + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + + if l[0] <= key_limit and l[1] <= value_limit: + self.assert_(file.write()) + self.assert_status(201) + self.assert_(file.sync_metadata()) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + file.metadata = {} + self.assert_(file.write()) + self.assert_status(201) + file.metadata = metadata + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(400) + + def testEtagWayoff(self): + file = self.env.container.file(Utils.create_name()) + hdrs = {'etag': 'reallylonganddefinitelynotavalidetagvalue'} + self.assertRaises(ResponseError, file.write_random, hdrs=hdrs) + self.assert_status(422) + + def testFileCreate(self): + for i in range(10): + file = self.env.container.file(Utils.create_name()) + data = file.write_random() + self.assert_status(201) + self.assert_(data == file.read()) + self.assert_status(200) + + def testHead(self): + file_name = Utils.create_name() + content_type = Utils.create_name() + + file = self.env.container.file(file_name) + file.content_type = content_type + file.write_random(self.env.file_size) + + md5 = file.md5 + + file = self.env.container.file(file_name) + info = file.info() + + self.assert_status(200) + self.assertEquals(info['content_length'], self.env.file_size) + self.assertEquals(info['etag'], md5) + self.assertEquals(info['content_type'], content_type) + self.assert_(info.has_key('last_modified')) + + def testDeleteOfFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.delete) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.delete) + self.assert_status(404) + + def testHeadOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.info) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.info) + self.assert_status(404) + + def testMetadataOnPost(self): + file = self.env.container.file(Utils.create_name()) + file.write_random(self.env.file_size) + + for i in range(10): + metadata = {} + for i in range(10): + metadata[Utils.create_name()] = Utils.create_name() + + file.metadata = metadata + self.assert_(file.sync_metadata()) + self.assert_status(202) + + file = self.env.container.file(file.name) + self.assert_(file.initialize()) + self.assert_status(200) + self.assertEquals(file.metadata, metadata) + + def testGetContentType(self): + file_name = Utils.create_name() + content_type = Utils.create_name() + + file = self.env.container.file(file_name) + file.content_type = content_type + file.write_random() + + file = self.env.container.file(file_name) + file.read() + + self.assertEquals(content_type, file.content_type) + + def testGetOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.read) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.read) + self.assert_status(404) + + def testPostOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + file.metadata['Field'] = 'Value' + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + file.metadata['Field'] = 'Value' + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(404) + + def testMetadataOnPut(self): + for i in range(10): + metadata = {} + for j in range(10): + metadata[Utils.create_name()] = Utils.create_name() + + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + file.write_random(self.env.file_size) + + file = self.env.container.file(file.name) + self.assert_(file.initialize()) + self.assert_status(200) + self.assertEquals(file.metadata, metadata) + + def testSerialization(self): + container = self.env.account.container(Utils.create_name()) + self.assert_(container.create()) + + files = [] + for i in (0, 1, 10, 100, 1000, 10000): + files.append({'name': Utils.create_name(), \ + 'content_type': Utils.create_name(), 'bytes':i}) + + write_time = time.time() + for f in files: + file = container.file(f['name']) + file.content_type = f['content_type'] + file.write_random(f['bytes']) + + f['hash'] = file.md5 + f['json'] = False + f['xml'] = False + write_time = time.time() - write_time + + for format in ['json', 'xml']: + for file in container.files(parms={'format': format}): + found = False + for f in files: + if f['name'] != file['name']: + continue + + self.assertEquals(file['content_type'], + f['content_type']) + self.assertEquals(int(file['bytes']), f['bytes']) + + d = datetime.strptime(file['last_modified'].\ + split('.')[0], "%Y-%m-%dT%H:%M:%S") + lm = time.mktime(d.timetuple()) + + if f.has_key('last_modified'): + self.assertEquals(f['last_modified'], lm) + else: + f['last_modified'] = lm + + f[format] = True + found = True + + self.assert_(found, 'Unexpected file %s found in ' \ + '%s listing' % (file['name'], format)) + + headers = dict(self.env.conn.response.getheaders()) + if format == 'json': + self.assertEquals(headers['content-type'], + 'application/json; charset=utf8') + elif format == 'xml': + self.assertEquals(headers['content-type'], + 'application/xml; charset=utf8') + + lm_diff = max([f['last_modified'] for f in files]) - \ + min([f['last_modified'] for f in files]) + self.assert_(lm_diff < write_time + 1, 'Diff in last ' + \ + 'modified times should be less than time to write files') + + for f in files: + for format in ['json', 'xml']: + self.assert_(f[format], 'File %s not found in %s listing' \ + % (f['name'], format)) + + def testStackedOverwrite(self): + file = self.env.container.file(Utils.create_name()) + + for i in range(1, 11): + data = file.write_random(512) + file.write(data) + + self.assert_(file.read() == data) + + def testTooLongName(self): + file = self.env.container.file('x'*1025) + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + + def testZeroByteFile(self): + file = self.env.container.file(Utils.create_name()) + + self.assert_(file.write('')) + self.assert_(file.name in self.env.container.files()) + self.assert_(file.read() == '') + + def testEtagResponse(self): + file = self.env.container.file(Utils.create_name()) + + data = StringIO.StringIO(file.write_random(512)) + etag = File.compute_md5sum(data) + + headers = dict(self.env.conn.response.getheaders()) + self.assert_('etag' in headers.keys()) + + header_etag = headers['etag'].strip('"') + self.assertEquals(etag, header_etag) + + def testChunkedPut(self): + data = File.random_data(10000) + etag = File.compute_md5sum(data) + + for i in (1, 10, 100, 1000): + file = self.env.container.file(Utils.create_name()) + + for j in chunks(data, i): + file.chunked_write(j) + + self.assert_(file.chunked_write()) + self.assert_(data == file.read()) + + info = file.info() + self.assertEquals(etag, info['etag']) + +class TestFileUTF8(Base2, TestFile): + set_up = False + +class TestFileComparisonEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config['account']) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_count = 20 + cls.file_size = 128 + cls.files = list() + for x in range(cls.file_count): + file = cls.container.file(Utils.create_name()) + file.write_random(cls.file_size) + cls.files.append(file) + + cls.time_old = time.asctime(time.localtime(time.time()-86400)) + cls.time_new = time.asctime(time.localtime(time.time()+86400)) + +class TestFileComparison(Base): + env = TestFileComparisonEnv + set_up = False + + def testIfMatch(self): + for file in self.env.files: + hdrs = {'If-Match': file.md5} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Match': 'bogus'} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + def testIfNoneMatch(self): + for file in self.env.files: + hdrs = {'If-None-Match': 'bogus'} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-None-Match': file.md5} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(304) + + def testIfModifiedSince(self): + for file in self.env.files: + hdrs = {'If-Modified-Since': self.env.time_old} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Modified-Since': self.env.time_new} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(304) + + def testIfUnmodifiedSince(self): + for file in self.env.files: + hdrs = {'If-Unmodified-Since': self.env.time_new} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Unmodified-Since': self.env.time_old} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + def testIfMatchAndUnmodified(self): + for file in self.env.files: + hdrs = {'If-Match': file.md5, 'If-Unmodified-Since': \ + self.env.time_new} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Match': 'bogus', 'If-Unmodified-Since': \ + self.env.time_new} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + hdrs = {'If-Match': file.md5, 'If-Unmodified-Since': \ + self.env.time_old} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + +class TestFileComparisonUTF8(Base2, TestFileComparison): + set_up = False + +if __name__ == '__main__': + unittest.main() diff --git a/test/probe/common.py b/test/probe/common.py new file mode 100644 index 0000000000..8465782f0c --- /dev/null +++ b/test/probe/common.py @@ -0,0 +1,96 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import kill +from signal import SIGTERM +from subprocess import call, Popen +from time import sleep + +from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.client import get_auth +from swift.common.ring import Ring + + +def kill_pids(pids): + for pid in pids.values(): + try: + kill(pid, SIGTERM) + except: + pass + + +def reset_environment(): + call(['resetswift']) + pids = {} + try: + pids['auth'] = Popen(['/usr/bin/swift-auth-server', + '/etc/swift/auth-server.conf']).pid + pids['proxy'] = Popen(['/usr/bin/swift-proxy-server', + '/etc/swift/proxy-server.conf']).pid + port2server = {} + for s, p in (('account', 6002), ('container', 6001), ('object', 6000)): + for n in xrange(1, 5): + pids['%s%d' % (s, n)] = \ + Popen(['/usr/bin/swift-%s-server' % s, + '/etc/swift/%s-server/%d.conf' % (s, n)]).pid + port2server[p + (n * 10)] = '%s%d' % (s, n) + account_ring = Ring('/etc/swift/account.ring.gz') + container_ring = Ring('/etc/swift/container.ring.gz') + object_ring = Ring('/etc/swift/object.ring.gz') + sleep(5) + conn = http_connect('127.0.0.1', '11000', 'POST', '/recreate_accounts') + resp = conn.getresponse() + if resp.status != 200: + raise Exception('Recreating accounts failed. (%d)' % resp.status) + url, token = \ + get_auth('http://127.0.0.1:11000/auth', 'test:tester', 'testing') + account = url.split('/')[-1] + except BaseException, err: + kill_pids(pids) + raise err + return pids, port2server, account_ring, container_ring, object_ring, url, \ + token, account + + +def get_to_final_state(): + ps = [] + for job in ('account-replicator', 'container-replicator', + 'object-replicator'): + for n in xrange(1, 5): + ps.append(Popen(['/usr/bin/swift-%s' % job, + '/etc/swift/%s-server/%d.conf' % + (job.split('-')[0], n), + 'once'])) + for p in ps: + p.wait() + ps = [] + for job in ('container-updater', 'object-updater'): + for n in xrange(1, 5): + ps.append(Popen(['/usr/bin/swift-%s' % job, + '/etc/swift/%s-server/%d.conf' % + (job.split('-')[0], n), + 'once'])) + for p in ps: + p.wait() + ps = [] + for job in ('account-replicator', 'container-replicator', + 'object-replicator'): + for n in xrange(1, 5): + ps.append(Popen(['/usr/bin/swift-%s' % job, + '/etc/swift/%s-server/%d.conf' % + (job.split('-')[0], n), + 'once'])) + for p in ps: + p.wait() diff --git a/test/probe/test_account_failures.py b/test/probe/test_account_failures.py new file mode 100755 index 0000000000..2e84cc0640 --- /dev/null +++ b/test/probe/test_account_failures.py @@ -0,0 +1,174 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from os import kill +from signal import SIGTERM +from subprocess import Popen +from time import sleep + +from swift.common import client +from common import get_to_final_state, kill_pids, reset_environment + + +class TestAccountFailures(unittest.TestCase): + + def setUp(self): + self.pids, self.port2server, self.account_ring, self.container_ring, \ + self.object_ring, self.url, self.token, self.account = \ + reset_environment() + + def tearDown(self): + kill_pids(self.pids) + + def test_main(self): + container1 = 'container1' + client.put_container(self.url, self.token, container1) + container2 = 'container2' + client.put_container(self.url, self.token, container2) + self.assert_(client.head_account(self.url, self.token), (2, 0, 0)) + containers = client.get_account(self.url, self.token) + found1 = False + found2 = False + for c in containers: + if c['name'] == container1: + found1 = True + self.assertEquals(c['count'], 0) + self.assertEquals(c['bytes'], 0) + elif c['name'] == container2: + found2 = True + self.assertEquals(c['count'], 0) + self.assertEquals(c['bytes'], 0) + self.assert_(found1) + self.assert_(found2) + + client.put_object(self.url, self.token, container2, 'object1', '1234') + self.assert_(client.head_account(self.url, self.token), (2, 0, 0)) + containers = client.get_account(self.url, self.token) + found1 = False + found2 = False + for c in containers: + if c['name'] == container1: + found1 = True + self.assertEquals(c['count'], 0) + self.assertEquals(c['bytes'], 0) + elif c['name'] == container2: + found2 = True + self.assertEquals(c['count'], 0) + self.assertEquals(c['bytes'], 0) + self.assert_(found1) + self.assert_(found2) + + get_to_final_state() + containers = client.get_account(self.url, self.token) + self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) + found1 = False + found2 = False + for c in containers: + if c['name'] == container1: + found1 = True + self.assertEquals(c['count'], 0) + self.assertEquals(c['bytes'], 0) + elif c['name'] == container2: + found2 = True + self.assertEquals(c['count'], 1) + self.assertEquals(c['bytes'], 4) + self.assert_(found1) + self.assert_(found2) + + apart, anodes = self.account_ring.get_nodes(self.account) + kill(self.pids[self.port2server[anodes[0]['port']]], SIGTERM) + + client.delete_container(self.url, self.token, container1) + client.put_object(self.url, self.token, container2, 'object2', '12345') + self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) + containers = client.get_account(self.url, self.token) + found1 = False + found2 = False + for c in containers: + if c['name'] == container1: + found1 = True + elif c['name'] == container2: + found2 = True + self.assertEquals(c['count'], 1) + self.assertEquals(c['bytes'], 4) + self.assert_(not found1) + self.assert_(found2) + + ps = [] + for n in xrange(1, 5): + ps.append(Popen(['/usr/bin/swift-container-updater', + '/etc/swift/container-server/%d.conf' % n, + 'once'])) + for p in ps: + p.wait() + self.assert_(client.head_account(self.url, self.token), (2, 2, 9)) + containers = client.get_account(self.url, self.token) + found1 = False + found2 = False + for c in containers: + if c['name'] == container1: + found1 = True + elif c['name'] == container2: + found2 = True + self.assertEquals(c['count'], 2) + self.assertEquals(c['bytes'], 9) + self.assert_(not found1) + self.assert_(found2) + + self.pids[self.port2server[anodes[0]['port']]] = \ + Popen(['/usr/bin/swift-account-server', + '/etc/swift/account-server/%d.conf' % + ((anodes[0]['port'] - 6002) / 10)]).pid + sleep(2) + # This is the earlier object count and bytes because the first node + # doesn't have the newest udpates yet. + self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) + containers = client.get_account(self.url, self.token) + found1 = False + found2 = False + for c in containers: + if c['name'] == container1: + found1 = True + elif c['name'] == container2: + found2 = True + # This is the earlier count and bytes because the first node + # doesn't have the newest udpates yet. + self.assertEquals(c['count'], 1) + self.assertEquals(c['bytes'], 4) + # This okay because the first node hasn't got the update that + # container1 was deleted yet. + self.assert_(found1) + self.assert_(found2) + + get_to_final_state() + containers = client.get_account(self.url, self.token) + self.assert_(client.head_account(self.url, self.token), (2, 2, 9)) + found1 = False + found2 = False + for c in containers: + if c['name'] == container1: + found1 = True + elif c['name'] == container2: + found2 = True + self.assertEquals(c['count'], 2) + self.assertEquals(c['bytes'], 9) + self.assert_(not found1) + self.assert_(found2) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py new file mode 100755 index 0000000000..cc042cb1c1 --- /dev/null +++ b/test/probe/test_container_failures.py @@ -0,0 +1,343 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from os import kill +from signal import SIGTERM +from subprocess import Popen +from time import sleep +from uuid import uuid4 + +from swift.common import client + +from common import get_to_final_state, kill_pids, reset_environment + + +class TestContainerFailures(unittest.TestCase): + + def setUp(self): + self.pids, self.port2server, self.account_ring, self.container_ring, \ + self.object_ring, self.url, self.token, self.account = \ + reset_environment() + + def tearDown(self): + kill_pids(self.pids) + + def test_first_node_fail(self): + container = 'container-%s' % uuid4() + client.put_container(self.url, self.token, container) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + + object1 = 'object1' + client.put_object(self.url, self.token, container, object1, 'test') + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + cpart, cnodes = self.container_ring.get_nodes(self.account, container) + kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM) + + client.delete_object(self.url, self.token, container, object1) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + self.pids[self.port2server[cnodes[0]['port']]] = \ + Popen(['/usr/bin/swift-container-server', + '/etc/swift/container-server/%d.conf' % + ((cnodes[0]['port'] - 6001) / 10)]).pid + sleep(2) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + # This okay because the first node hasn't got the update that the + # object was deleted yet. + self.assert_(object1 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + # This fails because all three nodes have to indicate deletion before + # we tell the user it worked. Since the first node 409s (it hasn't got + # the update that the object was deleted yet), the whole must 503 + # (until every is synced up, then the delete would work). + exc = None + try: + client.delete_container(self.url, self.token, container) + except client.ClientException, err: + exc = err + self.assert_(exc) + self.assert_(exc.http_status, 503) + # Unfortunately, the following might pass or fail, depending on the + # position of the account server associated with the first container + # server we had killed. If the associated happens to be the first + # account server, this'll pass, otherwise the first account server will + # serve the listing and not have the container. + # self.assert_(container in [c['name'] for c in + # client.get_account(self.url, self.token)]) + + object2 = 'object2' + # This will work because at least one (in this case, just one) account + # server has to indicate the container exists for the put to continue. + client.put_object(self.url, self.token, container, object2, 'test') + # First node still doesn't know object1 was deleted yet; this is okay. + self.assert_(object1 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + # And, of course, our new object2 exists. + self.assert_(object2 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + get_to_final_state() + # Our container delete never "finalized" because we started using it + # before the delete settled. + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + # And, so our object2 should still exist and object1's delete should + # have finalized. + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + self.assert_(object2 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + def test_second_node_fail(self): + container = 'container-%s' % uuid4() + client.put_container(self.url, self.token, container) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + + object1 = 'object1' + client.put_object(self.url, self.token, container, object1, 'test') + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + cpart, cnodes = self.container_ring.get_nodes(self.account, container) + kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM) + + client.delete_object(self.url, self.token, container, object1) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + self.pids[self.port2server[cnodes[1]['port']]] = \ + Popen(['/usr/bin/swift-container-server', + '/etc/swift/container-server/%d.conf' % + ((cnodes[1]['port'] - 6001) / 10)]).pid + sleep(2) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + # This fails because all three nodes have to indicate deletion before + # we tell the user it worked. Since the first node 409s (it hasn't got + # the update that the object was deleted yet), the whole must 503 + # (until every is synced up, then the delete would work). + exc = None + try: + client.delete_container(self.url, self.token, container) + except client.ClientException, err: + exc = err + self.assert_(exc) + self.assert_(exc.http_status, 503) + # Unfortunately, the following might pass or fail, depending on the + # position of the account server associated with the first container + # server we had killed. If the associated happens to be the first + # account server, this'll pass, otherwise the first account server will + # serve the listing and not have the container. + # self.assert_(container in [c['name'] for c in + # client.get_account(self.url, self.token)]) + + object2 = 'object2' + # This will work because at least one (in this case, just one) account + # server has to indicate the container exists for the put to continue. + client.put_object(self.url, self.token, container, object2, 'test') + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + # And, of course, our new object2 exists. + self.assert_(object2 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + get_to_final_state() + # Our container delete never "finalized" because we started using it + # before the delete settled. + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + # And, so our object2 should still exist and object1's delete should + # have finalized. + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + self.assert_(object2 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + def test_first_two_nodes_fail(self): + container = 'container-%s' % uuid4() + client.put_container(self.url, self.token, container) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + + object1 = 'object1' + client.put_object(self.url, self.token, container, object1, 'test') + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + cpart, cnodes = self.container_ring.get_nodes(self.account, container) + for x in xrange(2): + kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM) + + client.delete_object(self.url, self.token, container, object1) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + for x in xrange(2): + self.pids[self.port2server[cnodes[x]['port']]] = \ + Popen(['/usr/bin/swift-container-server', + '/etc/swift/container-server/%d.conf' % + ((cnodes[x]['port'] - 6001) / 10)]).pid + sleep(2) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + # This okay because the first node hasn't got the update that the + # object was deleted yet. + self.assert_(object1 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + # This fails because all three nodes have to indicate deletion before + # we tell the user it worked. Since the first node 409s (it hasn't got + # the update that the object was deleted yet), the whole must 503 + # (until every is synced up, then the delete would work). + exc = None + try: + client.delete_container(self.url, self.token, container) + except client.ClientException, err: + exc = err + self.assert_(exc) + self.assert_(exc.http_status, 503) + # Unfortunately, the following might pass or fail, depending on the + # position of the account server associated with the first container + # server we had killed. If the associated happens to be the first + # account server, this'll pass, otherwise the first account server will + # serve the listing and not have the container. + # self.assert_(container in [c['name'] for c in + # client.get_account(self.url, self.token)]) + + object2 = 'object2' + # This will work because at least one (in this case, just one) account + # server has to indicate the container exists for the put to continue. + client.put_object(self.url, self.token, container, object2, 'test') + # First node still doesn't know object1 was deleted yet; this is okay. + self.assert_(object1 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + # And, of course, our new object2 exists. + self.assert_(object2 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + get_to_final_state() + # Our container delete never "finalized" because we started using it + # before the delete settled. + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + # And, so our object2 should still exist and object1's delete should + # have finalized. + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + self.assert_(object2 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + def test_last_two_nodes_fail(self): + container = 'container-%s' % uuid4() + client.put_container(self.url, self.token, container) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + + object1 = 'object1' + client.put_object(self.url, self.token, container, object1, 'test') + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + cpart, cnodes = self.container_ring.get_nodes(self.account, container) + for x in (1, 2): + kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM) + + client.delete_object(self.url, self.token, container, object1) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + for x in (1, 2): + self.pids[self.port2server[cnodes[x]['port']]] = \ + Popen(['/usr/bin/swift-container-server', + '/etc/swift/container-server/%d.conf' % + ((cnodes[x]['port'] - 6001) / 10)]).pid + sleep(2) + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + # This fails because all three nodes have to indicate deletion before + # we tell the user it worked. Since the first node 409s (it hasn't got + # the update that the object was deleted yet), the whole must 503 + # (until every is synced up, then the delete would work). + exc = None + try: + client.delete_container(self.url, self.token, container) + except client.ClientException, err: + exc = err + self.assert_(exc) + self.assert_(exc.http_status, 503) + # Unfortunately, the following might pass or fail, depending on the + # position of the account server associated with the first container + # server we had killed. If the associated happens to be the first + # account server, this'll pass, otherwise the first account server will + # serve the listing and not have the container. + # self.assert_(container in [c['name'] for c in + # client.get_account(self.url, self.token)]) + + object2 = 'object2' + # This will work because at least one (in this case, just one) account + # server has to indicate the container exists for the put to continue. + client.put_object(self.url, self.token, container, object2, 'test') + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + # And, of course, our new object2 exists. + self.assert_(object2 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + get_to_final_state() + # Our container delete never "finalized" because we started using it + # before the delete settled. + self.assert_(container in [c['name'] for c in + client.get_account(self.url, self.token)]) + # And, so our object2 should still exist and object1's delete should + # have finalized. + self.assert_(object1 not in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + self.assert_(object2 in [o['name'] for o in + client.get_container(self.url, self.token, container)]) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py new file mode 100755 index 0000000000..841a7e6e7d --- /dev/null +++ b/test/probe/test_object_async_update.py @@ -0,0 +1,68 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from os import kill +from signal import SIGTERM +from subprocess import Popen +from time import sleep +from uuid import uuid4 + +from swift.common import client, direct_client + +from common import kill_pids, reset_environment + + +class TestObjectAsyncUpdate(unittest.TestCase): + + def setUp(self): + self.pids, self.port2server, self.account_ring, self.container_ring, \ + self.object_ring, self.url, self.token, self.account = \ + reset_environment() + + def tearDown(self): + kill_pids(self.pids) + + def test_main(self): + container = 'container-%s' % uuid4() + client.put_container(self.url, self.token, container) + apart, anodes = self.account_ring.get_nodes(self.account) + anode = anodes[0] + cpart, cnodes = self.container_ring.get_nodes(self.account, container) + cnode = cnodes[0] + kill(self.pids[self.port2server[cnode['port']]], SIGTERM) + obj = 'object-%s' % uuid4() + client.put_object(self.url, self.token, container, obj, '') + self.pids[self.port2server[cnode['port']]] = \ + Popen(['/usr/bin/swift-container-server', + '/etc/swift/container-server/%d.conf' % + ((cnode['port'] - 6001) / 10)]).pid + sleep(2) + self.assert_(not direct_client.direct_get_container(cnode, cpart, + self.account, container)) + ps = [] + for n in xrange(1, 5): + ps.append(Popen(['/usr/bin/swift-object-updater', + '/etc/swift/object-server/%d.conf' % n, 'once'])) + for p in ps: + p.wait() + objs = [o['name'] for o in direct_client.direct_get_container(cnode, + cpart, self.account, container)] + self.assert_(obj in objs) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py new file mode 100755 index 0000000000..885a6fb198 --- /dev/null +++ b/test/probe/test_object_handoff.py @@ -0,0 +1,218 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from os import kill +from signal import SIGTERM +from subprocess import call, Popen +from time import sleep +from uuid import uuid4 + +from swift.common import client, direct_client + +from common import kill_pids, reset_environment + + +class TestObjectHandoff(unittest.TestCase): + + def setUp(self): + self.pids, self.port2server, self.account_ring, self.container_ring, \ + self.object_ring, self.url, self.token, self.account = \ + reset_environment() + + def tearDown(self): + kill_pids(self.pids) + + def test_main(self): + container = 'container-%s' % uuid4() + client.put_container(self.url, self.token, container) + apart, anodes = self.account_ring.get_nodes(self.account) + anode = anodes[0] + cpart, cnodes = self.container_ring.get_nodes(self.account, container) + cnode = cnodes[0] + obj = 'object-%s' % uuid4() + opart, onodes = self.object_ring.get_nodes(self.account, container, obj) + onode = onodes[0] + kill(self.pids[self.port2server[onode['port']]], SIGTERM) + client.put_object(self.url, self.token, container, obj, 'VERIFY') + odata = client.get_object(self.url, self.token, container, obj)[-1] + if odata != 'VERIFY': + raise Exception('Object GET did not return VERIFY, instead it ' + 'returned: %s' % repr(odata)) + # Kill all primaries to ensure GET handoff works + for node in onodes[1:]: + kill(self.pids[self.port2server[node['port']]], SIGTERM) + odata = client.get_object(self.url, self.token, container, obj)[-1] + if odata != 'VERIFY': + raise Exception('Object GET did not return VERIFY, instead it ' + 'returned: %s' % repr(odata)) + for node in onodes[1:]: + self.pids[self.port2server[node['port']]] = Popen([ + '/usr/bin/swift-object-server', + '/etc/swift/object-server/%d.conf' % + ((node['port'] - 6000) / 10)]).pid + sleep(2) + # We've indirectly verified the handoff node has the object, but let's + # directly verify it. + another_onode = self.object_ring.get_more_nodes(opart).next() + odata = direct_client.direct_get_object(another_onode, opart, + self.account, container, obj)[-1] + if odata != 'VERIFY': + raise Exception('Direct object GET did not return VERIFY, instead ' + 'it returned: %s' % repr(odata)) + objs = [o['name'] for o in + client.get_container(self.url, self.token, container)] + if obj not in objs: + raise Exception('Container listing did not know about object') + for cnode in cnodes: + objs = [o['name'] for o in + direct_client.direct_get_container(cnode, cpart, + self.account, container)] + if obj not in objs: + raise Exception( + 'Container server %s:%s did not know about object' % + (cnode['ip'], cnode['port'])) + self.pids[self.port2server[onode['port']]] = Popen([ + '/usr/bin/swift-object-server', + '/etc/swift/object-server/%d.conf' % + ((onode['port'] - 6000) / 10)]).pid + sleep(2) + exc = False + try: + direct_client.direct_get_object(onode, opart, self.account, + container, obj) + except: + exc = True + if not exc: + raise Exception('Previously downed object server had test object') + # Run the extra server last so it'll remove it's extra partition + ps = [] + for n in onodes: + ps.append(Popen(['/usr/bin/swift-object-replicator', + '/etc/swift/object-server/%d.conf' % + ((n['port'] - 6000) / 10), 'once'])) + for p in ps: + p.wait() + call(['/usr/bin/swift-object-replicator', + '/etc/swift/object-server/%d.conf' % + ((another_onode['port'] - 6000) / 10), 'once']) + odata = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[-1] + if odata != 'VERIFY': + raise Exception('Direct object GET did not return VERIFY, instead ' + 'it returned: %s' % repr(odata)) + exc = False + try: + direct_client.direct_get_object(another_onode, opart, self.account, + container, obj) + except: + exc = True + if not exc: + raise Exception('Handoff object server still had test object') + + kill(self.pids[self.port2server[onode['port']]], SIGTERM) + client.post_object(self.url, self.token, container, obj, + {'probe': 'value'}) + ometadata = client.head_object(self.url, self.token, container, obj)[-1] + if ometadata.get('probe') != 'value': + raise Exception('Metadata incorrect, was %s' % repr(ometadata)) + exc = False + try: + direct_client.direct_get_object(another_onode, opart, self.account, + container, obj) + except: + exc = True + if not exc: + raise Exception('Handoff server claimed it had the object when ' + 'it should not have it') + self.pids[self.port2server[onode['port']]] = Popen([ + '/usr/bin/swift-object-server', + '/etc/swift/object-server/%d.conf' % + ((onode['port'] - 6000) / 10)]).pid + sleep(2) + ometadata = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[-2] + if ometadata.get('probe') == 'value': + raise Exception('Previously downed object server had the new ' + 'metadata when it should not have it') + # Run the extra server last so it'll remove it's extra partition + ps = [] + for n in onodes: + ps.append(Popen(['/usr/bin/swift-object-replicator', + '/etc/swift/object-server/%d.conf' % + ((n['port'] - 6000) / 10), 'once'])) + for p in ps: + p.wait() + call(['/usr/bin/swift-object-replicator', + '/etc/swift/object-server/%d.conf' % + ((another_onode['port'] - 6000) / 10), 'once']) + ometadata = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[-2] + if ometadata.get('probe') != 'value': + raise Exception( + 'Previously downed object server did not have the new metadata') + + kill(self.pids[self.port2server[onode['port']]], SIGTERM) + client.delete_object(self.url, self.token, container, obj) + exc = False + try: + client.head_object(self.url, self.token, container, obj) + except: + exc = True + if not exc: + raise Exception('Regular object HEAD was still successful') + objs = [o['name'] for o in + client.get_container(self.url, self.token, container)] + if obj in objs: + raise Exception('Container listing still knew about object') + for cnode in cnodes: + objs = [o['name'] for o in + direct_client.direct_get_container(cnode, cpart, + self.account, container)] + if obj in objs: + raise Exception( + 'Container server %s:%s still knew about object' % + (cnode['ip'], cnode['port'])) + self.pids[self.port2server[onode['port']]] = Popen([ + '/usr/bin/swift-object-server', + '/etc/swift/object-server/%d.conf' % + ((onode['port'] - 6000) / 10)]).pid + sleep(2) + direct_client.direct_get_object(onode, opart, self.account, container, + obj) + # Run the extra server last so it'll remove it's extra partition + ps = [] + for n in onodes: + ps.append(Popen(['/usr/bin/swift-object-replicator', + '/etc/swift/object-server/%d.conf' % + ((n['port'] - 6000) / 10), 'once'])) + for p in ps: + p.wait() + call(['/usr/bin/swift-object-replicator', + '/etc/swift/object-server/%d.conf' % + ((another_onode['port'] - 6000) / 10), 'once']) + exc = False + try: + direct_client.direct_get_object(another_onode, opart, self.account, + container, obj) + except: + exc = True + if not exc: + raise Exception('Handoff object server still had the object') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/probe/test_running_with_each_type_down.py b/test/probe/test_running_with_each_type_down.py new file mode 100755 index 0000000000..a5682c7e3f --- /dev/null +++ b/test/probe/test_running_with_each_type_down.py @@ -0,0 +1,121 @@ +#!/usr/bin/python -u +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from os import kill +from signal import SIGTERM +from subprocess import Popen +from time import sleep + +from swift.common import client + +from common import get_to_final_state, kill_pids, reset_environment + + +class TestRunningWithEachTypeDown(unittest.TestCase): + + def setUp(self): + self.pids, self.port2server, self.account_ring, self.container_ring, \ + self.object_ring, self.url, self.token, self.account = \ + reset_environment() + + def tearDown(self): + kill_pids(self.pids) + + def test_main(self): + # TODO: This test "randomly" pass or doesn't pass; need to find out why + return + apart, anodes = self.account_ring.get_nodes(self.account) + kill(self.pids[self.port2server[anodes[0]['port']]], SIGTERM) + cpart, cnodes = \ + self.container_ring.get_nodes(self.account, 'container1') + kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM) + opart, onodes = \ + self.object_ring.get_nodes(self.account, 'container1', 'object1') + kill(self.pids[self.port2server[onodes[0]['port']]], SIGTERM) + + try: + client.put_container(self.url, self.token, 'container1') + except client.ClientException, err: + # This might 503 if one of the up container nodes tries to update + # the down account node. It'll still be saved on one node, but we + # can't assure the user. + pass + client.put_object(self.url, self.token, 'container1', 'object1', '1234') + get_to_final_state() + self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + found1 = False + for container in client.get_account(self.url, self.token): + if container['name'] == 'container1': + found1 = True + self.assertEquals(container['count'], 1) + self.assertEquals(container['bytes'], 4) + self.assert_(found1) + found1 = False + for obj in client.get_container(self.url, self.token, 'container1'): + if obj['name'] == 'object1': + found1 = True + self.assertEquals(obj['bytes'], 4) + self.assert_(found1) + + self.pids[self.port2server[anodes[0]['port']]] = \ + Popen(['/usr/bin/swift-account-server', + '/etc/swift/account-server/%d.conf' % + ((anodes[0]['port'] - 6002) / 10)]).pid + self.pids[self.port2server[cnodes[0]['port']]] = \ + Popen(['/usr/bin/swift-container-server', + '/etc/swift/container-server/%d.conf' % + ((cnodes[0]['port'] - 6001) / 10)]).pid + self.pids[self.port2server[onodes[0]['port']]] = \ + Popen(['/usr/bin/swift-object-server', + '/etc/swift/object-server/%d.conf' % + ((onodes[0]['port'] - 6000) / 10)]).pid + sleep(2) + self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + found1 = False + for container in client.get_account(self.url, self.token): + if container['name'] == 'container1': + found1 = True + # The account node was previously down. + self.assert_(not found1) + found1 = False + for obj in client.get_container(self.url, self.token, 'container1'): + if obj['name'] == 'object1': + found1 = True + self.assertEquals(obj['bytes'], 4) + # The first container node 404s, but the proxy will try the next node + # and succeed. + self.assert_(found1) + + get_to_final_state() + self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + found1 = False + for container in client.get_account(self.url, self.token): + if container['name'] == 'container1': + found1 = True + self.assertEquals(container['count'], 1) + self.assertEquals(container['bytes'], 4) + self.assert_(found1) + found1 = False + for obj in client.get_container(self.url, self.token, 'container1'): + if obj['name'] == 'object1': + found1 = True + self.assertEquals(obj['bytes'], 4) + self.assert_(found1) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 0000000000..0a6e2d3b1e --- /dev/null +++ b/test/unit/__init__.py @@ -0,0 +1,24 @@ +""" Swift tests """ + +from eventlet.green import socket + + +def readuntil2crlfs(fd): + rv = '' + lc = '' + crlfs = 0 + while crlfs < 2: + c = fd.read(1) + rv = rv + c + if c == '\r' and lc != '\n': + crlfs = 0 + if lc == '\r' and c == '\n': + crlfs += 1 + lc = c + return rv + + +def connect_tcp(hostport): + rv = socket.socket() + rv.connect(hostport) + return rv diff --git a/test/unit/account/__init__.py b/test/unit/account/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/account/test_auditor.py b/test/unit/account/test_auditor.py new file mode 100644 index 0000000000..f7678ec1c1 --- /dev/null +++ b/test/unit/account/test_auditor.py @@ -0,0 +1,28 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Tests + +import unittest +from swift.account import auditor + +class TestAuditor(unittest.TestCase): + + def test_placeholder(self): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py new file mode 100644 index 0000000000..c69fc2229d --- /dev/null +++ b/test/unit/account/test_reaper.py @@ -0,0 +1,28 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Tests + +import unittest +from swift.account import reaper + +class TestReaper(unittest.TestCase): + + def test_placeholder(self): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py new file mode 100644 index 0000000000..84ef0ed897 --- /dev/null +++ b/test/unit/account/test_server.py @@ -0,0 +1,889 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import os +import unittest +from shutil import rmtree +from StringIO import StringIO + +import simplejson +import xml.dom.minidom +from webob import Request + +from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT +from swift.common.utils import normalize_timestamp + + +class TestAccountController(unittest.TestCase): + """ Test swift.account_server.AccountController """ + def setUp(self): + """ Set up for testing swift.account_server.AccountController """ + self.testdir = os.path.join(os.path.dirname(__file__), 'account_server') + self.controller = AccountController( + {'devices': self.testdir, 'mount_check': 'false'}) + + def tearDown(self): + """ Tear down for testing swift.account_server.AccountController """ + try: + rmtree(self.testdir) + except OSError, err: + if err.errno != errno.ENOENT: + raise + + def test_DELETE_not_found(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 404) + + def test_DELETE_empty(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + + def test_DELETE_not_empty(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.DELETE(req) + # We now allow deleting non-empty accounts + self.assertEquals(resp.status_int, 204) + + def test_DELETE_now_empty(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '2', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + + def test_HEAD_not_found(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 404) + + def test_HEAD_empty_account(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers['x-account-container-count'], 0) + self.assertEquals(resp.headers['x-account-object-count'], 0) + self.assertEquals(resp.headers['x-account-bytes-used'], 0) + + def test_HEAD_with_containers(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '2', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers['x-account-container-count'], 2) + self.assertEquals(resp.headers['x-account-object-count'], 0) + self.assertEquals(resp.headers['x-account-bytes-used'], 0) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '1', + 'X-Bytes-Used': '2', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '2', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '3', + 'X-Bytes-Used': '4', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD', + 'HTTP_X_TIMESTAMP': '5'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers['x-account-container-count'], 2) + self.assertEquals(resp.headers['x-account-object-count'], 4) + self.assertEquals(resp.headers['x-account-bytes-used'], 6) + + def test_PUT_not_found(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-PUT-Timestamp': normalize_timestamp(1), + 'X-DELETE-Timestamp': normalize_timestamp(0), + 'X-Object-Count': '1', + 'X-Bytes-Used': '1', + 'X-Timestamp': normalize_timestamp(0)}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 404) + + def test_PUT(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 202) + + def test_PUT_after_DELETE(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1)}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': normalize_timestamp(1)}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(2)}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 403) + self.assertEquals(resp.body, 'Recently deleted') + + def test_GET_not_found_plain(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 404) + + def test_GET_not_found_json(self): + req = Request.blank('/sda1/p/a?format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 404) + + def test_GET_not_found_xml(self): + req = Request.blank('/sda1/p/a?format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 404) + + def test_GET_empty_account_plain(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + + def test_GET_empty_account_json(self): + req = Request.blank('/sda1/p/a?format=json', + environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + + def test_GET_empty_account_xml(self): + req = Request.blank('/sda1/p/a?format=xml', + environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + + def test_GET_over_limit(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?limit=%d' % + (ACCOUNT_LISTING_LIMIT + 1), environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 412) + + def test_GET_with_containers_plain(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '2', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body.strip().split('\n'), ['c1', 'c2']) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '1', + 'X-Bytes-Used': '2', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '2', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '3', + 'X-Bytes-Used': '4', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body.strip().split('\n'), ['c1', 'c2']) + self.assertEquals(resp.content_type, 'text/plain') + + def test_GET_with_containers_json(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '2', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(simplejson.loads(resp.body), + [{'count': 0, 'bytes': 0, 'name': 'c1'}, + {'count': 0, 'bytes': 0, 'name': 'c2'}]) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '1', + 'X-Bytes-Used': '2', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '2', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '3', + 'X-Bytes-Used': '4', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(simplejson.loads(resp.body), + [{'count': 1, 'bytes': 2, 'name': 'c1'}, + {'count': 3, 'bytes': 4, 'name': 'c2'}]) + self.assertEquals(resp.content_type, 'application/json') + + def test_GET_with_containers_xml(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '2', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'application/xml') + self.assertEquals(resp.status_int, 200) + dom = xml.dom.minidom.parseString(resp.body) + self.assertEquals(dom.firstChild.nodeName, 'account') + listing = \ + [n for n in dom.firstChild.childNodes if n.nodeName != '#text'] + self.assertEquals(len(listing), 2) + self.assertEquals(listing[0].nodeName, 'container') + container = [n for n in listing[0].childNodes if n.nodeName != '#text'] + self.assertEquals(sorted([n.nodeName for n in container]), + ['bytes', 'count', 'name']) + node = [n for n in container if n.nodeName == 'name'][0] + self.assertEquals(node.firstChild.nodeValue, 'c1') + node = [n for n in container if n.nodeName == 'count'][0] + self.assertEquals(node.firstChild.nodeValue, '0') + node = [n for n in container if n.nodeName == 'bytes'][0] + self.assertEquals(node.firstChild.nodeValue, '0') + self.assertEquals(listing[-1].nodeName, 'container') + container = \ + [n for n in listing[-1].childNodes if n.nodeName != '#text'] + self.assertEquals(sorted([n.nodeName for n in container]), + ['bytes', 'count', 'name']) + node = [n for n in container if n.nodeName == 'name'][0] + self.assertEquals(node.firstChild.nodeValue, 'c2') + node = [n for n in container if n.nodeName == 'count'][0] + self.assertEquals(node.firstChild.nodeValue, '0') + node = [n for n in container if n.nodeName == 'bytes'][0] + self.assertEquals(node.firstChild.nodeValue, '0') + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '1', + 'X-Bytes-Used': '2', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '2', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '3', + 'X-Bytes-Used': '4', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + dom = xml.dom.minidom.parseString(resp.body) + self.assertEquals(dom.firstChild.nodeName, 'account') + listing = \ + [n for n in dom.firstChild.childNodes if n.nodeName != '#text'] + self.assertEquals(len(listing), 2) + self.assertEquals(listing[0].nodeName, 'container') + container = [n for n in listing[0].childNodes if n.nodeName != '#text'] + self.assertEquals(sorted([n.nodeName for n in container]), + ['bytes', 'count', 'name']) + node = [n for n in container if n.nodeName == 'name'][0] + self.assertEquals(node.firstChild.nodeValue, 'c1') + node = [n for n in container if n.nodeName == 'count'][0] + self.assertEquals(node.firstChild.nodeValue, '1') + node = [n for n in container if n.nodeName == 'bytes'][0] + self.assertEquals(node.firstChild.nodeValue, '2') + self.assertEquals(listing[-1].nodeName, 'container') + container = [n for n in listing[-1].childNodes if n.nodeName != '#text'] + self.assertEquals(sorted([n.nodeName for n in container]), + ['bytes', 'count', 'name']) + node = [n for n in container if n.nodeName == 'name'][0] + self.assertEquals(node.firstChild.nodeValue, 'c2') + node = [n for n in container if n.nodeName == 'count'][0] + self.assertEquals(node.firstChild.nodeValue, '3') + node = [n for n in container if n.nodeName == 'bytes'][0] + self.assertEquals(node.firstChild.nodeValue, '4') + + def test_GET_limit_marker_plain(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + for c in xrange(5): + req = Request.blank('/sda1/p/a/c%d' % c, + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': str(c + 1), + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '2', + 'X-Bytes-Used': '3', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?limit=3', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body.strip().split('\n'), ['c0', 'c1', 'c2']) + req = Request.blank('/sda1/p/a?limit=3&marker=c2', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body.strip().split('\n'), ['c3', 'c4']) + + def test_GET_limit_marker_json(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + for c in xrange(5): + req = Request.blank('/sda1/p/a/c%d' % c, + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': str(c + 1), + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '2', + 'X-Bytes-Used': '3', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?limit=3&format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(simplejson.loads(resp.body), + [{'count': 2, 'bytes': 3, 'name': 'c0'}, + {'count': 2, 'bytes': 3, 'name': 'c1'}, + {'count': 2, 'bytes': 3, 'name': 'c2'}]) + req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(simplejson.loads(resp.body), + [{'count': 2, 'bytes': 3, 'name': 'c3'}, + {'count': 2, 'bytes': 3, 'name': 'c4'}]) + + def test_GET_limit_marker_xml(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + for c in xrange(5): + req = Request.blank('/sda1/p/a/c%d' % c, + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': str(c + 1), + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '2', + 'X-Bytes-Used': '3', + 'X-Timestamp': normalize_timestamp(c)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?limit=3&format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + dom = xml.dom.minidom.parseString(resp.body) + self.assertEquals(dom.firstChild.nodeName, 'account') + listing = \ + [n for n in dom.firstChild.childNodes if n.nodeName != '#text'] + self.assertEquals(len(listing), 3) + self.assertEquals(listing[0].nodeName, 'container') + container = [n for n in listing[0].childNodes if n.nodeName != '#text'] + self.assertEquals(sorted([n.nodeName for n in container]), + ['bytes', 'count', 'name']) + node = [n for n in container if n.nodeName == 'name'][0] + self.assertEquals(node.firstChild.nodeValue, 'c0') + node = [n for n in container if n.nodeName == 'count'][0] + self.assertEquals(node.firstChild.nodeValue, '2') + node = [n for n in container if n.nodeName == 'bytes'][0] + self.assertEquals(node.firstChild.nodeValue, '3') + self.assertEquals(listing[-1].nodeName, 'container') + container = [n for n in listing[-1].childNodes if n.nodeName != '#text'] + self.assertEquals(sorted([n.nodeName for n in container]), + ['bytes', 'count', 'name']) + node = [n for n in container if n.nodeName == 'name'][0] + self.assertEquals(node.firstChild.nodeValue, 'c2') + node = [n for n in container if n.nodeName == 'count'][0] + self.assertEquals(node.firstChild.nodeValue, '2') + node = [n for n in container if n.nodeName == 'bytes'][0] + self.assertEquals(node.firstChild.nodeValue, '3') + req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + dom = xml.dom.minidom.parseString(resp.body) + self.assertEquals(dom.firstChild.nodeName, 'account') + listing = \ + [n for n in dom.firstChild.childNodes if n.nodeName != '#text'] + self.assertEquals(len(listing), 2) + self.assertEquals(listing[0].nodeName, 'container') + container = [n for n in listing[0].childNodes if n.nodeName != '#text'] + self.assertEquals(sorted([n.nodeName for n in container]), + ['bytes', 'count', 'name']) + node = [n for n in container if n.nodeName == 'name'][0] + self.assertEquals(node.firstChild.nodeValue, 'c3') + node = [n for n in container if n.nodeName == 'count'][0] + self.assertEquals(node.firstChild.nodeValue, '2') + node = [n for n in container if n.nodeName == 'bytes'][0] + self.assertEquals(node.firstChild.nodeValue, '3') + self.assertEquals(listing[-1].nodeName, 'container') + container = [n for n in listing[-1].childNodes if n.nodeName != '#text'] + self.assertEquals(sorted([n.nodeName for n in container]), + ['bytes', 'count', 'name']) + node = [n for n in container if n.nodeName == 'name'][0] + self.assertEquals(node.firstChild.nodeValue, 'c4') + node = [n for n in container if n.nodeName == 'count'][0] + self.assertEquals(node.firstChild.nodeValue, '2') + node = [n for n in container if n.nodeName == 'bytes'][0] + self.assertEquals(node.firstChild.nodeValue, '3') + + def test_GET_accept_wildcard(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + req.accept = '*/*' + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body, 'c1\n') + + def test_GET_accept_application_wildcard(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + resp = self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/*' + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(len(simplejson.loads(resp.body)), 1) + + def test_GET_accept_json(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/json' + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(len(simplejson.loads(resp.body)), 1) + + def test_GET_accept_xml(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/xml' + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + dom = xml.dom.minidom.parseString(resp.body) + self.assertEquals(dom.firstChild.nodeName, 'account') + listing = \ + [n for n in dom.firstChild.childNodes if n.nodeName != '#text'] + self.assertEquals(len(listing), 1) + + def test_GET_accept_conflicting(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?format=plain', + environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/json' + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body, 'c1\n') + + def test_GET_prefix_delimeter_plain(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + for first in range(3): + req = Request.blank('/sda1/p/a/sub.%s' % first, + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + for second in range(3): + req = Request.blank('/sda1/p/a/sub.%s.%s' % (first, second), + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?delimiter=.', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body.strip().split('\n'), ['sub.']) + req = Request.blank('/sda1/p/a?prefix=sub.&delimiter=.', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body.strip().split('\n'), + ['sub.0', 'sub.0.', 'sub.1', 'sub.1.', 'sub.2', 'sub.2.']) + req = Request.blank('/sda1/p/a?prefix=sub.1.&delimiter=.', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body.strip().split('\n'), + ['sub.1.0', 'sub.1.1', 'sub.1.2']) + + def test_GET_prefix_delimeter_json(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + for first in range(3): + req = Request.blank('/sda1/p/a/sub.%s' % first, + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + for second in range(3): + req = Request.blank('/sda1/p/a/sub.%s.%s' % (first, second), + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?delimiter=.&format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals([n.get('name', 's:' + n.get('subdir', 'error')) + for n in simplejson.loads(resp.body)], ['s:sub.']) + req = Request.blank('/sda1/p/a?prefix=sub.&delimiter=.&format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals([n.get('name', 's:' + n.get('subdir', 'error')) + for n in simplejson.loads(resp.body)], + ['sub.0', 's:sub.0.', 'sub.1', 's:sub.1.', 'sub.2', 's:sub.2.']) + req = Request.blank('/sda1/p/a?prefix=sub.1.&delimiter=.&format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals([n.get('name', 's:' + n.get('subdir', 'error')) + for n in simplejson.loads(resp.body)], + ['sub.1.0', 'sub.1.1', 'sub.1.2']) + + def test_GET_prefix_delimeter_xml(self): + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + for first in range(3): + req = Request.blank('/sda1/p/a/sub.%s' % first, + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + for second in range(3): + req = Request.blank('/sda1/p/a/sub.%s.%s' % (first, second), + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Put-Timestamp': '1', + 'X-Delete-Timestamp': '0', + 'X-Object-Count': '0', + 'X-Bytes-Used': '0', + 'X-Timestamp': normalize_timestamp(0)}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a?delimiter=.&format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + dom = xml.dom.minidom.parseString(resp.body) + listing = [] + for node1 in dom.firstChild.childNodes: + if node1.nodeName == 'subdir': + listing.append('s:' + node1.attributes['name'].value) + elif node1.nodeName == 'container': + for node2 in node1.childNodes: + if node2.nodeName == 'name': + listing.append(node2.firstChild.nodeValue) + self.assertEquals(listing, ['s:sub.']) + req = Request.blank('/sda1/p/a?prefix=sub.&delimiter=.&format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + dom = xml.dom.minidom.parseString(resp.body) + listing = [] + for node1 in dom.firstChild.childNodes: + if node1.nodeName == 'subdir': + listing.append('s:' + node1.attributes['name'].value) + elif node1.nodeName == 'container': + for node2 in node1.childNodes: + if node2.nodeName == 'name': + listing.append(node2.firstChild.nodeValue) + self.assertEquals(listing, + ['sub.0', 's:sub.0.', 'sub.1', 's:sub.1.', 'sub.2', 's:sub.2.']) + req = Request.blank('/sda1/p/a?prefix=sub.1.&delimiter=.&format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + dom = xml.dom.minidom.parseString(resp.body) + listing = [] + for node1 in dom.firstChild.childNodes: + if node1.nodeName == 'subdir': + listing.append('s:' + node1.attributes['name'].value) + elif node1.nodeName == 'container': + for node2 in node1.childNodes: + if node2.nodeName == 'name': + listing.append(node2.firstChild.nodeValue) + self.assertEquals(listing, ['sub.1.0', 'sub.1.1', 'sub.1.2']) + + def test_healthcheck(self): + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + + def start_response(*args): + """ Sends args to outbuf """ + outbuf.writelines(args) + + self.controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/healthcheck', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '200 ') + + def test_through_call(self): + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + def start_response(*args): + outbuf.writelines(args) + self.controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/sda1/p/a', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '404 ') + + def test_through_call_invalid_path(self): + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + def start_response(*args): + outbuf.writelines(args) + self.controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/bob', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '400 ') + + def test_params_utf8(self): + self.controller.PUT(Request.blank('/sda1/p/a', + headers={'X-Timestamp': normalize_timestamp(1)}, + environ={'REQUEST_METHOD': 'PUT'})) + for param in ('delimiter', 'format', 'limit', 'marker', 'prefix'): + req = Request.blank('/sda1/p/a?%s=\xce' % param, + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 400) + req = Request.blank('/sda1/p/a?%s=\xce\xa9' % param, + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assert_(resp.status_int in (204, 412), resp.status_int) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/auth/__init__.py b/test/unit/auth/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py new file mode 100644 index 0000000000..9bb23a925d --- /dev/null +++ b/test/unit/auth/test_server.py @@ -0,0 +1,599 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import unittest +import os +from shutil import rmtree +from StringIO import StringIO +from uuid import uuid4 +from logging import StreamHandler + +from webob import Request + +from swift.auth import server as auth_server +from swift.common.db import DatabaseConnectionError +from swift.common.utils import get_logger + + +class TestException(Exception): + pass + + +def fake_http_connect(*code_iter, **kwargs): + class FakeConn(object): + def __init__(self, status): + self.status = status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = '1234' + def getresponse(self): + if 'slow' in kwargs: + sleep(0.2) + if 'raise_exc' in kwargs: + raise kwargs['raise_exc'] + return self + def getheaders(self): + return {'x-account-bytes-used': '20'} + def read(self, amt=None): + return '' + def getheader(self, name): + return self.getheaders().get(name.lower()) + code_iter = iter(code_iter) + def connect(*args, **ckwargs): + if 'give_content_type' in kwargs: + if len(args) >= 7 and 'content_type' in args[6]: + kwargs['give_content_type'](args[6]['content-type']) + else: + kwargs['give_content_type']('') + return FakeConn(code_iter.next()) + return connect + + +class FakeRing(object): + def get_nodes(self, path): + return 1, [{'ip': '10.0.0.%s' % x, 'port': 1000+x, 'device': 'sda'} + for x in xrange(3)] + + +class TestAuthServer(unittest.TestCase): + + def setUp(self): + self.testdir = os.path.join(os.path.dirname(__file__), + 'auth_server') + rmtree(self.testdir, ignore_errors=1) + os.mkdir(self.testdir) + self.conf = {'swift_dir': self.testdir} + self.controller = auth_server.AuthController(self.conf, FakeRing()) + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def test_get_conn(self): + with self.controller.get_conn() as conn: + pass + exc = False + try: + with self.controller.get_conn() as conn: + raise TestException('test') + except TestException: + exc = True + self.assert_(exc) + # We allow reentrant calls for the auth-server + with self.controller.get_conn() as conn1: + exc = False + try: + with self.controller.get_conn() as conn2: + self.assert_(conn1 is not conn2) + except DatabaseConnectionError: + exc = True + self.assert_(not exc) + self.controller.conn = None + with self.controller.get_conn() as conn: + self.assert_(conn is not None) + + def test_validate_token_non_existant_token(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing',).split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': 'testing'})) + token = res.headers['x-storage-token'] + self.assertEquals(self.controller.validate_token(token + 'bad', + cfaccount), False) + + def test_validate_token_non_existant_cfaccount(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': 'testing'})) + token = res.headers['x-storage-token'] + self.assertEquals(self.controller.validate_token(token, + cfaccount + 'bad'), False) + + def test_validate_token_good(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing',).split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': 'testing'})) + token = res.headers['x-storage-token'] + ttl = self.controller.validate_token(token, cfaccount) + self.assert_(ttl > 0, repr(ttl)) + + def test_validate_token_expired(self): + orig_time = auth_server.time + try: + auth_server.time = lambda: 1 + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account('test', 'tester', + 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': 'testing'})) + token = res.headers['x-storage-token'] + ttl = self.controller.validate_token( + token, cfaccount) + self.assert_(ttl > 0, repr(ttl)) + auth_server.time = lambda: 1 + self.controller.token_life + self.assertEquals(self.controller.validate_token( + token, cfaccount), False) + finally: + auth_server.time = orig_time + + def test_create_account_no_new_account(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + result = self.controller.create_account('', 'tester', 'testing') + self.assertFalse(result) + + def test_create_account_no_new_user(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + result = self.controller.create_account('test', '', 'testing') + self.assertFalse(result) + + def test_create_account_no_new_password(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + result = self.controller.create_account('test', 'tester', '') + self.assertFalse(result) + + def test_create_account_good(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test', 'tester', 'testing') + self.assert_(url) + self.assertEquals('/'.join(url.split('/')[:-1]), + self.controller.default_cluster_url.rstrip('/'), repr(url)) + + def test_recreate_accounts_none(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + rv = self.controller.recreate_accounts() + self.assertEquals(rv.split()[0], '0', repr(rv)) + self.assertEquals(rv.split()[-1], '[]', repr(rv)) + + def test_recreate_accounts_one(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.controller.create_account('test', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201) + rv = self.controller.recreate_accounts() + self.assertEquals(rv.split()[0], '1', repr(rv)) + self.assertEquals(rv.split()[-1], '[]', repr(rv)) + + def test_recreate_accounts_several(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.controller.create_account('test1', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.controller.create_account('test2', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.controller.create_account('test3', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.controller.create_account('test4', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201, + 201, 201, 201, + 201, 201, 201, + 201, 201, 201) + rv = self.controller.recreate_accounts() + self.assertEquals(rv.split()[0], '4', repr(rv)) + self.assertEquals(rv.split()[-1], '[]', repr(rv)) + + def test_recreate_accounts_one_fail(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test', 'tester', 'testing') + cfaccount = url.split('/')[-1] + auth_server.http_connect = fake_http_connect(500, 500, 500) + rv = self.controller.recreate_accounts() + self.assertEquals(rv.split()[0], '1', repr(rv)) + self.assertEquals(rv.split()[-1], '[%s]' % repr(cfaccount), + repr(rv)) + + def test_recreate_accounts_several_fail(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test1', 'tester', 'testing') + cfaccounts = [url.split('/')[-1]] + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test2', 'tester', 'testing') + cfaccounts.append(url.split('/')[-1]) + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test3', 'tester', 'testing') + cfaccounts.append(url.split('/')[-1]) + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test4', 'tester', 'testing') + cfaccounts.append(url.split('/')[-1]) + auth_server.http_connect = fake_http_connect(500, 500, 500, + 500, 500, 500, + 500, 500, 500, + 500, 500, 500) + rv = self.controller.recreate_accounts() + self.assertEquals(rv.split()[0], '4', repr(rv)) + failed = rv.split('[', 1)[-1][:-1].split(', ') + self.assertEquals(failed, [repr(a) for a in cfaccounts]) + + def test_recreate_accounts_several_fail_some(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test1', 'tester', 'testing') + cfaccounts = [url.split('/')[-1]] + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test2', 'tester', 'testing') + cfaccounts.append(url.split('/')[-1]) + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test3', 'tester', 'testing') + cfaccounts.append(url.split('/')[-1]) + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test4', 'tester', 'testing') + cfaccounts.append(url.split('/')[-1]) + auth_server.http_connect = fake_http_connect(500, 500, 500, + 201, 201, 201, + 500, 500, 500, + 201, 201, 201) + rv = self.controller.recreate_accounts() + self.assertEquals(rv.split()[0], '4', repr(rv)) + failed = rv.split('[', 1)[-1][:-1].split(', ') + expected = [] + for i, value in enumerate(cfaccounts): + if not i % 2: + expected.append(repr(value)) + self.assertEquals(failed, expected) + + def test_auth_bad_path(self): + self.assertRaises(ValueError, self.controller.handle_auth, + Request.blank('', environ={'REQUEST_METHOD': 'GET'})) + res = self.controller.handle_auth(Request.blank('/bad', + environ={'REQUEST_METHOD': 'GET'})) + self.assertEquals(res.status_int, 400) + + def test_auth_SOSO_missing_headers(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-Pass': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester'})) + self.assertEquals(res.status_int, 401) + + def test_auth_SOSO_bad_account(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/testbad/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/v1//auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': 'testing'})) + self.assertEquals(res.status_int, 401) + + def test_auth_SOSO_bad_user(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'testerbad', + 'X-Storage-Pass': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': '', + 'X-Storage-Pass': 'testing'})) + self.assertEquals(res.status_int, 401) + + def test_auth_SOSO_bad_password(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': 'testingbad'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': ''})) + self.assertEquals(res.status_int, 401) + + def test_auth_SOSO_good(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'tester', + 'X-Storage-Pass': 'testing'})) + token = res.headers['x-storage-token'] + ttl = self.controller.validate_token(token, cfaccount) + self.assert_(ttl > 0, repr(ttl)) + + def test_auth_SOSO_good_Mosso_headers(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test:tester', + 'X-Auth-Key': 'testing'})) + token = res.headers['x-storage-token'] + ttl = self.controller.validate_token(token, cfaccount) + self.assert_(ttl > 0, repr(ttl)) + + def test_auth_SOSO_bad_Mosso_headers(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing',).split('/')[-1] + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test2:tester', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': ':tester', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/v1/test/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test:', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + + def test_auth_Mosso_missing_headers(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test:tester'})) + self.assertEquals(res.status_int, 401) + + def test_auth_Mosso_bad_header_format(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'badformat', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': '', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + + def test_auth_Mosso_bad_account(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'testbad:tester', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': ':tester', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + + def test_auth_Mosso_bad_user(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test:testerbad', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test:', + 'X-Auth-Key': 'testing'})) + self.assertEquals(res.status_int, 401) + + def test_auth_Mosso_bad_password(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test:tester', + 'X-Auth-Key': 'testingbad'})) + self.assertEquals(res.status_int, 401) + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test:tester', + 'X-Auth-Key': ''})) + self.assertEquals(res.status_int, 401) + + def test_auth_Mosso_good(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Auth-User': 'test:tester', + 'X-Auth-Key': 'testing'})) + token = res.headers['x-storage-token'] + ttl = self.controller.validate_token(token, cfaccount) + self.assert_(ttl > 0, repr(ttl)) + + def test_auth_Mosso_good_SOSO_header_names(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + cfaccount = self.controller.create_account( + 'test', 'tester', 'testing').split('/')[-1] + res = self.controller.handle_auth(Request.blank('/auth', + environ={'REQUEST_METHOD': 'GET'}, + headers={'X-Storage-User': 'test:tester', + 'X-Storage-Pass': 'testing'})) + token = res.headers['x-storage-token'] + ttl = self.controller.validate_token(token, cfaccount) + self.assert_(ttl > 0, repr(ttl)) + + def test_basic_logging(self): + log = StringIO() + log_handler = StreamHandler(log) + logger = get_logger(self.conf, 'auth') + logger.logger.addHandler(log_handler) + try: + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_account('test', 'tester', 'testing') + self.assertEquals(log.getvalue().rsplit(' ', 1)[0], + "auth SUCCESS create_account('test', 'tester', _) = %s" % + repr(url)) + log.truncate(0) + def start_response(*args): + pass + self.controller.handleREST({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/v1/test/auth', + 'QUERY_STRING': 'test=True', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': StringIO(), + 'wsgi.errors': StringIO(), + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False, + 'HTTP_X_FORWARDED_FOR': 'testhost', + 'HTTP_X_STORAGE_USER': 'tester', + 'HTTP_X_STORAGE_PASS': 'testing'}, + start_response) + logsegs = log.getvalue().split(' [', 1) + logsegs[1:] = logsegs[1].split('] ', 1) + logsegs[1] = '[01/Jan/2001:01:02:03 +0000]' + logsegs[2:] = logsegs[2].split(' ') + logsegs[-1] = '0.1234' + self.assertEquals(' '.join(logsegs), 'auth testhost - - ' + '[01/Jan/2001:01:02:03 +0000] "GET /v1/test/auth?test=True ' + 'HTTP/1.0" 204 - "-" "-" - - - - - - - - - "-" "None" "-" ' + '0.1234') + self.controller.log_headers = True + log.truncate(0) + self.controller.handleREST({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/v1/test/auth', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': StringIO(), + 'wsgi.errors': StringIO(), + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False, + 'HTTP_X_STORAGE_USER': 'tester', + 'HTTP_X_STORAGE_PASS': 'testing'}, + start_response) + logsegs = log.getvalue().split(' [', 1) + logsegs[1:] = logsegs[1].split('] ', 1) + logsegs[1] = '[01/Jan/2001:01:02:03 +0000]' + logsegs[2:] = logsegs[2].split(' ') + logsegs[-1] = '0.1234' + self.assertEquals(' '.join(logsegs), 'auth None - - [01/Jan/2001:' + '01:02:03 +0000] "GET /v1/test/auth HTTP/1.0" 204 - "-" "-" - ' + '- - - - - - - - "-" "None" "Content-Length: 0\n' + 'X-Storage-User: tester\nX-Storage-Pass: testing" 0.1234') + finally: + logger.logger.handlers.remove(log_handler) + + def test_unhandled_exceptions(self): + def request_causing_exception(*args, **kwargs): + pass + def start_response(*args): + pass + orig_Request = auth_server.Request + log = StringIO() + log_handler = StreamHandler(log) + logger = get_logger(self.conf, 'auth') + logger.logger.addHandler(log_handler) + try: + auth_server.Request = request_causing_exception + self.controller.handleREST({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/v1/test/auth', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': StringIO(), + 'wsgi.errors': StringIO(), + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False, + 'HTTP_X_STORAGE_USER': 'tester', + 'HTTP_X_STORAGE_PASS': 'testing'}, + start_response) + self.assert_(log.getvalue().startswith( + 'auth ERROR Unhandled exception in ReST request'), + log.getvalue()) + log.truncate(0) + finally: + auth_server.Request = orig_Request + logger.logger.handlers.remove(log_handler) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/__init__.py b/test/unit/common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/common/ring/__init__.py b/test/unit/common/ring/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/common/ring/test_builder.py b/test/unit/common/ring/test_builder.py new file mode 100644 index 0000000000..e4be88b6dc --- /dev/null +++ b/test/unit/common/ring/test_builder.py @@ -0,0 +1,245 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest +from shutil import rmtree + +from swift.common.ring import RingBuilder, RingData +from swift.common import ring + +class TestRingBuilder(unittest.TestCase): + + def setUp(self): + self.testdir = os.path.join(os.path.dirname(__file__), + 'ring_builder') + rmtree(self.testdir, ignore_errors=1) + os.mkdir(self.testdir) + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def test_init(self): + rb = ring.RingBuilder(8, 3, 1) + self.assertEquals(rb.part_power, 8) + self.assertEquals(rb.replicas, 3) + self.assertEquals(rb.min_part_hours, 1) + self.assertEquals(rb.parts, 2**8) + self.assertEquals(rb.devs, []) + self.assertEquals(rb.devs_changed, False) + self.assertEquals(rb.version, 0) + + def test_get_ring(self): + rb = ring.RingBuilder(8, 3, 1) + self.assertRaises(Exception, rb.get_ring) + rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10004, 'device': 'sda1'}) + rb.remove_dev(1) + rb.rebalance() + r = rb.get_ring() + self.assert_(isinstance(r, ring.RingData)) + r2 = rb.get_ring() + self.assert_(r is r2) + rb.rebalance() + r3 = rb.get_ring() + self.assert_(r3 is not r2) + r4 = rb.get_ring() + self.assert_(r3 is r4) + + def test_add_dev(self): + rb = ring.RingBuilder(8, 3, 1) + dev = \ + {'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000} + rb.add_dev(dev) + self.assertRaises(Exception, rb.add_dev, dev) + + def test_set_dev_weight(self): + rb = ring.RingBuilder(8, 3, 1) + rb.add_dev({'id': 0, 'zone': 0, 'weight': 0.5, 'ip': '127.0.0.1', + 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'zone': 0, 'weight': 0.5, 'ip': '127.0.0.1', + 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10003, 'device': 'sda1'}) + rb.rebalance() + r = rb.get_ring() + counts = {} + for part2dev_id in r._replica2part2dev_id: + for dev_id in part2dev_id: + counts[dev_id] = counts.get(dev_id, 0) + 1 + self.assertEquals(counts, {0: 128, 1: 128, 2: 256, 3: 256}) + rb.set_dev_weight(0, 0.75) + rb.set_dev_weight(1, 0.25) + rb.pretend_min_part_hours_passed() + rb.rebalance() + r = rb.get_ring() + counts = {} + for part2dev_id in r._replica2part2dev_id: + for dev_id in part2dev_id: + counts[dev_id] = counts.get(dev_id, 0) + 1 + self.assertEquals(counts, {0: 192, 1: 64, 2: 256, 3: 256}) + + def test_remove_dev(self): + rb = ring.RingBuilder(8, 3, 1) + rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10003, 'device': 'sda1'}) + rb.rebalance() + r = rb.get_ring() + counts = {} + for part2dev_id in r._replica2part2dev_id: + for dev_id in part2dev_id: + counts[dev_id] = counts.get(dev_id, 0) + 1 + self.assertEquals(counts, {0: 192, 1: 192, 2: 192, 3: 192}) + rb.remove_dev(1) + rb.pretend_min_part_hours_passed() + rb.rebalance() + r = rb.get_ring() + counts = {} + for part2dev_id in r._replica2part2dev_id: + for dev_id in part2dev_id: + counts[dev_id] = counts.get(dev_id, 0) + 1 + self.assertEquals(counts, {0: 256, 2: 256, 3: 256}) + + def test_rerebalance(self): + rb = ring.RingBuilder(8, 3, 1) + rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10002, 'device': 'sda1'}) + rb.rebalance() + r = rb.get_ring() + counts = {} + for part2dev_id in r._replica2part2dev_id: + for dev_id in part2dev_id: + counts[dev_id] = counts.get(dev_id, 0) + 1 + self.assertEquals(counts, {0: 256, 1: 256, 2: 256}) + rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10003, 'device': 'sda1'}) + rb.pretend_min_part_hours_passed() + rb.rebalance() + r = rb.get_ring() + counts = {} + for part2dev_id in r._replica2part2dev_id: + for dev_id in part2dev_id: + counts[dev_id] = counts.get(dev_id, 0) + 1 + self.assertEquals(counts, {0: 192, 1: 192, 2: 192, 3: 192}) + rb.set_dev_weight(3, 100) + rb.rebalance() + r = rb.get_ring() + counts = {} + for part2dev_id in r._replica2part2dev_id: + for dev_id in part2dev_id: + counts[dev_id] = counts.get(dev_id, 0) + 1 + self.assertEquals(counts[3], 256) + + def test_validate(self): + rb = ring.RingBuilder(8, 3, 1) + rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', + 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.1', + 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.1', + 'port': 10003, 'device': 'sda1'}) + rb.rebalance() + r = rb.get_ring() + counts = {} + for part2dev_id in r._replica2part2dev_id: + for dev_id in part2dev_id: + counts[dev_id] = counts.get(dev_id, 0) + 1 + self.assertEquals(counts, {0: 128, 1: 128, 2: 256, 3: 256}) + + dev_usage, worst = rb.validate() + self.assert_(dev_usage is None) + self.assert_(worst is None) + + dev_usage, worst = rb.validate(stats=True) + self.assertEquals(list(dev_usage), [128, 128, 256, 256]) + self.assertEquals(int(worst), 0) + + rb.set_dev_weight(2, 0) + rb.rebalance() + self.assertEquals(rb.validate(stats=True)[1], 999.99) + + # Test not all partitions doubly accounted for + rb.devs[1]['parts'] -= 1 + self.assertRaises(Exception, rb.validate) + rb.devs[1]['parts'] += 1 + + # Test duplicate device for partition + orig_dev_id = rb._replica2part2dev[0][0] + rb._replica2part2dev[0][0] = rb._replica2part2dev[1][0] + self.assertRaises(Exception, rb.validate) + rb._replica2part2dev[0][0] = orig_dev_id + + # Test duplicate zone for partition + rb.add_dev({'id': 5, 'zone': 0, 'weight': 2, 'ip': '127.0.0.1', + 'port': 10005, 'device': 'sda1'}) + rb.pretend_min_part_hours_passed() + rb.rebalance() + rb.validate() + orig_replica = orig_partition = orig_device = None + for part2dev in rb._replica2part2dev: + for p in xrange(2**8): + if part2dev[p] == 5: + for r in xrange(len(rb._replica2part2dev)): + if rb._replica2part2dev[r][p] != 5: + orig_replica = r + orig_partition = p + orig_device = rb._replica2part2dev[r][p] + rb._replica2part2dev[r][p] = 0 + break + if orig_replica is not None: + break + if orig_replica is not None: + break + self.assertRaises(Exception, rb.validate) + rb._replica2part2dev[orig_replica][orig_partition] = orig_device + + # Tests that validate can handle 'holes' in .devs + rb.remove_dev(2) + rb.pretend_min_part_hours_passed() + rb.rebalance() + rb.validate(stats=True) + + # Validate that zero weight devices with no partitions don't count on + # the 'worst' value. + self.assertNotEquals(rb.validate(stats=True)[1], 999.99) + rb.add_dev({'id': 4, 'zone': 0, 'weight': 0, 'ip': '127.0.0.1', + 'port': 10004, 'device': 'sda1'}) + rb.pretend_min_part_hours_passed() + rb.rebalance() + self.assertNotEquals(rb.validate(stats=True)[1], 999.99) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py new file mode 100644 index 0000000000..5c668c8c57 --- /dev/null +++ b/test/unit/common/ring/test_ring.py @@ -0,0 +1,204 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cPickle as pickle +import os +import unittest +from gzip import GzipFile +from shutil import rmtree +from time import sleep, time + +from swift.common import ring, utils + + +class TestRingData(unittest.TestCase): + + def test_attrs(self): + r2p2d = [[0, 1, 0, 1], [0, 1, 0, 1]] + d = [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}] + s = 30 + rd = ring.RingData(r2p2d, d, s) + self.assertEquals(rd._replica2part2dev_id, r2p2d) + self.assertEquals(rd.devs, d) + self.assertEquals(rd._part_shift, s) + + def test_pickleable(self): + rd = ring.RingData([[0, 1, 0, 1], [0, 1, 0, 1]], + [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}], 30) + for p in xrange(pickle.HIGHEST_PROTOCOL): + pickle.loads(pickle.dumps(rd, protocol=p)) + + +class TestRing(unittest.TestCase): + + def setUp(self): + utils.HASH_PATH_SUFFIX = 'endcap' + self.testdir = os.path.join(os.path.dirname(__file__), 'ring') + rmtree(self.testdir, ignore_errors=1) + os.mkdir(self.testdir) + self.testgz = os.path.join(self.testdir, 'ring.gz') + self.intended_replica2part2dev_id = [[0, 2, 0, 2], [2, 0, 2, 0]] + self.intended_devs = [{'id': 0, 'zone': 0}, None, {'id': 2, 'zone': 2}] + self.intended_part_shift = 30 + self.intended_reload_time = 15 + pickle.dump(ring.RingData(self.intended_replica2part2dev_id, + self.intended_devs, self.intended_part_shift), + GzipFile(self.testgz, 'wb')) + self.ring = \ + ring.Ring(self.testgz, reload_time=self.intended_reload_time) + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def test_creation(self): + self.assertEquals(self.ring._replica2part2dev_id, + self.intended_replica2part2dev_id) + self.assertEquals(self.ring._part_shift, self.intended_part_shift) + self.assertEquals(self.ring.devs, self.intended_devs) + self.assertEquals(self.ring.reload_time, self.intended_reload_time) + self.assertEquals(self.ring.pickle_gz_path, self.testgz) + + def test_has_changed(self): + self.assertEquals(self.ring.has_changed(), False) + os.utime(self.testgz, (time()+60, time()+60)) + self.assertEquals(self.ring.has_changed(), True) + + def test_reload(self): + os.utime(self.testgz, (time() - 300, time() - 300)) + self.ring = ring.Ring(self.testgz, reload_time=0.001) + orig_mtime = self.ring._mtime + self.assertEquals(len(self.ring.devs), 3) + self.intended_devs.append({'id': 3, 'zone': 3}) + pickle.dump(ring.RingData(self.intended_replica2part2dev_id, + self.intended_devs, self.intended_part_shift), + GzipFile(self.testgz, 'wb')) + sleep(0.1) + self.ring.get_nodes('a') + self.assertEquals(len(self.ring.devs), 4) + self.assertNotEquals(self.ring._mtime, orig_mtime) + + os.utime(self.testgz, (time() - 300, time() - 300)) + self.ring = ring.Ring(self.testgz, reload_time=0.001) + orig_mtime = self.ring._mtime + self.assertEquals(len(self.ring.devs), 4) + self.intended_devs.append({'id': 4, 'zone': 4}) + pickle.dump(ring.RingData(self.intended_replica2part2dev_id, + self.intended_devs, self.intended_part_shift), + GzipFile(self.testgz, 'wb')) + sleep(0.1) + self.ring.get_part_nodes(0) + self.assertEquals(len(self.ring.devs), 5) + self.assertNotEquals(self.ring._mtime, orig_mtime) + + os.utime(self.testgz, (time() - 300, time() - 300)) + self.ring = \ + ring.Ring(self.testgz, reload_time=0.001) + orig_mtime = self.ring._mtime + part, nodes = self.ring.get_nodes('a') + self.assertEquals(len(self.ring.devs), 5) + self.intended_devs.append({'id': 5, 'zone': 5}) + pickle.dump(ring.RingData(self.intended_replica2part2dev_id, + self.intended_devs, self.intended_part_shift), + GzipFile(self.testgz, 'wb')) + sleep(0.1) + self.ring.get_more_nodes(part).next() + self.assertEquals(len(self.ring.devs), 6) + self.assertNotEquals(self.ring._mtime, orig_mtime) + + def test_get_part_nodes(self): + part, nodes = self.ring.get_nodes('a') + self.assertEquals(nodes, self.ring.get_part_nodes(part)) + + def test_get_nodes(self): + # Yes, these tests are deliberately very fragile. We want to make sure + # that if someones changes the results the ring produces, they know it. + self.assertRaises(TypeError, self.ring.get_nodes) + part, nodes = self.ring.get_nodes('a') + self.assertEquals(part, 0) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + part, nodes = self.ring.get_nodes('a1') + self.assertEquals(part, 0) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + part, nodes = self.ring.get_nodes('a4') + self.assertEquals(part, 1) + self.assertEquals(nodes, [{'id': 2, 'zone': 2}, {'id': 0, 'zone': 0}]) + part, nodes = self.ring.get_nodes('aa') + self.assertEquals(part, 1) + self.assertEquals(nodes, [{'id': 2, 'zone': 2}, {'id': 0, 'zone': 0}]) + + part, nodes = self.ring.get_nodes('a', 'c1') + self.assertEquals(part, 0) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + part, nodes = self.ring.get_nodes('a', 'c0') + self.assertEquals(part, 3) + self.assertEquals(nodes, [{'id': 2, 'zone': 2}, {'id': 0, 'zone': 0}]) + part, nodes = self.ring.get_nodes('a', 'c3') + self.assertEquals(part, 2) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + part, nodes = self.ring.get_nodes('a', 'c2') + self.assertEquals(part, 2) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + + part, nodes = self.ring.get_nodes('a', 'c', 'o1') + self.assertEquals(part, 1) + self.assertEquals(nodes, [{'id': 2, 'zone': 2}, {'id': 0, 'zone': 0}]) + part, nodes = self.ring.get_nodes('a', 'c', 'o5') + self.assertEquals(part, 0) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + part, nodes = self.ring.get_nodes('a', 'c', 'o0') + self.assertEquals(part, 0) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + part, nodes = self.ring.get_nodes('a', 'c', 'o2') + self.assertEquals(part, 2) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + + def test_get_more_nodes(self): + # Yes, these tests are deliberately very fragile. We want to make sure + # that if someone changes the results the ring produces, they know it. + part, nodes = self.ring.get_nodes('a', 'c', 'o2') + self.assertEquals(part, 2) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + nodes = list(self.ring.get_more_nodes(part)) + self.assertEquals(nodes, []) + + self.ring.devs.append({'id': 3, 'zone': 0}) + self.ring.zone2devs[0].append(self.ring.devs[3]) + part, nodes = self.ring.get_nodes('a', 'c', 'o2') + self.assertEquals(part, 2) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + nodes = list(self.ring.get_more_nodes(part)) + self.assertEquals(nodes, []) + + self.ring.zone2devs[self.ring.devs[3]['zone']].remove(self.ring.devs[3]) + self.ring.devs[3]['zone'] = 3 + self.ring.zone2devs[3] = [self.ring.devs[3]] + part, nodes = self.ring.get_nodes('a', 'c', 'o2') + self.assertEquals(part, 2) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + nodes = list(self.ring.get_more_nodes(part)) + self.assertEquals(nodes, [{'id': 3, 'zone': 3}]) + + self.ring.devs.append(None) + self.ring.devs.append({'id': 5, 'zone': 5}) + self.ring.zone2devs[5] = [self.ring.devs[5]] + part, nodes = self.ring.get_nodes('a', 'c', 'o2') + self.assertEquals(part, 2) + self.assertEquals(nodes, [{'id': 0, 'zone': 0}, {'id': 2, 'zone': 2}]) + nodes = list(self.ring.get_more_nodes(part)) + self.assertEquals(nodes, [{'id': 3, 'zone': 3}, {'id': 5, 'zone': 5}]) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_auth.py b/test/unit/common/test_auth.py new file mode 100644 index 0000000000..cf35827522 --- /dev/null +++ b/test/unit/common/test_auth.py @@ -0,0 +1,177 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import logging +import os +import sys +import unittest +from contextlib import contextmanager + +import eventlet +from webob import Request + +from swift.common import auth + +# mocks +logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) + + +class FakeMemcache(object): + def __init__(self): + self.store = {} + + def get(self, key): + return self.store.get(key) + + def set(self, key, value, timeout=0): + self.store[key] = value + return True + + def incr(self, key, timeout=0): + self.store[key] = self.store.setdefault(key, 0) + 1 + return self.store[key] + + @contextmanager + def soft_lock(self, key, timeout=0, retries=5): + yield True + + def delete(self, key): + try: + del self.store[key] + except: + pass + return True + + +def mock_http_connect(response, headers=None, with_exc=False): + class FakeConn(object): + def __init__(self, status, headers, with_exc): + self.status = status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = '1234' + self.with_exc = with_exc + self.headers = headers + if self.headers is None: + self.headers = {} + def getresponse(self): + if self.with_exc: + raise Exception('test') + return self + def getheader(self, header): + return self.headers[header] + def read(self, amt=None): + return '' + def close(self): + return + return lambda *args, **kwargs: FakeConn(response, headers, with_exc) + + +class Logger(object): + def __init__(self): + self.error_value = None + self.exception_value = None + def error(self, msg, *args, **kwargs): + self.error_value = (msg, args, kwargs) + def exception(self, msg, *args, **kwargs): + _, exc, _ = sys.exc_info() + self.exception_value = (msg, + '%s %s' % (exc.__class__.__name__, str(exc)), args, kwargs) +# tests + +class FakeApp(object): + def __call__(self, env, start_response): + return "OK" + +def start_response(*args): + pass + +class TestAuth(unittest.TestCase): + + def setUp(self): + self.test_auth = auth.DevAuthMiddleware( + FakeApp(), {}, FakeMemcache(), Logger()) + + def test_auth_fail(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(404) + self.assertFalse(self.test_auth.auth('a','t')) + finally: + auth.http_connect = old_http_connect + + def test_auth_success(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) + self.assertTrue(self.test_auth.auth('a','t')) + finally: + auth.http_connect = old_http_connect + + def test_auth_memcache(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) + self.assertTrue(self.test_auth.auth('a','t')) + auth.http_connect = mock_http_connect(404) + # Should still be in memcache + self.assertTrue(self.test_auth.auth('a','t')) + finally: + auth.http_connect = old_http_connect + + def test_middleware_success(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) + req = Request.blank('/v/a/c/o', headers={'x-auth-token':'t'}) + resp = self.test_auth(req.environ, start_response) + self.assertEquals(resp, 'OK') + finally: + auth.http_connect = old_http_connect + + def test_middleware_no_header(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) + req = Request.blank('/v/a/c/o') + resp = self.test_auth(req.environ, start_response) + self.assertEquals(resp, ['Missing Auth Token']) + finally: + auth.http_connect = old_http_connect + + def test_middleware_storage_token(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) + req = Request.blank('/v/a/c/o', headers={'x-storage-token':'t'}) + resp = self.test_auth(req.environ, start_response) + self.assertEquals(resp, 'OK') + finally: + auth.http_connect = old_http_connect + + def test_middleware_only_version(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) + req = Request.blank('/v', headers={'x-auth-token':'t'}) + resp = self.test_auth(req.environ, start_response) + self.assertEquals(resp, ['Bad URL']) + finally: + auth.http_connect = old_http_connect + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_bufferedhttp.py b/test/unit/common/test_bufferedhttp.py new file mode 100644 index 0000000000..d453442f04 --- /dev/null +++ b/test/unit/common/test_bufferedhttp.py @@ -0,0 +1,73 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from eventlet import spawn, TimeoutError, listen +from eventlet.timeout import Timeout + +from swift.common import bufferedhttp + + +class TestBufferedHTTP(unittest.TestCase): + + def test_http_connect(self): + bindsock = listen(('127.0.0.1', 0)) + def accept(expected_par): + try: + with Timeout(3): + sock, addr = bindsock.accept() + fp = sock.makefile() + fp.write('HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\n' + 'RESPONSE') + fp.flush() + self.assertEquals(fp.readline(), + 'PUT /dev/%s/path/..%%25/?omg&no=%%7f HTTP/1.1\r\n' % + expected_par) + headers = {} + line = fp.readline() + while line and line != '\r\n': + headers[line.split(':')[0].lower()] = \ + line.split(':')[1].strip() + line = fp.readline() + self.assertEquals(headers['content-length'], '7') + self.assertEquals(headers['x-header'], 'value') + self.assertEquals(fp.readline(), 'REQUEST\r\n') + except BaseException, err: + return err + return None + for par in ('par', 1357): + event = spawn(accept, par) + try: + with Timeout(3): + conn = bufferedhttp.http_connect('127.0.0.1', + bindsock.getsockname()[1], 'dev', par, 'PUT', + '/path/..%/', {'content-length': 7, 'x-header': + 'value'}, query_string='omg&no=%7f') + conn.send('REQUEST\r\n') + resp = conn.getresponse() + body = resp.read() + conn.close() + self.assertEquals(resp.status, 200) + self.assertEquals(resp.reason, 'OK') + self.assertEquals(body, 'RESPONSE') + finally: + err = event.wait() + if err: + raise Exception(err) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_client.py b/test/unit/common/test_client.py new file mode 100644 index 0000000000..24c9ca1982 --- /dev/null +++ b/test/unit/common/test_client.py @@ -0,0 +1,28 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Tests + +import unittest +from swift.common import client + +class TestAuditor(unittest.TestCase): + + def test_placeholder(self): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py new file mode 100644 index 0000000000..da509bcd08 --- /dev/null +++ b/test/unit/common/test_constraints.py @@ -0,0 +1,142 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from webob import Request +from webob.exc import HTTPBadRequest, HTTPLengthRequired, \ + HTTPRequestEntityTooLarge + +from swift.common import constraints + + +class TestConstraints(unittest.TestCase): + + def test_check_metadata_empty(self): + headers = {} + self.assertEquals(constraints.check_metadata(Request.blank('/', + headers=headers)), None) + + def test_check_metadata_good(self): + headers = {'X-Object-Meta-Name': 'Value'} + self.assertEquals(constraints.check_metadata(Request.blank('/', + headers=headers)), None) + + def test_check_metadata_empty_name(self): + headers = {'X-Object-Meta-': 'Value'} + self.assert_(constraints.check_metadata(Request.blank('/', + headers=headers)), HTTPBadRequest) + + def test_check_metadata_name_length(self): + name = 'a' * constraints.MAX_META_NAME_LENGTH + headers = {'X-Object-Meta-%s' % name: 'v'} + self.assertEquals(constraints.check_metadata(Request.blank('/', + headers=headers)), None) + name = 'a' * (constraints.MAX_META_NAME_LENGTH + 1) + headers = {'X-Object-Meta-%s' % name: 'v'} + self.assert_(isinstance(constraints.check_metadata(Request.blank('/', + headers=headers)), HTTPBadRequest)) + + def test_check_metadata_value_length(self): + value = 'a' * constraints.MAX_META_VALUE_LENGTH + headers = {'X-Object-Meta-Name': value} + self.assertEquals(constraints.check_metadata(Request.blank('/', + headers=headers)), None) + value = 'a' * (constraints.MAX_META_VALUE_LENGTH + 1) + headers = {'X-Object-Meta-Name': value} + self.assert_(isinstance(constraints.check_metadata(Request.blank('/', + headers=headers)), HTTPBadRequest)) + + def test_check_metadata_count(self): + headers = {} + for x in xrange(constraints.MAX_META_COUNT): + headers['X-Object-Meta-%d' % x] = 'v' + self.assertEquals(constraints.check_metadata(Request.blank('/', + headers=headers)), None) + headers['X-Object-Meta-Too-Many'] = 'v' + self.assert_(isinstance(constraints.check_metadata(Request.blank('/', + headers=headers)), HTTPBadRequest)) + + def test_check_metadata_size(self): + headers = {} + size = 0 + chunk = constraints.MAX_META_NAME_LENGTH + \ + constraints.MAX_META_VALUE_LENGTH + x = 0 + while size + chunk < constraints.MAX_META_OVERALL_SIZE: + headers['X-Object-Meta-%04d%s' % + (x, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \ + 'v' * constraints.MAX_META_VALUE_LENGTH + size += chunk + x += 1 + self.assertEquals(constraints.check_metadata(Request.blank('/', + headers=headers)), None) + headers['X-Object-Meta-9999%s' % + ('a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \ + 'v' * constraints.MAX_META_VALUE_LENGTH + self.assert_(isinstance(constraints.check_metadata(Request.blank('/', + headers=headers)), HTTPBadRequest)) + + def test_check_object_creation_content_length(self): + headers = {'Content-Length': str(constraints.MAX_FILE_SIZE), + 'Content-Type': 'text/plain'} + self.assertEquals(constraints.check_object_creation(Request.blank('/', + headers=headers), 'object_name'), None) + headers = {'Content-Length': str(constraints.MAX_FILE_SIZE + 1), + 'Content-Type': 'text/plain'} + self.assert_(isinstance(constraints.check_object_creation( + Request.blank('/', headers=headers), 'object_name'), + HTTPRequestEntityTooLarge)) + headers = {'Transfer-Encoding': 'chunked', + 'Content-Type': 'text/plain'} + self.assertEquals(constraints.check_object_creation(Request.blank('/', + headers=headers), 'object_name'), None) + headers = {'Content-Type': 'text/plain'} + self.assert_(isinstance(constraints.check_object_creation( + Request.blank('/', headers=headers), 'object_name'), + HTTPLengthRequired)) + + def test_check_object_creation_name_length(self): + headers = {'Transfer-Encoding': 'chunked', + 'Content-Type': 'text/plain'} + name = 'o' * constraints.MAX_OBJECT_NAME_LENGTH + self.assertEquals(constraints.check_object_creation(Request.blank('/', + headers=headers), name), None) + name = 'o' * (constraints.MAX_OBJECT_NAME_LENGTH + 1) + self.assert_(isinstance(constraints.check_object_creation( + Request.blank('/', headers=headers), name), + HTTPBadRequest)) + + def test_check_object_creation_content_type(self): + headers = {'Transfer-Encoding': 'chunked', + 'Content-Type': 'text/plain'} + self.assertEquals(constraints.check_object_creation(Request.blank('/', + headers=headers), 'object_name'), None) + headers = {'Transfer-Encoding': 'chunked'} + self.assert_(isinstance(constraints.check_object_creation( + Request.blank('/', headers=headers), 'object_name'), + HTTPBadRequest)) + + def test_check_object_creation_bad_content_type(self): + headers = {'Transfer-Encoding': 'chunked', + 'Content-Type': '\xff\xff'} + resp = constraints.check_object_creation( + Request.blank('/', headers=headers), 'object_name') + self.assert_(isinstance(resp, HTTPBadRequest)) + self.assert_('Content-Type' in resp.body) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py new file mode 100644 index 0000000000..1979681e27 --- /dev/null +++ b/test/unit/common/test_db.py @@ -0,0 +1,1579 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for swift.common.db """ + +from __future__ import with_statement +import hashlib +import os +import unittest +from shutil import rmtree +from StringIO import StringIO +from time import sleep, time +from uuid import uuid4 + +import simplejson +import sqlite3 + +from swift.common.db import AccountBroker, chexor, ContainerBroker, \ + DatabaseBroker, DatabaseConnectionError, dict_factory, get_db_connection +from swift.common.utils import normalize_timestamp +from swift.common.exceptions import LockTimeout + + +class TestDatabaseConnectionError(unittest.TestCase): + + def test_str(self): + err = \ + DatabaseConnectionError(':memory:', 'No valid database connection') + self.assert_(':memory:' in str(err)) + self.assert_('No valid database connection' in str(err)) + err = DatabaseConnectionError(':memory:', + 'No valid database connection', timeout=1357) + self.assert_(':memory:' in str(err)) + self.assert_('No valid database connection' in str(err)) + self.assert_('1357' in str(err)) + + +class TestDictFactory(unittest.TestCase): + + def test_normal_case(self): + conn = sqlite3.connect(':memory:') + conn.execute('CREATE TABLE test (one TEXT, two INTEGER)') + conn.execute('INSERT INTO test (one, two) VALUES ("abc", 123)') + conn.execute('INSERT INTO test (one, two) VALUES ("def", 456)') + conn.commit() + curs = conn.execute('SELECT one, two FROM test') + self.assertEquals(dict_factory(curs, curs.next()), + {'one': 'abc', 'two': 123}) + self.assertEquals(dict_factory(curs, curs.next()), + {'one': 'def', 'two': 456}) + + +class TestChexor(unittest.TestCase): + + def test_normal_case(self): + self.assertEquals(chexor('d41d8cd98f00b204e9800998ecf8427e', + 'new name', normalize_timestamp(1)), + '4f2ea31ac14d4273fe32ba08062b21de') + + def test_invalid_old_hash(self): + self.assertRaises(TypeError, chexor, 'oldhash', 'name', + normalize_timestamp(1)) + + def test_no_name(self): + self.assertRaises(Exception, chexor, + 'd41d8cd98f00b204e9800998ecf8427e', None, normalize_timestamp(1)) + + +class TestGetDBConnection(unittest.TestCase): + + def test_normal_case(self): + conn = get_db_connection(':memory:') + self.assert_(hasattr(conn, 'execute')) + + def test_invalid_path(self): + self.assertRaises(DatabaseConnectionError, get_db_connection, + 'invalid database path / name') + + +class TestDatabaseBroker(unittest.TestCase): + + def setUp(self): + self.testdir = os.path.join(os.path.dirname(__file__), 'db') + rmtree(self.testdir, ignore_errors=1) + os.mkdir(self.testdir) + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def test_memory_db_init(self): + broker = DatabaseBroker(':memory:') + self.assertEqual(broker.db_file, ':memory:') + self.assertRaises(AttributeError, broker.initialize, + normalize_timestamp('0')) + + def test_disk_db_init(self): + db_file = os.path.join(self.testdir, '1.db') + broker = DatabaseBroker(db_file) + self.assertEqual(broker.db_file, db_file) + self.assert_(broker.conn is None) + + def test_initialize(self): + self.assertRaises(AttributeError, + DatabaseBroker(':memory:').initialize, + normalize_timestamp('1')) + stub_dict = {} + def stub(*args, **kwargs): + for key in stub_dict.keys(): + del stub_dict[key] + stub_dict['args'] = args + for key, value in kwargs.items(): + stub_dict[key] = value + broker = DatabaseBroker(':memory:') + broker._initialize = stub + broker.initialize(normalize_timestamp('1')) + self.assert_(hasattr(stub_dict['args'][0], 'execute')) + self.assertEquals(stub_dict['args'][1], '0000000001.00000') + with broker.get() as conn: + conn.execute('SELECT * FROM outgoing_sync') + conn.execute('SELECT * FROM incoming_sync') + broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) + broker._initialize = stub + broker.initialize(normalize_timestamp('1')) + self.assert_(hasattr(stub_dict['args'][0], 'execute')) + self.assertEquals(stub_dict['args'][1], '0000000001.00000') + with broker.get() as conn: + conn.execute('SELECT * FROM outgoing_sync') + conn.execute('SELECT * FROM incoming_sync') + + def test_delete_db(self): + stub_called = [False] + def stub(*args, **kwargs): + stub_called[0] = True + broker = DatabaseBroker(':memory:') + broker._initialize = stub + broker.initialize(normalize_timestamp('1')) + self.assert_(broker.conn is not None) + broker._delete_db = stub + stub_called[0] = False + broker.delete_db('2') + self.assert_(stub_called[0]) + broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) + broker._initialize = stub + broker.initialize(normalize_timestamp('1')) + broker._delete_db = stub + stub_called[0] = False + broker.delete_db('2') + self.assert_(stub_called[0]) + + def test_get(self): + broker = DatabaseBroker(':memory:') + got_exc = False + try: + with broker.get() as conn: + conn.execute('SELECT 1') + except: + got_exc = True + broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) + got_exc = False + try: + with broker.get() as conn: + conn.execute('SELECT 1') + except: + got_exc = True + self.assert_(got_exc) + def stub(*args, **kwargs): + pass + broker._initialize = stub + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + conn.execute('CREATE TABLE test (one TEXT)') + try: + with broker.get() as conn: + conn.execute('INSERT INTO test (one) VALUES ("1")') + raise Exception('test') + conn.commit() + except: + pass + broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) + with broker.get() as conn: + self.assertEquals( + [r[0] for r in conn.execute('SELECT * FROM test')], []) + with broker.get() as conn: + conn.execute('INSERT INTO test (one) VALUES ("1")') + conn.commit() + broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) + with broker.get() as conn: + self.assertEquals( + [r[0] for r in conn.execute('SELECT * FROM test')], ['1']) + + def test_lock(self): + broker = DatabaseBroker(os.path.join(self.testdir, '1.db'), timeout=.1) + got_exc = False + try: + with broker.lock(): + pass + except Exception: + got_exc = True + self.assert_(got_exc) + def stub(*args, **kwargs): + pass + broker._initialize = stub + broker.initialize(normalize_timestamp('1')) + with broker.lock(): + pass + with broker.lock(): + pass + broker2 = DatabaseBroker(os.path.join(self.testdir, '1.db'), timeout=.1) + broker2._initialize = stub + with broker.lock(): + got_exc = False + try: + with broker2.lock(): + pass + except LockTimeout: + got_exc = True + self.assert_(got_exc) + try: + with broker.lock(): + raise Exception('test') + except: + pass + with broker.lock(): + pass + + def test_newid(self): + broker = DatabaseBroker(':memory:') + broker.db_type = 'test' + broker.db_contains_type = 'test' + uuid1 = str(uuid4()) + def _initialize(conn, timestamp): + conn.execute('CREATE TABLE test (one TEXT)') + conn.execute('CREATE TABLE test_stat (id TEXT)') + conn.execute('INSERT INTO test_stat (id) VALUES (?)', (uuid1,)) + conn.commit() + broker._initialize = _initialize + broker.initialize(normalize_timestamp('1')) + uuid2 = str(uuid4()) + broker.newid(uuid2) + with broker.get() as conn: + uuids = [r[0] for r in conn.execute('SELECT * FROM test_stat')] + self.assertEquals(len(uuids), 1) + self.assertNotEquals(uuids[0], uuid1) + uuid1 = uuids[0] + points = [(r[0], r[1]) for r in conn.execute('SELECT sync_point, ' + 'remote_id FROM incoming_sync WHERE remote_id = ?', (uuid2,))] + self.assertEquals(len(points), 1) + self.assertEquals(points[0][0], -1) + self.assertEquals(points[0][1], uuid2) + conn.execute('INSERT INTO test (one) VALUES ("1")') + conn.commit() + uuid3 = str(uuid4()) + broker.newid(uuid3) + with broker.get() as conn: + uuids = [r[0] for r in conn.execute('SELECT * FROM test_stat')] + self.assertEquals(len(uuids), 1) + self.assertNotEquals(uuids[0], uuid1) + uuid1 = uuids[0] + points = [(r[0], r[1]) for r in conn.execute('SELECT sync_point, ' + 'remote_id FROM incoming_sync WHERE remote_id = ?', (uuid3,))] + self.assertEquals(len(points), 1) + self.assertEquals(points[0][1], uuid3) + broker.newid(uuid2) + with broker.get() as conn: + uuids = [r[0] for r in conn.execute('SELECT * FROM test_stat')] + self.assertEquals(len(uuids), 1) + self.assertNotEquals(uuids[0], uuid1) + points = [(r[0], r[1]) for r in conn.execute('SELECT sync_point, ' + 'remote_id FROM incoming_sync WHERE remote_id = ?', (uuid2,))] + self.assertEquals(len(points), 1) + self.assertEquals(points[0][1], uuid2) + + def test_get_items_since(self): + broker = DatabaseBroker(':memory:') + broker.db_type = 'test' + broker.db_contains_type = 'test' + def _initialize(conn, timestamp): + conn.execute('CREATE TABLE test (one TEXT)') + conn.execute('INSERT INTO test (one) VALUES ("1")') + conn.execute('INSERT INTO test (one) VALUES ("2")') + conn.execute('INSERT INTO test (one) VALUES ("3")') + conn.commit() + broker._initialize = _initialize + broker.initialize(normalize_timestamp('1')) + self.assertEquals(broker.get_items_since(-1, 10), + [{'one': '1'}, {'one': '2'}, {'one': '3'}]) + self.assertEquals(broker.get_items_since(-1, 2), + [{'one': '1'}, {'one': '2'}]) + self.assertEquals(broker.get_items_since(1, 2), + [{'one': '2'}, {'one': '3'}]) + self.assertEquals(broker.get_items_since(3, 2), []) + self.assertEquals(broker.get_items_since(999, 2), []) + + def test_get_sync(self): + broker = DatabaseBroker(':memory:') + broker.db_type = 'test' + broker.db_contains_type = 'test' + uuid1 = str(uuid4()) + def _initialize(conn, timestamp): + conn.execute('CREATE TABLE test (one TEXT)') + conn.execute('CREATE TABLE test_stat (id TEXT)') + conn.execute('INSERT INTO test_stat (id) VALUES (?)', (uuid1,)) + conn.execute('INSERT INTO test (one) VALUES ("1")') + conn.commit() + pass + broker._initialize = _initialize + broker.initialize(normalize_timestamp('1')) + uuid2 = str(uuid4()) + self.assertEquals(broker.get_sync(uuid2), -1) + broker.newid(uuid2) + self.assertEquals(broker.get_sync(uuid2), 1) + uuid3 = str(uuid4()) + self.assertEquals(broker.get_sync(uuid3), -1) + with broker.get() as conn: + conn.execute('INSERT INTO test (one) VALUES ("2")') + conn.commit() + broker.newid(uuid3) + self.assertEquals(broker.get_sync(uuid2), 1) + self.assertEquals(broker.get_sync(uuid3), 2) + self.assertEquals(broker.get_sync(uuid2, incoming=False), -1) + self.assertEquals(broker.get_sync(uuid3, incoming=False), -1) + broker.merge_syncs([{'sync_point': 1, 'remote_id': uuid2}], + incoming=False) + self.assertEquals(broker.get_sync(uuid2), 1) + self.assertEquals(broker.get_sync(uuid3), 2) + self.assertEquals(broker.get_sync(uuid2, incoming=False), 1) + self.assertEquals(broker.get_sync(uuid3, incoming=False), -1) + broker.merge_syncs([{'sync_point': 2, 'remote_id': uuid3}], + incoming=False) + self.assertEquals(broker.get_sync(uuid2, incoming=False), 1) + self.assertEquals(broker.get_sync(uuid3, incoming=False), 2) + + def test_merge_syncs(self): + broker = DatabaseBroker(':memory:') + def stub(*args, **kwargs): + pass + broker._initialize = stub + broker.initialize(normalize_timestamp('1')) + uuid2 = str(uuid4()) + broker.merge_syncs([{'sync_point': 1, 'remote_id': uuid2}]) + self.assertEquals(broker.get_sync(uuid2), 1) + uuid3 = str(uuid4()) + broker.merge_syncs([{'sync_point': 2, 'remote_id': uuid3}]) + self.assertEquals(broker.get_sync(uuid2), 1) + self.assertEquals(broker.get_sync(uuid3), 2) + self.assertEquals(broker.get_sync(uuid2, incoming=False), -1) + self.assertEquals(broker.get_sync(uuid3, incoming=False), -1) + broker.merge_syncs([{'sync_point': 3, 'remote_id': uuid2}, + {'sync_point': 4, 'remote_id': uuid3}], + incoming=False) + self.assertEquals(broker.get_sync(uuid2, incoming=False), 3) + self.assertEquals(broker.get_sync(uuid3, incoming=False), 4) + self.assertEquals(broker.get_sync(uuid2), 1) + self.assertEquals(broker.get_sync(uuid3), 2) + broker.merge_syncs([{'sync_point': 5, 'remote_id': uuid2}]) + self.assertEquals(broker.get_sync(uuid2), 5) + + +class TestContainerBroker(unittest.TestCase): + """ Tests for swift.common.db.ContainerBroker """ + + def test_creation(self): + """ Test swift.common.db.ContainerBroker.__init__ """ + broker = ContainerBroker(':memory:', account='a', container='c') + self.assertEqual(broker.db_file, ':memory:') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + curs = conn.cursor() + curs.execute('SELECT 1') + self.assertEqual(curs.fetchall()[0][0], 1) + + def test_exception(self): + """ Test swift.common.db.ContainerBroker throwing a conn away after + unhandled exception """ + first_conn = None + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + first_conn = conn + try: + with broker.get() as conn: + self.assertEquals(first_conn, conn) + raise Exception('OMG') + except: + pass + self.assert_(broker.conn == None) + + def test_empty(self): + """ Test swift.common.db.ContainerBroker.empty """ + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + self.assert_(broker.empty()) + broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + self.assert_(not broker.empty()) + sleep(.00001) + broker.delete_object('o', normalize_timestamp(time())) + self.assert_(broker.empty()) + + def test_reclaim(self): + broker = ContainerBroker(':memory:', account='test_account', + container='test_container') + broker.initialize(normalize_timestamp('1')) + broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 0) + broker.reclaim(normalize_timestamp(time() - 999), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 0) + sleep(.00001) + broker.delete_object('o', normalize_timestamp(time())) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 1) + broker.reclaim(normalize_timestamp(time() - 999), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 1) + sleep(.00001) + broker.reclaim(normalize_timestamp(time()), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 0) + # Test the return values of reclaim() + broker.put_object('w', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('x', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('y', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('z', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + # Test before deletion + res = broker.reclaim(normalize_timestamp(time()), time()) + broker.delete_db(normalize_timestamp(time())) + + + def test_delete_object(self): + """ Test swift.common.db.ContainerBroker.delete_object """ + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 0) + sleep(.00001) + broker.delete_object('o', normalize_timestamp(time())) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 1) + + def test_put_object(self): + """ Test swift.common.db.ContainerBroker.put_object """ + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + # Create initial object + timestamp = normalize_timestamp(time()) + broker.put_object('"{}"', timestamp, 123, + 'application/x-test', + '5af83e3196bf99f440f31f2e1a6c9afe') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 123) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '5af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Reput same event + broker.put_object('"{}"', timestamp, 123, + 'application/x-test', + '5af83e3196bf99f440f31f2e1a6c9afe') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 123) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '5af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put new event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_object('"{}"', timestamp, 124, + 'application/x-test', + 'aa0749bacbc79ec65fe206943d8fe449') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 124) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + 'aa0749bacbc79ec65fe206943d8fe449') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put old event + otimestamp = normalize_timestamp(float(timestamp) - 1) + broker.put_object('"{}"', otimestamp, 124, + 'application/x-test', + 'aa0749bacbc79ec65fe206943d8fe449') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 124) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + 'aa0749bacbc79ec65fe206943d8fe449') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put old delete event + dtimestamp = normalize_timestamp(float(timestamp) - 1) + broker.put_object('"{}"', dtimestamp, 0, '', '', + deleted=1) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 124) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + 'aa0749bacbc79ec65fe206943d8fe449') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put new delete event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_object('"{}"', timestamp, 0, '', '', + deleted=1) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 1) + + # Put new event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_object('"{}"', timestamp, 123, + 'application/x-test', + '5af83e3196bf99f440f31f2e1a6c9afe') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 123) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '5af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # We'll use this later + sleep(.0001) + in_between_timestamp = normalize_timestamp(time()) + + # New post event + sleep(.0001) + previous_timestamp = timestamp + timestamp = normalize_timestamp(time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], + previous_timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 123) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '5af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put event from after last put but before last post + timestamp = in_between_timestamp + broker.put_object('"{}"', timestamp, 456, + 'application/x-test3', + '6af83e3196bf99f440f31f2e1a6c9afe') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 456) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test3') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '6af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + def test_get_info(self): + """ Test swift.common.db.ContainerBroker.get_info """ + broker = ContainerBroker(':memory:', account='test1', container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + self.assertEquals(info['container'], 'test2') + self.assertEquals(info['hash'], '00000000000000000000000000000000') + + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', + '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 123) + + sleep(.00001) + broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', + '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 246) + + sleep(.00001) + broker.put_object('o2', normalize_timestamp(time()), 1000, + 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 1123) + + sleep(.00001) + broker.delete_object('o1', normalize_timestamp(time())) + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 1000) + + sleep(.00001) + broker.delete_object('o2', normalize_timestamp(time())) + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + def test_get_report_info(self): + broker = ContainerBroker(':memory:', account='test1', container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + self.assertEquals(info['container'], 'test2') + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', + '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 123) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + sleep(.00001) + broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', + '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 246) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + sleep(.00001) + broker.put_object('o2', normalize_timestamp(time()), 1000, + 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 1123) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + put_timestamp = normalize_timestamp(time()) + sleep(.001) + delete_timestamp = normalize_timestamp(time()) + broker.reported(put_timestamp, delete_timestamp, 2, 1123) + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 1123) + self.assertEquals(info['reported_put_timestamp'], put_timestamp) + self.assertEquals(info['reported_delete_timestamp'], delete_timestamp) + self.assertEquals(info['reported_object_count'], 2) + self.assertEquals(info['reported_bytes_used'], 1123) + + sleep(.00001) + broker.delete_object('o1', normalize_timestamp(time())) + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 1000) + self.assertEquals(info['reported_object_count'], 2) + self.assertEquals(info['reported_bytes_used'], 1123) + + sleep(.00001) + broker.delete_object('o2', normalize_timestamp(time())) + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + self.assertEquals(info['reported_object_count'], 2) + self.assertEquals(info['reported_bytes_used'], 1123) + + def test_list_objects_iter(self): + """ Test swift.common.db.ContainerBroker.list_objects_iter """ + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + for obj1 in xrange(4): + for obj2 in xrange(125): + broker.put_object('%d/%04d' % (obj1, obj2), + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + for obj in xrange(125): + broker.put_object('2/0051/%04d' % obj, + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + + for obj in xrange(125): + broker.put_object('3/%04d/0049' % obj, + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + + listing = broker.list_objects_iter(100, '', None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0099') + + listing = broker.list_objects_iter(100, '0/0099', None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0/0100') + self.assertEquals(listing[-1][0], '1/0074') + + listing = broker.list_objects_iter(55, '1/0074', None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1/0075') + self.assertEquals(listing[-1][0], '2/0004') + + listing = broker.list_objects_iter(10, '', '0/01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0/0100') + self.assertEquals(listing[-1][0], '0/0109') + + listing = broker.list_objects_iter(10, '', '0/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0009') + + listing = broker.list_objects_iter(10, '', '', '/') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['0/', '1/', '2/', '3/']) + + listing = broker.list_objects_iter(10, '2', None, '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['2/', '3/']) + + listing = broker.list_objects_iter(10, '2/', None, '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3/']) + + listing = broker.list_objects_iter(10, '2/0050', '2/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '2/0051') + self.assertEquals(listing[1][0], '2/0051/') + self.assertEquals(listing[2][0], '2/0052') + self.assertEquals(listing[-1][0], '2/0059') + + listing = broker.list_objects_iter(10, '3/0045', '3/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3/0045/', '3/0046', '3/0046/', '3/0047', + '3/0047/', '3/0048', '3/0048/', '3/0049', + '3/0049/', '3/0050']) + + broker.put_object('3/0049/', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + listing = broker.list_objects_iter(10, '3/0048', None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3/0048/0049', '3/0049', '3/0049/', + '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049', + '3/0052', '3/0052/0049']) + + listing = broker.list_objects_iter(10, '3/0048', '3/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3/0048/', '3/0049', '3/0049/', '3/0050', + '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053']) + + listing = broker.list_objects_iter(10, None, '3/0049/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], + ['3/0049/', '3/0049/0049']) + + listing = broker.list_objects_iter(10, None, None, None, '3/0049') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3/0049/0049']) + + listing = broker.list_objects_iter(2, None, '3/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/']) + + listing = broker.list_objects_iter(2, None, None, None, '3') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001']) + + def test_list_objects_iter_prefix_delim(self): + """ Test swift.common.db.ContainerBroker.list_objects_iter """ + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + broker.put_object('/pets/dogs/1', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('/pets/dogs/2', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('/pets/fish/a', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('/pets/fish/b', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('/pets/fish_info.txt', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('/snakes', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + + #def list_objects_iter(self, limit, marker, prefix, delimiter, path=None, + # format=None): + listing = broker.list_objects_iter(100, None, '/pets/f', '/') + self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt']) + listing = broker.list_objects_iter(100, None, '/pets/fish', '/') + self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt']) + listing = broker.list_objects_iter(100, None, '/pets/fish/', '/') + self.assertEquals([row[0] for row in listing], ['/pets/fish/a', '/pets/fish/b']) + + def test_double_check_trailing_delimiter(self): + """ Test swift.common.db.ContainerBroker.list_objects_iter for a + container that has an odd file with a trailing delimiter """ + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/a/a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/a/b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b/a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b/b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('c', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + listing = broker.list_objects_iter(15, None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', 'b/a', 'b/b', 'c']) + listing = broker.list_objects_iter(15, None, '', '/') + self.assertEquals(len(listing), 5) + self.assertEquals([row[0] for row in listing], + ['a', 'a/', 'b', 'b/', 'c']) + listing = broker.list_objects_iter(15, None, 'a/', '/') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['a/', 'a/a', 'a/a/', 'a/b']) + listing = broker.list_objects_iter(15, None, 'b/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b/a', 'b/b']) + + def test_chexor(self): + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('a', normalize_timestamp(1), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + hasha = hashlib.md5('%s-%s' % ('a', '0000000001.00000')).digest() + hashb = hashlib.md5('%s-%s' % ('b', '0000000002.00000')).digest() + hashc = ''.join(('%2x' % (ord(a)^ord(b)) for a, b in zip(hasha, hashb))) + self.assertEquals(broker.get_info()['hash'], hashc) + broker.put_object('b', normalize_timestamp(3), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + hashb = hashlib.md5('%s-%s' % ('b', '0000000003.00000')).digest() + hashc = ''.join(('%02x' % (ord(a)^ord(b)) for a, b in zip(hasha, hashb))) + self.assertEquals(broker.get_info()['hash'], hashc) + + def test_newid(self): + """test DatabaseBroker.newid""" + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + id = broker.get_info()['id'] + broker.newid('someid') + self.assertNotEquals(id, broker.get_info()['id']) + + def test_get_items_since(self): + """test DatabaseBroker.get_items_since""" + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('a', normalize_timestamp(1), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + max_row = broker.get_replication_info()['max_row'] + broker.put_object('b', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + items = broker.get_items_since(max_row, 1000) + self.assertEquals(len(items), 1) + self.assertEquals(items[0]['name'], 'b') + + def test_sync_merging(self): + """ exercise the DatabaseBroker sync functions a bit """ + broker1 = ContainerBroker(':memory:', account='a', container='c') + broker1.initialize(normalize_timestamp('1')) + broker2 = ContainerBroker(':memory:', account='a', container='c') + broker2.initialize(normalize_timestamp('1')) + self.assertEquals(broker2.get_sync('12345'), -1) + broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}]) + broker2.merge_syncs(broker1.get_syncs()) + self.assertEquals(broker2.get_sync('12345'), 3) + + def test_merge_items(self): + broker1 = ContainerBroker(':memory:', account='a', container='c') + broker1.initialize(normalize_timestamp('1')) + broker2 = ContainerBroker(':memory:', account='a', container='c') + broker2.initialize(normalize_timestamp('1')) + broker1.put_object('a', normalize_timestamp(1), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker1.put_object('b', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + id = broker1.get_info()['id'] + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(len(items), 2) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + broker1.put_object('c', normalize_timestamp(3), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(len(items), 3) + self.assertEquals(['a', 'b', 'c'], + sorted([rec['name'] for rec in items])) + + def test_merge_items_overwrite(self): + """test DatabaseBroker.merge_items""" + broker1 = ContainerBroker(':memory:', account='a', container='c') + broker1.initialize(normalize_timestamp('1')) + id = broker1.get_info()['id'] + broker2 = ContainerBroker(':memory:', account='a', container='c') + broker2.initialize(normalize_timestamp('1')) + broker1.put_object('a', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker1.put_object('b', normalize_timestamp(3), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + broker1.put_object('a', normalize_timestamp(4), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + for rec in items: + if rec['name'] == 'a': + self.assertEquals(rec['created_at'], normalize_timestamp(4)) + if rec['name'] == 'b': + self.assertEquals(rec['created_at'], normalize_timestamp(3)) + + def test_merge_items_post_overwrite_out_of_order(self): + """test DatabaseBroker.merge_items""" + broker1 = ContainerBroker(':memory:', account='a', container='c') + broker1.initialize(normalize_timestamp('1')) + id = broker1.get_info()['id'] + broker2 = ContainerBroker(':memory:', account='a', container='c') + broker2.initialize(normalize_timestamp('1')) + broker1.put_object('a', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker1.put_object('b', normalize_timestamp(3), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + broker1.put_object('a', normalize_timestamp(4), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + for rec in items: + if rec['name'] == 'a': + self.assertEquals(rec['created_at'], normalize_timestamp(4)) + if rec['name'] == 'b': + self.assertEquals(rec['created_at'], normalize_timestamp(3)) + self.assertEquals(rec['content_type'], 'text/plain') + items = broker2.get_items_since(-1, 1000) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + for rec in items: + if rec['name'] == 'a': + self.assertEquals(rec['created_at'], normalize_timestamp(4)) + if rec['name'] == 'b': + self.assertEquals(rec['created_at'], normalize_timestamp(3)) + broker1.put_object('b', normalize_timestamp(5), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + for rec in items: + if rec['name'] == 'a': + self.assertEquals(rec['created_at'], normalize_timestamp(4)) + if rec['name'] == 'b': + self.assertEquals(rec['created_at'], normalize_timestamp(5)) + self.assertEquals(rec['content_type'], 'text/plain') + + +class TestAccountBroker(unittest.TestCase): + """ Tests for swift.common.db.AccountBroker """ + + def test_creation(self): + """ Test swift.common.db.AccountBroker.__init__ """ + broker = AccountBroker(':memory:', account='a') + self.assertEqual(broker.db_file, ':memory:') + got_exc = False + try: + with broker.get() as conn: + pass + except: + got_exc = True + self.assert_(got_exc) + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + curs = conn.cursor() + curs.execute('SELECT 1') + self.assertEqual(curs.fetchall()[0][0], 1) + + def test_exception(self): + """ Test swift.common.db.AccountBroker throwing a conn away after + exception """ + first_conn = None + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + first_conn = conn + try: + with broker.get() as conn: + self.assertEquals(first_conn, conn) + raise Exception('OMG') + except: + pass + self.assert_(broker.conn == None) + + def test_empty(self): + """ Test swift.common.db.AccountBroker.empty """ + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + self.assert_(broker.empty()) + broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) + self.assert_(not broker.empty()) + sleep(.00001) + broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) + self.assert_(broker.empty()) + + def test_reclaim(self): + broker = AccountBroker(':memory:', account='test_account') + broker.initialize(normalize_timestamp('1')) + broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 0) + broker.reclaim(normalize_timestamp(time() - 999), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 0) + sleep(.00001) + broker.put_container('c', 0, normalize_timestamp(time()), 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 1) + broker.reclaim(normalize_timestamp(time() - 999), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 1) + sleep(.00001) + broker.reclaim(normalize_timestamp(time()), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 0) + # Test reclaim after deletion. Create 3 test containers + broker.put_container('x', 0, 0, 0, 0) + broker.put_container('y', 0, 0, 0, 0) + broker.put_container('z', 0, 0, 0, 0) + res = broker.reclaim(normalize_timestamp(time()), time()) + # self.assertEquals(len(res), 2) + # self.assert_(isinstance(res, tuple)) + # containers, account_name = res + # self.assert_(containers is None) + # self.assert_(account_name is None) + # Now delete the account + broker.delete_db(normalize_timestamp(time())) + res = broker.reclaim(normalize_timestamp(time()), time()) + # self.assertEquals(len(res), 2) + # self.assert_(isinstance(res, tuple)) + # containers, account_name = res + # self.assertEquals(account_name, 'test_account') + # self.assertEquals(len(containers), 3) + # self.assert_('x' in containers) + # self.assert_('y' in containers) + # self.assert_('z' in containers) + # self.assert_('a' not in containers) + + + def test_delete_container(self): + """ Test swift.common.db.AccountBroker.delete_container """ + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 0) + sleep(.00001) + broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 1) + + def test_get_container_timestamp(self): + """ Test swift.common.db.AccountBroker.get_container_timestamp """ + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + + # Create initial container + timestamp = normalize_timestamp(time()) + broker.put_container('container_name', timestamp, 0, 0, 0) + # test extant map + ts = broker.get_container_timestamp('container_name') + self.assertEquals(ts, timestamp) + # test missing map + ts = broker.get_container_timestamp('something else') + self.assertEquals(ts, None) + + def test_put_container(self): + """ Test swift.common.db.AccountBroker.put_container """ + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + + # Create initial container + timestamp = normalize_timestamp(time()) + broker.put_container('"{}"', timestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Reput same event + broker.put_container('"{}"', timestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Put new event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_container('"{}"', timestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Put old event + otimestamp = normalize_timestamp(float(timestamp) - 1) + broker.put_container('"{}"', otimestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Put old delete event + dtimestamp = normalize_timestamp(float(timestamp) - 1) + broker.put_container('"{}"', 0, dtimestamp, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT delete_timestamp FROM container").fetchone()[0], + dtimestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Put new delete event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_container('"{}"', 0, timestamp, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT delete_timestamp FROM container").fetchone()[0], + timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 1) + + # Put new event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_container('"{}"', timestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + def test_get_info(self): + """ Test swift.common.db.AccountBroker.get_info """ + broker = AccountBroker(':memory:', account='test1') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + self.assertEquals(info['hash'], '00000000000000000000000000000000') + + info = broker.get_info() + self.assertEquals(info['container_count'], 0) + + broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 1) + + sleep(.00001) + broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 2) + + sleep(.00001) + broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 2) + + sleep(.00001) + broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 1) + + sleep(.00001) + broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 0) + + def test_list_containers_iter(self): + """ Test swift.common.db.AccountBroker.list_containers_iter """ + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + for cont1 in xrange(4): + for cont2 in xrange(125): + broker.put_container('%d/%04d' % (cont1, cont2), + normalize_timestamp(time()), 0, 0, 0) + for cont in xrange(125): + broker.put_container('2/0051/%04d' % cont, + normalize_timestamp(time()), 0, 0, 0) + + for cont in xrange(125): + broker.put_container('3/%04d/0049' % cont, + normalize_timestamp(time()), 0, 0, 0) + + listing = broker.list_containers_iter(100, '', None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0099') + + listing = broker.list_containers_iter(100, '0/0099', None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0/0100') + self.assertEquals(listing[-1][0], '1/0074') + + listing = broker.list_containers_iter(55, '1/0074', None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1/0075') + self.assertEquals(listing[-1][0], '2/0004') + + listing = broker.list_containers_iter(10, '', '0/01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0/0100') + self.assertEquals(listing[-1][0], '0/0109') + + listing = broker.list_containers_iter(10, '', '0/01', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0/0100') + self.assertEquals(listing[-1][0], '0/0109') + + listing = broker.list_containers_iter(10, '', '0/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0009') + + listing = broker.list_containers_iter(10, '', '', '/') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['0/', '1/', '2/', '3/']) + + listing = broker.list_containers_iter(10, '2/', None, '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3/']) + + listing = broker.list_containers_iter(10, '', '2', '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['2/']) + + listing = broker.list_containers_iter(10, '2/0050', '2/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '2/0051') + self.assertEquals(listing[1][0], '2/0051/') + self.assertEquals(listing[2][0], '2/0052') + self.assertEquals(listing[-1][0], '2/0059') + + listing = broker.list_containers_iter(10, '3/0045', '3/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3/0045/', '3/0046', '3/0046/', '3/0047', + '3/0047/', '3/0048', '3/0048/', '3/0049', + '3/0049/', '3/0050']) + + broker.put_container('3/0049/', normalize_timestamp(time()), 0, 0, 0) + listing = broker.list_containers_iter(10, '3/0048', None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3/0048/0049', '3/0049', '3/0049/', '3/0049/0049', + '3/0050', '3/0050/0049', '3/0051', '3/0051/0049', + '3/0052', '3/0052/0049']) + + listing = broker.list_containers_iter(10, '3/0048', '3/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3/0048/', '3/0049', '3/0049/', '3/0050', + '3/0050/', '3/0051', '3/0051/', '3/0052', + '3/0052/', '3/0053']) + + listing = broker.list_containers_iter(10, None, '3/0049/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], + ['3/0049/', '3/0049/0049']) + + def test_double_check_trailing_delimiter(self): + """ Test swift.common.db.AccountBroker.list_containers_iter for an + account that has an odd file with a trailing delimiter """ + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + broker.put_container('a', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a/', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a/a', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a/a/a', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a/a/b', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a/b', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('b', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('b/a', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('b/b', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) + listing = broker.list_containers_iter(15, None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', + 'b/a', 'b/b', 'c']) + listing = broker.list_containers_iter(15, None, '', '/') + self.assertEquals(len(listing), 5) + self.assertEquals([row[0] for row in listing], + ['a', 'a/', 'b', 'b/', 'c']) + listing = broker.list_containers_iter(15, None, 'a/', '/') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['a/', 'a/a', 'a/a/', 'a/b']) + listing = broker.list_containers_iter(15, None, 'b/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b/a', 'b/b']) + + def test_chexor(self): + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + broker.put_container('a', normalize_timestamp(1), + normalize_timestamp(0), 0, 0) + broker.put_container('b', normalize_timestamp(2), + normalize_timestamp(0), 0, 0) + hasha = hashlib.md5('%s-%s' % + ('a', '0000000001.00000-0000000000.00000-0-0') + ).digest() + hashb = hashlib.md5('%s-%s' % + ('b', '0000000002.00000-0000000000.00000-0-0') + ).digest() + hashc = \ + ''.join(('%02x' % (ord(a)^ord(b)) for a, b in zip(hasha, hashb))) + self.assertEquals(broker.get_info()['hash'], hashc) + broker.put_container('b', normalize_timestamp(3), + normalize_timestamp(0), 0, 0) + hashb = hashlib.md5('%s-%s' % + ('b', '0000000003.00000-0000000000.00000-0-0') + ).digest() + hashc = \ + ''.join(('%02x' % (ord(a)^ord(b)) for a, b in zip(hasha, hashb))) + self.assertEquals(broker.get_info()['hash'], hashc) + + def test_merge_items(self): + broker1 = AccountBroker(':memory:', account='a') + broker1.initialize(normalize_timestamp('1')) + broker2 = AccountBroker(':memory:', account='a') + broker2.initialize(normalize_timestamp('1')) + broker1.put_container('a', normalize_timestamp(1), 0, 0, 0) + broker1.put_container('b', normalize_timestamp(2), 0, 0, 0) + id = broker1.get_info()['id'] + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(len(items), 2) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + broker1.put_container('c', normalize_timestamp(3), 0, 0, 0) + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(len(items), 3) + self.assertEquals(['a', 'b', 'c'], + sorted([rec['name'] for rec in items])) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py new file mode 100644 index 0000000000..bd247772f0 --- /dev/null +++ b/test/unit/common/test_db_replicator.py @@ -0,0 +1,240 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from contextlib import contextmanager +import os +import logging + +from swift.common import db_replicator +from swift.common import db, utils +from swift.container import server as container_server + + +def teardown_module(): + "clean up my monkey patching" + reload(db_replicator) + +@contextmanager +def lock_parent_directory(filename): + yield True + +class FakeRing: + class Ring: + devs = [] + def __init__(self, path): + pass + def get_part_nodes(self, part): + return [] + def get_more_nodes(self, *args): + return [] + +class FakeProcess: + def __init__(self, *codes): + self.codes = iter(codes) + def __call__(self, *args, **kwargs): + class Failure: + def communicate(innerself): + next = self.codes.next() + if isinstance(next, int): + innerself.returncode = next + return next + raise next + return Failure() + +@contextmanager +def _mock_process(*args): + orig_process = db_replicator.subprocess.Popen + db_replicator.subprocess.Popen = FakeProcess(*args) + yield + db_replicator.subprocess.Popen = orig_process + +class PostReplHttp: + def __init__(self, response=None): + self.response = response + posted = False + host = 'localhost' + def post(self, *args): + self.posted = True + class Response: + status = 200 + data = self.response + def read(innerself): + return self.response + return Response() + +class ChangingMtimesOs: + def __init__(self): + self.mtime = 0 + self.path = self + self.basename = os.path.basename + def getmtime(self, file): + self.mtime += 1 + return self.mtime + +class FakeBroker: + db_file = __file__ + def __init__(self, *args, **kwargs): + return None + @contextmanager + def lock(self): + yield True + def get_sync(self, *args, **kwargs): + return 5 + def get_syncs(self): + return [] + def get_items_since(self, point, *args): + if point == 0: + return [{'ROWID': 1}] + return [] + def merge_syncs(self, *args, **kwargs): + self.args = args + def merge_items(self, *args): + self.args = args + def get_replication_info(self): + return {'delete_timestamp': 0, 'put_timestamp': 1, 'count': 0} + def reclaim(self, item_timestamp, sync_timestamp): + pass + +db_replicator.ring = FakeRing() + + +class TestReplicator(db_replicator.Replicator): + server_type = 'container' + ring_file = 'container.ring.gz' + brokerclass = FakeBroker + datadir = container_server.DATADIR + default_port = 1000 + +class TestDBReplicator(unittest.TestCase): + + def test_repl_connection(self): + node = {'ip': '127.0.0.1', 'port': 80, 'device': 'sdb1'} + conn = db_replicator.ReplConnection(node, '1234567890', 'abcdefg', + logging.getLogger()) + def req(method, path, body, headers): + self.assertEquals(method, 'POST') + self.assertEquals(headers['Content-Type'], 'application/json') + class Resp: + def read(self): return 'data' + resp = Resp() + conn.request = req + conn.getresponse = lambda *args: resp + self.assertEquals(conn.post(1, 2, 3), resp) + def other_req(method, path, body, headers): + raise Exception('blah') + conn.request = other_req + self.assertEquals(conn.post(1, 2, 3), None) + + def test_rsync_file(self): + replicator = TestReplicator({}, {}) + with _mock_process(-1): + fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} + self.assertEquals(False, + replicator._rsync_file('/some/file', 'remote:/some/file')) + with _mock_process(0): + fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} + self.assertEquals(True, + replicator._rsync_file('/some/file', 'remote:/some/file')) + + def test_rsync_db(self): + replicator = TestReplicator({}, {}) + replicator._rsync_file = lambda *args: True + fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} + replicator._rsync_db(FakeBroker(), fake_device, PostReplHttp(), 'abcd') + + def test_in_sync(self): + replicator = TestReplicator({}, {}) + self.assertEquals(replicator._in_sync( + {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, + {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, + FakeBroker(), -1), True) + self.assertEquals(replicator._in_sync( + {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, + {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'b'}, + FakeBroker(), -1), True) + self.assertEquals(bool(replicator._in_sync( + {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'c'}, + {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'd'}, + FakeBroker(), -1)), False) + + def test_replicate_once(self): + replicator = TestReplicator({}, {}) + replicator.replicate_once() + + def test_usync(self): + fake_http = PostReplHttp() + replicator = TestReplicator({}, {}) + replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890') + + def test_repl_to_node(self): + replicator = TestReplicator({}, {}) + fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000} + fake_info = {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b', + 'created_at': 100, 'put_timestamp': 0, + 'delete_timestamp': 0} + replicator._http_connect = lambda *args: PostReplHttp('{"id": 3, "point": -1}') + self.assertEquals(replicator._repl_to_node( + fake_node, FakeBroker(), '0', fake_info), True) + + def test_stats(self): + # I'm not sure how to test that this logs the right thing, + # but we can at least make sure it gets covered. + replicator = TestReplicator({}, {}) + replicator._zero_stats() + replicator._report_stats() + + def test_replicate_object(self): + db_replicator.lock_parent_directory = lock_parent_directory + replicator = TestReplicator({}, {}) + replicator._replicate_object('0', 'file', 'node_id') + + +# def test_dispatch(self): +# rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) +# no_op = lambda *args, **kwargs: True +# self.assertEquals(rpc.dispatch(('drv', 'part', 'hash'), ('op',) +# ).status_int, 400) +# rpc.mount_check = True +# self.assertEquals(rpc.dispatch(('drv', 'part', 'hash'), ['op',] +# ).status_int, 507) +# rpc.mount_check = False +# rpc.rsync_then_merge = lambda drive, db_file, args: self.assertEquals(args, ['test1']) +# rpc.complete_rsync = lambda drive, db_file, args: self.assertEquals(args, ['test2']) +# rpc.dispatch(('drv', 'part', 'hash'), ['rsync_then_merge','test1']) +# rpc.dispatch(('drv', 'part', 'hash'), ['complete_rsync','test2']) +# rpc.dispatch(('drv', 'part', 'hash'), ['other_op',]) + + def test_rsync_then_merge(self): + rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) + rpc.rsync_then_merge('sda1', '/srv/swift/blah', ('a', 'b')) + + def test_merge_items(self): + rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) + fake_broker = FakeBroker() + args = ('a', 'b') + rpc.merge_items(fake_broker, args) + self.assertEquals(fake_broker.args, args) + + def test_merge_syncs(self): + rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) + fake_broker = FakeBroker() + args = ('a', 'b') + rpc.merge_syncs(fake_broker, args) + self.assertEquals(fake_broker.args, (args[0],)) + +if __name__ == '__main__': + unittest.main() + diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py new file mode 100644 index 0000000000..029791289e --- /dev/null +++ b/test/unit/common/test_direct_client.py @@ -0,0 +1,28 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Tests + +import unittest +from swift.common import direct_client + +class TestAuditor(unittest.TestCase): + + def test_placeholder(self): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_exceptions.py b/test/unit/common/test_exceptions.py new file mode 100644 index 0000000000..bfb251b139 --- /dev/null +++ b/test/unit/common/test_exceptions.py @@ -0,0 +1,28 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Tests + +import unittest +from swift.common import exceptions + +class TestAuditor(unittest.TestCase): + + def test_placeholder(self): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_healthcheck.py b/test/unit/common/test_healthcheck.py new file mode 100644 index 0000000000..3256d474c0 --- /dev/null +++ b/test/unit/common/test_healthcheck.py @@ -0,0 +1,33 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from webob import Request + +from swift.common import healthcheck + + +class TestHealthCheck(unittest.TestCase): + + def test_healthcheck(self): + controller = healthcheck.HealthCheckController() + req = Request.blank('/any/path', environ={'REQUEST_METHOD': 'GET'}) + resp = controller.GET(req) + self.assertEquals(resp.status_int, 200) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_memcached.py b/test/unit/common/test_memcached.py new file mode 100644 index 0000000000..8dab623272 --- /dev/null +++ b/test/unit/common/test_memcached.py @@ -0,0 +1,196 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for swift.common.utils """ + +from __future__ import with_statement +import hashlib +import logging +import socket +import time +import unittest + +from swift.common import memcached + + +class NullLoggingHandler(logging.Handler): + + def emit(self, record): + pass + + +class ExplodingMockMemcached(object): + exploded = False + def sendall(self, string): + self.exploded = True + raise socket.error() + def readline(self): + self.exploded = True + raise socket.error() + def read(self, size): + self.exploded = True + raise socket.error() + +class MockMemcached(object): + def __init__(self): + self.inbuf = '' + self.outbuf = '' + self.cache = {} + self.down = False + self.exc_on_delete = False + + def sendall(self, string): + if self.down: + raise Exception('mock is down') + self.inbuf += string + while '\n' in self.inbuf: + cmd, self.inbuf = self.inbuf.split('\n', 1) + parts = cmd.split() + if parts[0].lower() == 'set': + self.cache[parts[1]] = parts[2], parts[3], \ + self.inbuf[:int(parts[4])] + self.inbuf = self.inbuf[int(parts[4])+2:] + if len(parts) < 6 or parts[5] != 'noreply': + self.outbuf += 'STORED\r\n' + elif parts[0].lower() == 'add': + value = self.inbuf[:int(parts[4])] + self.inbuf = self.inbuf[int(parts[4])+2:] + if parts[1] in self.cache: + if len(parts) < 6 or parts[5] != 'noreply': + self.outbuf += 'NOT_STORED\r\n' + else: + self.cache[parts[1]] = parts[2], parts[3], value + if len(parts) < 6 or parts[5] != 'noreply': + self.outbuf += 'STORED\r\n' + elif parts[0].lower() == 'delete': + if self.exc_on_delete: + raise Exception('mock is has exc_on_delete set') + if parts[1] in self.cache: + del self.cache[parts[1]] + if 'noreply' not in parts: + self.outbuf += 'DELETED\r\n' + elif 'noreply' not in parts: + self.outbuf += 'NOT_FOUND\r\n' + elif parts[0].lower() == 'get': + for key in parts[1:]: + if key in self.cache: + val = self.cache[key] + self.outbuf += 'VALUE %s %s %s\r\n' % (key, val[0], len(val[2])) + self.outbuf += val[2] + '\r\n' + self.outbuf += 'END\r\n' + elif parts[0].lower() == 'incr': + if parts[1] in self.cache: + val = list(self.cache[parts[1]]) + val[2] = str(int(val[2]) + int(parts[2])) + self.cache[parts[1]] = val + self.outbuf += str(val[2]) + '\r\n' + else: + self.outbuf += 'NOT_FOUND\r\n' + def readline(self): + if self.down: + raise Exception('mock is down') + if '\n' in self.outbuf: + response, self.outbuf = self.outbuf.split('\n', 1) + return response+'\n' + def read(self, size): + if self.down: + raise Exception('mock is down') + if len(self.outbuf) >= size: + response = self.outbuf[:size] + self.outbuf = self.outbuf[size:] + return response + +class TestMemcached(unittest.TestCase): + """ Tests for swift.common.memcached""" + + def test_get_conns(self): + sock1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock1.bind(('127.0.0.1', 0)) + sock1.listen(1) + sock1ipport = '%s:%s' % sock1.getsockname() + sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock2.bind(('127.0.0.1', 0)) + sock2.listen(1) + sock2ipport = '%s:%s' % sock2.getsockname() + memcache_client = memcached.MemcacheRing([sock1ipport, sock2ipport]) + for conn in memcache_client._get_conns('40000000000000000000000000000000'): + self.assert_('%s:%s' % conn[2].getpeername() in (sock1ipport, sock2ipport)) + + def test_set_get(self): + memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) + mock = MockMemcached() + memcache_client._client_cache['1.2.3.4:11211'] = [(mock, mock)] * 2 + memcache_client.set('some_key', [1, 2, 3]) + self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) + memcache_client.set('some_key', [4, 5, 6]) + self.assertEquals(memcache_client.get('some_key'), [4, 5, 6]) + self.assert_(float(mock.cache.values()[0][1]) == 0) + esttimeout = time.time() + 10 + memcache_client.set('some_key', [1, 2, 3], timeout=10) + self.assert_(-1 <= float(mock.cache.values()[0][1]) - esttimeout <= 1) + + def test_incr(self): + memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) + mock = MockMemcached() + memcache_client._client_cache['1.2.3.4:11211'] = [(mock, mock)] * 2 + memcache_client.incr('some_key', delta=5) + self.assertEquals(memcache_client.get('some_key'), '5') + memcache_client.incr('some_key', delta=5) + self.assertEquals(memcache_client.get('some_key'), '10') + memcache_client.incr('some_key', delta=1) + self.assertEquals(memcache_client.get('some_key'), '11') + + def test_retry(self): + logging.getLogger().addHandler(NullLoggingHandler()) + memcache_client = memcached.MemcacheRing(['1.2.3.4:11211', '1.2.3.5:11211']) + mock1 = ExplodingMockMemcached() + mock2 = MockMemcached() + memcache_client._client_cache['1.2.3.4:11211'] = [(mock2, mock2)] + memcache_client._client_cache['1.2.3.5:11211'] = [(mock1, mock1)] + memcache_client.set('some_key', [1, 2, 3]) + self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) + self.assertEquals(mock1.exploded, True) + + def test_delete(self): + memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) + mock = MockMemcached() + memcache_client._client_cache['1.2.3.4:11211'] = [(mock, mock)] * 2 + memcache_client.set('some_key', [1, 2, 3]) + self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) + memcache_client.delete('some_key') + self.assertEquals(memcache_client.get('some_key'), None) + + def test_multi(self): + memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) + mock = MockMemcached() + memcache_client._client_cache['1.2.3.4:11211'] = [(mock, mock)] * 2 + memcache_client.set_multi( + {'some_key1': [1, 2, 3], 'some_key2': [4, 5, 6]}, 'multi_key') + self.assertEquals( + memcache_client.get_multi(('some_key2', 'some_key1'), 'multi_key'), + [[4, 5, 6], [1, 2, 3]]) + esttimeout = time.time() + 10 + memcache_client.set_multi( + {'some_key1': [1, 2, 3], 'some_key2': [4, 5, 6]}, 'multi_key', + timeout=10) + self.assert_(-1 <= float(mock.cache.values()[0][1]) - esttimeout <= 1) + self.assert_(-1 <= float(mock.cache.values()[1][1]) - esttimeout <= 1) + self.assertEquals(memcache_client.get_multi(('some_key2', 'some_key1', + 'not_exists'), 'multi_key'), [[4, 5, 6], [1, 2, 3], None]) + + +if __name__ == '__main__': + unittest.main() + diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py new file mode 100644 index 0000000000..eef3b19b74 --- /dev/null +++ b/test/unit/common/test_utils.py @@ -0,0 +1,246 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for swift.common.utils """ + +from __future__ import with_statement +import logging +import mimetools +import os +import socket +import sys +import unittest +from getpass import getuser +from shutil import rmtree +from StringIO import StringIO + +from eventlet import sleep + +from swift.common import utils + + +class TestUtils(unittest.TestCase): + """ Tests for swift.common.utils """ + + def setUp(self): + utils.HASH_PATH_SUFFIX = 'endcap' + + def test_normalize_timestamp(self): + """ Test swift.common.utils.normalize_timestamp """ + self.assertEquals(utils.normalize_timestamp('1253327593.48174'), + "1253327593.48174") + self.assertEquals(utils.normalize_timestamp(1253327593.48174), + "1253327593.48174") + self.assertEquals(utils.normalize_timestamp('1253327593.48'), + "1253327593.48000") + self.assertEquals(utils.normalize_timestamp(1253327593.48), + "1253327593.48000") + self.assertEquals(utils.normalize_timestamp('253327593.48'), + "0253327593.48000") + self.assertEquals(utils.normalize_timestamp(253327593.48), + "0253327593.48000") + self.assertEquals(utils.normalize_timestamp('1253327593'), + "1253327593.00000") + self.assertEquals(utils.normalize_timestamp(1253327593), + "1253327593.00000") + self.assertRaises(ValueError, utils.normalize_timestamp, '') + self.assertRaises(ValueError, utils.normalize_timestamp, 'abc') + + def test_mkdirs(self): + testroot = os.path.join(os.path.dirname(__file__), 'mkdirs') + try: + os.unlink(testroot) + except: + pass + rmtree(testroot, ignore_errors=1) + self.assert_(not os.path.exists(testroot)) + utils.mkdirs(testroot) + self.assert_(os.path.exists(testroot)) + utils.mkdirs(testroot) + self.assert_(os.path.exists(testroot)) + rmtree(testroot, ignore_errors=1) + + testdir = os.path.join(testroot, 'one/two/three') + self.assert_(not os.path.exists(testdir)) + utils.mkdirs(testdir) + self.assert_(os.path.exists(testdir)) + utils.mkdirs(testdir) + self.assert_(os.path.exists(testdir)) + rmtree(testroot, ignore_errors=1) + + open(testroot, 'wb').close() + self.assert_(not os.path.exists(testdir)) + self.assertRaises(OSError, utils.mkdirs, testdir) + os.unlink(testroot) + + def test_split_path(self): + """ Test swift.common.utils.split_account_path """ + self.assertRaises(ValueError, utils.split_path, '') + self.assertRaises(ValueError, utils.split_path, '/') + self.assertRaises(ValueError, utils.split_path, '//') + self.assertEquals(utils.split_path('/a'), ['a']) + self.assertRaises(ValueError, utils.split_path, '//a') + self.assertEquals(utils.split_path('/a/'), ['a']) + self.assertRaises(ValueError, utils.split_path, '/a/c') + self.assertRaises(ValueError, utils.split_path, '//c') + self.assertRaises(ValueError, utils.split_path, '/a/c/') + self.assertRaises(ValueError, utils.split_path, '/a//') + self.assertRaises(ValueError, utils.split_path, '/a', 2) + self.assertRaises(ValueError, utils.split_path, '/a', 2, 3) + self.assertRaises(ValueError, utils.split_path, '/a', 2, 3, True) + self.assertEquals(utils.split_path('/a/c', 2), ['a', 'c']) + self.assertEquals(utils.split_path('/a/c/o', 3), ['a', 'c', 'o']) + self.assertRaises(ValueError, utils.split_path, '/a/c/o/r', 3, 3) + self.assertEquals(utils.split_path('/a/c/o/r', 3, 3, True), + ['a', 'c', 'o/r']) + self.assertEquals(utils.split_path('/a/c', 2, 3, True), + ['a', 'c', None]) + self.assertRaises(ValueError, utils.split_path, '/a', 5, 4) + self.assertEquals(utils.split_path('/a/c/', 2), ['a', 'c']) + self.assertEquals(utils.split_path('/a/c/', 2, 3), ['a', 'c', '']) + try: + utils.split_path('o\nn e', 2) + except ValueError, err: + self.assertEquals(str(err), 'Invalid path: o%0An%20e') + try: + utils.split_path('o\nn e', 2, 3, True) + except ValueError, err: + self.assertEquals(str(err), 'Invalid path: o%0An%20e') + + def test_NullLogger(self): + """ Test swift.common.utils.NullLogger """ + sio = StringIO() + nl = utils.NullLogger() + nl.write('test') + self.assertEquals(sio.getvalue(), '') + + def test_LoggerFileObject(self): + orig_stdout = sys.stdout + orig_stderr = sys.stderr + sio = StringIO() + handler = logging.StreamHandler(sio) + logger = logging.getLogger() + logger.addHandler(handler) + lfo = utils.LoggerFileObject(logger) + print 'test1' + self.assertEquals(sio.getvalue(), '') + sys.stdout = lfo + print 'test2' + self.assertEquals(sio.getvalue(), 'STDOUT: test2\n') + sys.stderr = lfo + print >>sys.stderr, 'test4' + self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n') + sys.stdout = orig_stdout + print 'test5' + self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n') + print >>sys.stderr, 'test6' + self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' + 'STDOUT: test6\n') + sys.stderr = orig_stderr + print 'test8' + self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' + 'STDOUT: test6\n') + lfo.writelines(['a', 'b', 'c']) + self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' + 'STDOUT: test6\nSTDOUT: a#012b#012c\n') + lfo.close() + lfo.write('d') + self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' + 'STDOUT: test6\nSTDOUT: a#012b#012c\nSTDOUT: d\n') + lfo.flush() + self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' + 'STDOUT: test6\nSTDOUT: a#012b#012c\nSTDOUT: d\n') + got_exc = False + try: + for line in lfo: + pass + except: + got_exc = True + self.assert_(got_exc) + got_exc = False + try: + for line in lfo.xreadlines(): + pass + except: + got_exc = True + self.assert_(got_exc) + self.assertRaises(IOError, lfo.read) + self.assertRaises(IOError, lfo.read, 1024) + self.assertRaises(IOError, lfo.readline) + self.assertRaises(IOError, lfo.readline, 1024) + lfo.tell() + + def test_drop_privileges(self): + # Note that this doesn't really drop privileges as it just sets them to + # what they already are; but it exercises the code at least. + utils.drop_privileges(getuser()) + + def test_NamedLogger(self): + sio = StringIO() + logger = logging.getLogger() + logger.addHandler(logging.StreamHandler(sio)) + nl = utils.NamedLogger(logger, 'server') + nl.warn('test') + self.assertEquals(sio.getvalue(), 'server test\n') + + def test_get_logger(self): + sio = StringIO() + logger = logging.getLogger() + logger.addHandler(logging.StreamHandler(sio)) + logger = utils.get_logger(None, 'server') + logger.warn('test1') + self.assertEquals(sio.getvalue(), 'server test1\n') + logger.debug('test2') + self.assertEquals(sio.getvalue(), 'server test1\n') + logger = utils.get_logger({'log_level': 'DEBUG'}, 'server') + logger.debug('test3') + self.assertEquals(sio.getvalue(), 'server test1\nserver test3\n') + # Doesn't really test that the log facility is truly being used all the + # way to syslog; but exercises the code. + logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server') + logger.warn('test4') + self.assertEquals(sio.getvalue(), + 'server test1\nserver test3\nserver test4\n') + logger.debug('test5') + self.assertEquals(sio.getvalue(), + 'server test1\nserver test3\nserver test4\n') + + def test_storage_directory(self): + self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), + 'objects/1/DEF/ABCDEF') + + def test_whataremyips(self): + myips = utils.whataremyips() + self.assert_(len(myips) > 1) + self.assert_('127.0.0.1' in myips) + + def test_hash_path(self): + # Yes, these tests are deliberately very fragile. We want to make sure + # that if someones changes the results hash_path produces, they know it. + self.assertEquals(utils.hash_path('a'), + '1c84525acb02107ea475dcd3d09c2c58') + self.assertEquals(utils.hash_path('a', 'c'), + '33379ecb053aa5c9e356c68997cbb59e') + self.assertEquals(utils.hash_path('a', 'c', 'o'), + '06fbf0b514e5199dfc4e00f42eb5ea83') + self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), + '06fbf0b514e5199dfc4e00f42eb5ea83') + self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True), + '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN\x00\xf4.\xb5\xea\x83') + self.assertRaises(ValueError, utils.hash_path, 'a', object='o') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py new file mode 100644 index 0000000000..1f81962ff3 --- /dev/null +++ b/test/unit/common/test_wsgi.py @@ -0,0 +1,76 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for swift.common.utils """ + +from __future__ import with_statement +import logging +import mimetools +import os +import socket +import sys +import unittest +from getpass import getuser +from shutil import rmtree +from StringIO import StringIO + +from eventlet import sleep + +from swift.common import wsgi + + +class TestWSGI(unittest.TestCase): + """ Tests for swift.common.wsgi """ + + def test_monkey_patch_mimetools(self): + sio = StringIO('blah') + self.assertEquals(mimetools.Message(sio).type, 'text/plain') + sio = StringIO('blah') + self.assertEquals(mimetools.Message(sio).plisttext, '') + sio = StringIO('blah') + self.assertEquals(mimetools.Message(sio).maintype, 'text') + sio = StringIO('blah') + self.assertEquals(mimetools.Message(sio).subtype, 'plain') + sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') + self.assertEquals(mimetools.Message(sio).type, 'text/html') + sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') + self.assertEquals(mimetools.Message(sio).plisttext, + '; charset=ISO-8859-4') + sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') + self.assertEquals(mimetools.Message(sio).maintype, 'text') + sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') + self.assertEquals(mimetools.Message(sio).subtype, 'html') + + wsgi.monkey_patch_mimetools() + sio = StringIO('blah') + self.assertEquals(mimetools.Message(sio).type, None) + sio = StringIO('blah') + self.assertEquals(mimetools.Message(sio).plisttext, '') + sio = StringIO('blah') + self.assertEquals(mimetools.Message(sio).maintype, None) + sio = StringIO('blah') + self.assertEquals(mimetools.Message(sio).subtype, None) + sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') + self.assertEquals(mimetools.Message(sio).type, 'text/html') + sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') + self.assertEquals(mimetools.Message(sio).plisttext, + '; charset=ISO-8859-4') + sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') + self.assertEquals(mimetools.Message(sio).maintype, 'text') + sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') + self.assertEquals(mimetools.Message(sio).subtype, 'html') + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/container/__init__.py b/test/unit/container/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/container/test_auditor.py b/test/unit/container/test_auditor.py new file mode 100644 index 0000000000..1093cc809d --- /dev/null +++ b/test/unit/container/test_auditor.py @@ -0,0 +1,28 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Tests + +import unittest +from swift.container import auditor + +class TestReaper(unittest.TestCase): + + def test_placeholder(self): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py new file mode 100644 index 0000000000..b162343d1b --- /dev/null +++ b/test/unit/container/test_server.py @@ -0,0 +1,642 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest +from shutil import rmtree +from StringIO import StringIO +from time import time + +from eventlet import spawn, TimeoutError, listen +from eventlet.timeout import Timeout +import simplejson +from webob import Request + +from swift.container import server as container_server +from swift.common.utils import normalize_timestamp, mkdirs + + +class TestContainerController(unittest.TestCase): + """ Test swift.container_server.ContainerController """ + def setUp(self): + """ Set up for testing swift.object_server.ObjectController """ + path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS') + if not path_to_test_xfs or not os.path.exists(path_to_test_xfs): + raise Exception('PATH_TO_TEST_XFS not set or not pointing to a ' + 'valid directory.\nPlease set PATH_TO_TEST_XFS to ' + 'a directory on an XFS file system for testing.') + self.testdir = os.path.join(path_to_test_xfs, + 'tmp_test_object_server_ObjectController') + mkdirs(self.testdir) + rmtree(self.testdir) + mkdirs(os.path.join(self.testdir, 'sda1')) + mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) + self.controller = container_server.ContainerController( + {'devices': self.testdir, 'mount_check': 'false'}) + + def tearDown(self): + """ Tear down for testing swift.object_server.ObjectController """ + rmtree(self.testdir, ignore_errors=1) + + def test_HEAD(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assertEquals(int(response.headers['x-container-bytes-used']), 0) + self.assertEquals(int(response.headers['x-container-object-count']), 0) + req2 = Request.blank('/sda1/p/a/c/o', environ= + {'HTTP_X_TIMESTAMP': '1', 'HTTP_X_SIZE': 42, + 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x'}) + self.controller.PUT(req2) + response = self.controller.HEAD(req) + self.assertEquals(int(response.headers['x-container-bytes-used']), 42) + self.assertEquals(int(response.headers['x-container-object-count']), 1) + + def test_HEAD_not_found(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 404) + + def test_PUT(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '2'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 202) + + def test_PUT_obj_not_found(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '1', 'X-Size': '0', + 'X-Content-Type': 'text/plain', 'X-ETag': 'e'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 404) + + def test_DELETE_obj_not_found(self): + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': '1'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 404) + + def test_PUT_utf8(self): + snowman = u'\u2603' + container_name = snowman.encode('utf-8') + req = Request.blank('/sda1/p/a/%s'%container_name, environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + def test_PUT_account_update(self): + bindsock = listen(('127.0.0.1', 0)) + def accept(return_code, expected_timestamp): + try: + with Timeout(3): + sock, addr = bindsock.accept() + inc = sock.makefile('rb') + out = sock.makefile('wb') + out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % + return_code) + out.flush() + self.assertEquals(inc.readline(), + 'PUT /sda1/123/a/c HTTP/1.1\r\n') + headers = {} + line = inc.readline() + while line and line != '\r\n': + headers[line.split(':')[0].lower()] = \ + line.split(':')[1].strip() + line = inc.readline() + self.assertEquals(headers['x-put-timestamp'], + expected_timestamp) + except BaseException, err: + return err + return None + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '0000000001.00000', + 'X-Account-Host': '%s:%s' % bindsock.getsockname(), + 'X-Account-Partition': '123', + 'X-Account-Device': 'sda1'}) + event = spawn(accept, 201, '0000000001.00000') + try: + with Timeout(3): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + finally: + err = event.wait() + if err: + raise Exception(err) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': '2'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '0000000003.00000', + 'X-Account-Host': '%s:%s' % bindsock.getsockname(), + 'X-Account-Partition': '123', + 'X-Account-Device': 'sda1'}) + event = spawn(accept, 404, '0000000003.00000') + try: + with Timeout(3): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 404) + finally: + err = event.wait() + if err: + raise Exception(err) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '0000000005.00000', + 'X-Account-Host': '%s:%s' % bindsock.getsockname(), + 'X-Account-Partition': '123', + 'X-Account-Device': 'sda1'}) + event = spawn(accept, 503, '0000000005.00000') + got_exc = False + try: + with Timeout(3): + resp = self.controller.PUT(req) + except BaseException, err: + got_exc = True + finally: + err = event.wait() + if err: + raise Exception(err) + self.assert_(not got_exc) + + def test_DELETE(self): + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '1'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '2'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': '3'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 404) + + def test_DELETE_not_found(self): + # Even if the container wasn't previously heard of, the container + # server will accept the delete and replicate it to where it belongs + # later. + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 404) + + def test_DELETE_object(self): + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '2'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0', + 'HTTP_X_SIZE': 1, 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '3'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 409) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '4'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '5'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': '6'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 404) + + def test_DELETE_account_update(self): + bindsock = listen(('127.0.0.1', 0)) + def accept(return_code, expected_timestamp): + try: + with Timeout(3): + sock, addr = bindsock.accept() + inc = sock.makefile('rb') + out = sock.makefile('wb') + out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % + return_code) + out.flush() + self.assertEquals(inc.readline(), + 'PUT /sda1/123/a/c HTTP/1.1\r\n') + headers = {} + line = inc.readline() + while line and line != '\r\n': + headers[line.split(':')[0].lower()] = \ + line.split(':')[1].strip() + line = inc.readline() + self.assertEquals(headers['x-delete-timestamp'], + expected_timestamp) + except BaseException, err: + return err + return None + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '1'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': '0000000002.00000', + 'X-Account-Host': '%s:%s' % bindsock.getsockname(), + 'X-Account-Partition': '123', + 'X-Account-Device': 'sda1'}) + event = spawn(accept, 204, '0000000002.00000') + try: + with Timeout(3): + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + finally: + err = event.wait() + if err: + raise Exception(err) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '2'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': '0000000003.00000', + 'X-Account-Host': '%s:%s' % bindsock.getsockname(), + 'X-Account-Partition': '123', + 'X-Account-Device': 'sda1'}) + event = spawn(accept, 404, '0000000003.00000') + try: + with Timeout(3): + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 404) + finally: + err = event.wait() + if err: + raise Exception(err) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '4'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': '0000000005.00000', + 'X-Account-Host': '%s:%s' % bindsock.getsockname(), + 'X-Account-Partition': '123', + 'X-Account-Device': 'sda1'}) + event = spawn(accept, 503, '0000000005.00000') + got_exc = False + try: + with Timeout(3): + resp = self.controller.DELETE(req) + except BaseException, err: + got_exc = True + finally: + err = event.wait() + if err: + raise Exception(err) + self.assert_(not got_exc) + + def test_GET_over_limit(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '0'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c?limit=%d' % + (container_server.CONTAINER_LISTING_LIMIT + 1), + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 412) + + def test_GET_format(self): + # make a container + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + # test an empty container + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + # fill the container + for i in range(3): + req = Request.blank('/sda1/p/a/c/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + # test format + json_body = [{"name":"0", + "hash":"x", + "bytes":0, + "content_type":"text/plain", + "last_modified":"1970-01-01T00:00:01"}, + {"name":"1", + "hash":"x", + "bytes":0, + "content_type":"text/plain", + "last_modified":"1970-01-01T00:00:01"}, + {"name":"2", + "hash":"x", + "bytes":0, + "content_type":"text/plain", + "last_modified":"1970-01-01T00:00:01"}] + xml_body = '\n' \ + '' \ + '0x0' \ + 'text/plain' \ + '1970-01-01T00:00:01' \ + '' \ + '1x0' \ + 'text/plain' \ + '1970-01-01T00:00:01' \ + '' \ + '2x0' \ + 'text/plain' \ + '1970-01-01T00:00:01' \ + '' \ + '' + plain_body = '0\n1\n2\n' + req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'application/json') + result = eval(resp.body) + self.assertEquals(result, json_body) + req = Request.blank('/sda1/p/a/c?format=xml', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'application/xml') + result = resp.body + self.assertEquals(result, xml_body) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/json' + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'application/json') + result = eval(resp.body) + self.assertEquals(result, json_body) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.accept = '*/*' + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'text/plain') + result = resp.body + self.assertEquals(result, plain_body) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/*' + resp = self.controller.GET(req) + result = eval(resp.body) + self.assertEquals(result, json_body) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/xml' + resp = self.controller.GET(req) + result = resp.body + self.assertEquals(result, xml_body) + # test conflicting formats + req = Request.blank('/sda1/p/a/c?format=plain', environ={'REQUEST_METHOD': 'GET'}) + req.accept = 'application/json' + resp = self.controller.GET(req) + self.assertEquals(resp.content_type, 'text/plain') + result = resp.body + self.assertEquals(result, plain_body) + + def test_GET_marker(self): + # make a container + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + # fill the container + for i in range(3): + req = Request.blank('/sda1/p/a/c/%s'%i, environ= {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + # test limit with marker + req = Request.blank('/sda1/p/a/c?limit=2&marker=1', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + result = resp.body.split() + self.assertEquals(result, ['2',]) + + def test_weird_content_types(self): + snowman = u'\u2603' + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + for i, ctype in enumerate((snowman.encode('utf-8'), 'text/plain; "utf-8"')): + req = Request.blank('/sda1/p/a/c/%s'%i, environ= {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype, + 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + result = [x['content_type'] for x in simplejson.loads(resp.body)] + self.assertEquals(result, [u'\u2603', 'text/plain; "utf-8"']) + + def test_GET_limit(self): + # make a container + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + # fill the container + for i in range(3): + req = Request.blank('/sda1/p/a/c/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + # test limit + req = Request.blank('/sda1/p/a/c?limit=2', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + result = resp.body.split() + self.assertEquals(result, ['0','1']) + + def test_GET_prefix(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + for i in ('a1', 'b1', 'a2', 'b2', 'a3', 'b3'): + req = Request.blank('/sda1/p/a/c/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c?prefix=a', environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.body.split(), ['a1','a2', 'a3']) + + def test_GET_delimiter(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + for i in ('US-TX-A', 'US-TX-B', 'US-OK-A', 'US-OK-B', 'US-UT-A'): + req = Request.blank('/sda1/p/a/c/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c?prefix=US-&delimiter=-&format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(simplejson.loads(resp.body), + [{"subdir":"US-OK-"},{"subdir":"US-TX-"},{"subdir":"US-UT-"}]) + + def test_GET_delimiter_xml(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + for i in ('US-TX-A', 'US-TX-B', 'US-OK-A', 'US-OK-B', 'US-UT-A'): + req = Request.blank('/sda1/p/a/c/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c?prefix=US-&delimiter=-&format=xml', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.body, '' + '\n' + '') + + def test_GET_path(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + for i in ('US/TX', 'US/TX/B', 'US/OK', 'US/OK/B', 'US/UT/A'): + req = Request.blank('/sda1/p/a/c/%s'%i, environ= + {'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c?path=US&format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(simplejson.loads(resp.body), + [{"name":"US/OK","hash":"x","bytes":0,"content_type":"text/plain", + "last_modified":"1970-01-01T00:00:01"}, + {"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain", + "last_modified":"1970-01-01T00:00:01"}]) + + def test_healthcheck(self): + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + + def start_response(*args): + """ Sends args to outbuf """ + outbuf.writelines(args) + + self.controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/healthcheck', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '200 ') + + def test_through_call(self): + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + def start_response(*args): + outbuf.writelines(args) + self.controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/sda1/p/a/c', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '404 ') + + def test_through_call_invalid_path(self): + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + def start_response(*args): + outbuf.writelines(args) + self.controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/bob', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '400 ') + + def test_params_utf8(self): + self.controller.PUT(Request.blank('/sda1/p/a/c', + headers={'X-Timestamp': normalize_timestamp(1)}, + environ={'REQUEST_METHOD': 'PUT'})) + for param in ('delimiter', 'format', 'limit', 'marker', 'path', + 'prefix'): + req = Request.blank('/sda1/p/a/c?%s=\xce' % param, + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 400) + req = Request.blank('/sda1/p/a/c?%s=\xce\xa9' % param, + environ={'REQUEST_METHOD': 'GET'}) + resp = self.controller.GET(req) + self.assert_(resp.status_int in (204, 412), resp.status_int) + + +if __name__ == '__main__': + unittest.main() + diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py new file mode 100644 index 0000000000..092d383d74 --- /dev/null +++ b/test/unit/container/test_updater.py @@ -0,0 +1,201 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cPickle as pickle +import os +import unittest +from gzip import GzipFile +from shutil import rmtree + +from eventlet import spawn, TimeoutError, listen +from eventlet.timeout import Timeout + +from swift.container import updater as container_updater +from swift.container import server as container_server +from swift.common.db import ContainerBroker +from swift.common.ring import RingData +from swift.common.utils import normalize_timestamp + + +class TestContainerUpdater(unittest.TestCase): + + def setUp(self): + path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS') + if not path_to_test_xfs or not os.path.exists(path_to_test_xfs): + raise Exception('PATH_TO_TEST_XFS not set or not pointing to a ' + 'valid directory.\nPlease set PATH_TO_TEST_XFS to ' + 'a directory on an XFS file system for testing.') + self.testdir = os.path.join(path_to_test_xfs, + 'tmp_test_container_updater') + rmtree(self.testdir, ignore_errors=1) + os.mkdir(self.testdir) + pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'ip': '127.0.0.1', 'port': 12345, 'device': 'sda1', + 'zone': 0}, + {'id': 1, 'ip': '127.0.0.1', 'port': 12345, 'device': 'sda1', + 'zone': 2}], 30), + GzipFile(os.path.join(self.testdir, 'account.ring.gz'), 'wb')) + self.devices_dir = os.path.join(self.testdir, 'devices') + os.mkdir(self.devices_dir) + self.sda1 = os.path.join(self.devices_dir, 'sda1') + os.mkdir(self.sda1) + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def test_creation(self): + cu = container_updater.ContainerUpdater( + {'devices': self.devices_dir, 'mount_check': 'false', + 'swift_dir': self.testdir}, + {'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) + self.assert_(hasattr(cu, 'logger')) + self.assert_(cu.logger is not None) + self.assertEquals(cu.devices, self.devices_dir) + self.assertEquals(cu.interval, 1) + self.assertEquals(cu.concurrency, 2) + self.assertEquals(cu.node_timeout, 5) + self.assert_(cu.get_account_ring() is not None) + + def test_update_once_single_threaded(self): + cu = container_updater.ContainerUpdater( + {'devices': self.devices_dir, 'mount_check': 'false', + 'swift_dir': self.testdir}, + {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + cu.update_once_single_threaded() + containers_dir = os.path.join(self.sda1, container_server.DATADIR) + os.mkdir(containers_dir) + cu.update_once_single_threaded() + self.assert_(os.path.exists(containers_dir)) + subdir = os.path.join(containers_dir, 'subdir') + os.mkdir(subdir) + cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', + container='c') + cb.initialize(normalize_timestamp(1)) + cu.update_once_single_threaded() + info = cb.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + cb.put_object('o', normalize_timestamp(2), 3, 'text/plain', + '68b329da9893e34099c7d8ad5cb9c940') + cu.update_once_single_threaded() + info = cb.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 3) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + def accept(sock, addr, return_code): + try: + with Timeout(3): + inc = sock.makefile('rb') + out = sock.makefile('wb') + out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % + return_code) + out.flush() + self.assertEquals(inc.readline(), + 'PUT /sda1/0/a/c HTTP/1.1\r\n') + headers = {} + line = inc.readline() + while line and line != '\r\n': + headers[line.split(':')[0].lower()] = \ + line.split(':')[1].strip() + line = inc.readline() + self.assert_('x-put-timestamp' in headers) + self.assert_('x-delete-timestamp' in headers) + self.assert_('x-object-count' in headers) + self.assert_('x-bytes-used' in headers) + except BaseException, err: + import traceback + traceback.print_exc() + return err + return None + bindsock = listen(('127.0.0.1', 0)) + def spawn_accepts(): + events = [] + for _ in xrange(2): + sock, addr = bindsock.accept() + events.append(spawn(accept, sock, addr, 201)) + return events + spawned = spawn(spawn_accepts) + for dev in cu.get_account_ring().devs: + if dev is not None: + dev['port'] = bindsock.getsockname()[1] + cu.update_once_single_threaded() + for event in spawned.wait(): + err = event.wait() + if err: + raise err + info = cb.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 3) + self.assertEquals(info['reported_object_count'], 1) + self.assertEquals(info['reported_bytes_used'], 3) + + def test_unicode(self): + cu = container_updater.ContainerUpdater( + {'devices': self.devices_dir, 'mount_check': 'false', + 'swift_dir': self.testdir}, + {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + containers_dir = os.path.join(self.sda1, container_server.DATADIR) + os.mkdir(containers_dir) + subdir = os.path.join(containers_dir, 'subdir') + os.mkdir(subdir) + cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', + container='\xce\xa9') + cb.initialize(normalize_timestamp(1)) + cb.put_object('\xce\xa9', normalize_timestamp(2), 3, 'text/plain', + '68b329da9893e34099c7d8ad5cb9c940') + def accept(sock, addr): + try: + with Timeout(3): + inc = sock.makefile('rb') + out = sock.makefile('wb') + out.write('HTTP/1.1 201 OK\r\nContent-Length: 0\r\n\r\n') + out.flush() + inc.read() + except BaseException, err: + import traceback + traceback.print_exc() + return err + return None + bindsock = listen(('127.0.0.1', 0)) + def spawn_accepts(): + events = [] + for _ in xrange(2): + with Timeout(3): + sock, addr = bindsock.accept() + events.append(spawn(accept, sock, addr)) + return events + spawned = spawn(spawn_accepts) + for dev in cu.get_account_ring().devs: + if dev is not None: + dev['port'] = bindsock.getsockname()[1] + cu.update_once_single_threaded() + for event in spawned.wait(): + err = event.wait() + if err: + raise err + info = cb.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 3) + self.assertEquals(info['reported_object_count'], 1) + self.assertEquals(info['reported_bytes_used'], 3) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/obj/__init__.py b/test/unit/obj/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py new file mode 100644 index 0000000000..cf8a2bc37c --- /dev/null +++ b/test/unit/obj/test_auditor.py @@ -0,0 +1,28 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Tests + +import unittest +from swift.obj import auditor + +class TestAuditor(unittest.TestCase): + + def test_placeholder(self): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py new file mode 100644 index 0000000000..be04f9ee01 --- /dev/null +++ b/test/unit/obj/test_replicator.py @@ -0,0 +1,194 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement + +import unittest +import os +from gzip import GzipFile +from shutil import rmtree +import cPickle as pickle +import logging +import fcntl +from contextlib import contextmanager + +from eventlet.green import subprocess + +from swift.obj import replicator as object_replicator +from swift.common import ring + +def _ips(): + return ['127.0.0.0',] +object_replicator.whataremyips = _ips + +class NullHandler(logging.Handler): + def emit(self, record): + pass +null_logger = logging.getLogger("testing") +null_logger.addHandler(NullHandler()) + +class MockProcess(object): + ret_code = None + ret_log = None + + class Stream(object): + def read(self): + return MockProcess.ret_log.next() + + def __init__(self, *args, **kwargs): + self.stdout = self.Stream() + + def wait(self): + return self.ret_code.next() + +@contextmanager +def _mock_process(ret): + orig_process = subprocess.Popen + MockProcess.ret_code = (i[0] for i in ret) + MockProcess.ret_log = (i[1] for i in ret) + object_replicator.subprocess.Popen = MockProcess + yield + object_replicator.subprocess.Popen = orig_process + +def _create_test_ring(path): + testgz = os.path.join(path, 'object.ring.gz') + intended_replica2part2dev_id = [ + [0, 1, 2, 3, 4, 5, 6], + [1, 2, 3, 0, 5, 6, 4], + [2, 3, 0, 1, 6, 4, 5], + ] + intended_devs = [ + {'id': 0, 'device': 'sda', 'zone': 0, 'ip': '127.0.0.0', 'port': 6000}, + {'id': 1, 'device': 'sda', 'zone': 1, 'ip': '127.0.0.1', 'port': 6000}, + {'id': 2, 'device': 'sda', 'zone': 2, 'ip': '127.0.0.2', 'port': 6000}, + {'id': 3, 'device': 'sda', 'zone': 4, 'ip': '127.0.0.3', 'port': 6000}, + {'id': 4, 'device': 'sda', 'zone': 5, 'ip': '127.0.0.4', 'port': 6000}, + {'id': 5, 'device': 'sda', 'zone': 6, 'ip': '127.0.0.5', 'port': 6000}, + {'id': 6, 'device': 'sda', 'zone': 7, 'ip': '127.0.0.6', 'port': 6000}, + ] + intended_part_shift = 30 + intended_reload_time = 15 + pickle.dump(ring.RingData(intended_replica2part2dev_id, + intended_devs, intended_part_shift), + GzipFile(testgz, 'wb')) + return ring.Ring(testgz, reload_time=intended_reload_time) + + +class TestObjectReplicator(unittest.TestCase): + + def setUp(self): + # Setup a test ring (stolen from common/test_ring.py) + self.testdir = os.path.join('/dev/shm', 'test_replicator') + self.devices = os.path.join(self.testdir, 'node') + rmtree(self.testdir, ignore_errors=1) + os.mkdir(self.testdir) + os.mkdir(self.devices) + os.mkdir(os.path.join(self.devices, 'sda')) + self.objects = os.path.join(self.devices, 'sda', 'objects') + os.mkdir(self.objects) + for part in ['0','1','2', '3']: + os.mkdir(os.path.join(self.objects, part)) + self.ring = _create_test_ring(self.testdir) + self.conf = dict( + swift_dir=self.testdir, devices=self.devices, mount_check='false', + timeout='300', stats_interval='1') + self.replicator = object_replicator.ObjectReplicator( + self.conf, null_logger) + +# def test_check_ring(self): +# self.replicator.collect_jobs('sda', 0, self.ring) +# self.assertTrue(self.replicator.check_ring()) +# orig_check = self.replicator.next_check +# self.replicator.next_check = orig_check - 30 +# self.assertTrue(self.replicator.check_ring()) +# self.replicator.next_check = orig_check +# orig_ring_time = self.replicator.object_ring._mtime +# self.replicator.object_ring._mtime = orig_ring_time - 30 +# self.assertTrue(self.replicator.check_ring()) +# self.replicator.next_check = orig_check - 30 +# self.assertFalse(self.replicator.check_ring()) +# +# def test_collect_jobs(self): +# self.replicator.collect_jobs('sda', 0, self.ring) +# self.assertTrue('1' in self.replicator.parts_to_delete) +# self.assertEquals( +# [node['id'] for node in self.replicator.partitions['0']['nodes']], +# [1,2]) +# self.assertEquals( +# [node['id'] for node in self.replicator.partitions['1']['nodes']], +# [1,2,3]) +# self.assertEquals( +# [node['id'] for node in self.replicator.partitions['2']['nodes']], +# [2,3]) +# self.assertEquals( +# [node['id'] for node in self.replicator.partitions['3']['nodes']], +# [3,1]) +# for part in ['0', '1', '2', '3']: +# self.assertEquals(self.replicator.partitions[part]['device'], 'sda') +# self.assertEquals(self.replicator.partitions[part]['path'], +# self.objects) +# +# def test_delete_partition(self): +# self.replicator.collect_jobs('sda', 0, self.ring) +# part_path = os.path.join(self.objects, '1') +# self.assertTrue(os.access(part_path, os.F_OK)) +# self.replicator.delete_partition('1') +# self.assertFalse(os.access(part_path, os.F_OK)) +# +# def test_rsync(self): +# self.replicator.collect_jobs('sda', 0, self.ring) +# with _mock_process([(0,''), (0,''), (0,'')]): +# self.replicator.rsync('0') +# +# def test_rsync_delete_no(self): +# self.replicator.collect_jobs('sda', 0, self.ring) +# with _mock_process([(-1, "stuff in log"), (-1, "stuff in log"), +# (0,''), (0,'')]): +# self.replicator.rsync('1') +# self.assertEquals(self.replicator.parts_to_delete['1'], +# [False, True, True]) +# +# def test_rsync_delete_yes(self): +# self.replicator.collect_jobs('sda', 0, self.ring) +# with _mock_process([(0,''), (0,''), (0,'')]): +# self.replicator.rsync('1') +# self.assertEquals(self.replicator.parts_to_delete['1'], +# [True, True, True]) +# +# def test_rsync_delete_yes_with_failure(self): +# self.replicator.collect_jobs('sda', 0, self.ring) +# with _mock_process([(-1, "stuff in log"), (0, ''), (0,''), (0,'')]): +# self.replicator.rsync('1') +# self.assertEquals(self.replicator.parts_to_delete['1'], +# [True, True, True]) +# +# def test_rsync_failed_drive(self): +# self.replicator.collect_jobs('sda', 0, self.ring) +# with _mock_process([(12,'There was an error in file IO'), +# (0,''), (0,''), (0,'')]): +# self.replicator.rsync('1') +# self.assertEquals(self.replicator.parts_to_delete['1'], +# [True, True, True]) + + def test_run(self): + with _mock_process([(0,'')]*100): + self.replicator.run() + + def test_run_withlog(self): + with _mock_process([(0,"stuff in log")]*100): + self.replicator.run() + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py new file mode 100644 index 0000000000..37233e24e8 --- /dev/null +++ b/test/unit/obj/test_server.py @@ -0,0 +1,991 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for swift.object_server """ + +import cPickle as pickle +import os +import sys +import unittest +from shutil import rmtree +from StringIO import StringIO +from time import gmtime, sleep, strftime, time + +from eventlet import sleep, spawn, wsgi, listen +from webob import Request +from xattr import getxattr, setxattr + +from test.unit import connect_tcp, readuntil2crlfs +from swift.obj import server as object_server +from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ + NullLogger, storage_directory + + +class TestObjectController(unittest.TestCase): + """ Test swift.object_server.ObjectController """ + + def setUp(self): + """ Set up for testing swift.object_server.ObjectController """ + path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS') + if not path_to_test_xfs or not os.path.exists(path_to_test_xfs): + raise Exception('PATH_TO_TEST_XFS not set or not pointing to a ' + 'valid directory.\nPlease set PATH_TO_TEST_XFS to ' + 'a directory on an XFS file system for testing.') + self.testdir = os.path.join(path_to_test_xfs, + 'tmp_test_object_server_ObjectController') + mkdirs(self.testdir) + rmtree(self.testdir) + mkdirs(os.path.join(self.testdir, 'sda1')) + mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) + conf = {'devices': self.testdir, 'mount_check': 'false'} + self.object_controller = object_server.ObjectController(conf) + self.object_controller.chunks_per_sync = 1 + + def tearDown(self): + """ Tear down for testing swift.object_server.ObjectController """ + rmtree(self.testdir) + + def test_POST_update_meta(self): + """ Test swift.object_server.ObjectController.POST """ + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test', + 'X-Object-Meta-1': 'One', + 'X-Object-Meta-Two': 'Two'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Object-Meta-3': 'Three', + 'X-Object-Meta-4': 'Four', + 'Content-Type': 'application/x-test'}) + resp = self.object_controller.POST(req) + self.assertEquals(resp.status_int, 202) + + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.GET(req) + self.assert_("X-Object-Meta-1" not in resp.headers and \ + "X-Object-Meta-3" in resp.headers) + self.assertEquals(resp.headers['Content-Type'], 'application/x-test') + + def test_POST_not_exist(self): + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/fail', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Object-Meta-1': 'One', + 'X-Object-Meta-2': 'Two', + 'Content-Type': 'text/plain'}) + resp = self.object_controller.POST(req) + self.assertEquals(resp.status_int, 404) + + def test_POST_invalid_path(self): + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Object-Meta-1': 'One', + 'X-Object-Meta-2': 'Two', + 'Content-Type': 'text/plain'}) + resp = self.object_controller.POST(req) + self.assertEquals(resp.status_int, 400) + + def test_POST_container_connection(self): + def mock_http_connect(response, with_exc=False): + class FakeConn(object): + def __init__(self, status, with_exc): + self.status = status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = '1234' + self.with_exc = with_exc + def getresponse(self): + if self.with_exc: + raise Exception('test') + return self + def read(self, amt=None): + return '' + return lambda *args, **kwargs: FakeConn(response, with_exc) + old_http_connect = object_server.http_connect + try: + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', + 'Content-Length': '0'}) + resp = self.object_controller.PUT(req) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Container-Host': '1.2.3.4:0', + 'X-Container-Partition': '3', + 'X-Container-Device': 'sda1', + 'X-Container-Timestamp': '1', + 'Content-Type': 'application/new1'}) + object_server.http_connect = mock_http_connect(202) + resp = self.object_controller.POST(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Container-Host': '1.2.3.4:0', + 'X-Container-Partition': '3', + 'X-Container-Device': 'sda1', + 'X-Container-Timestamp': '1', + 'Content-Type': 'application/new1'}) + object_server.http_connect = mock_http_connect(202, with_exc=True) + resp = self.object_controller.POST(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Container-Host': '1.2.3.4:0', + 'X-Container-Partition': '3', + 'X-Container-Device': 'sda1', + 'X-Container-Timestamp': '1', + 'Content-Type': 'application/new2'}) + object_server.http_connect = mock_http_connect(500) + resp = self.object_controller.POST(req) + self.assertEquals(resp.status_int, 202) + finally: + object_server.http_connect = old_http_connect + + def test_PUT_invalid_path(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}) + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_PUT_no_timestamp(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', + 'CONTENT_LENGTH': '0'}) + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_PUT_no_content_type(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Length': '6'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_PUT_invalid_content_type(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Length': '6', + 'Content-Type': '\xff\xff'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 400) + self.assert_('Content-Type' in resp.body) + + def test_PUT_no_content_length(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Type': 'application/octet-stream'}) + req.body = 'VERIFY' + del req.headers['Content-Length'] + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 411) + + def test_PUT_common(self): + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Length': '6', + 'Content-Type': 'application/octet-stream'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + objfile = os.path.join(self.testdir, 'sda1', + storage_directory(object_server.DATADIR, 'p', + hash_path('a', 'c', 'o')), + timestamp + '.data') + self.assert_(os.path.isfile(objfile)) + self.assertEquals(open(objfile).read(), 'VERIFY') + self.assertEquals(pickle.loads(getxattr(objfile, object_server.METADATA_KEY)), + {'X-Timestamp': timestamp, + 'Content-Length': '6', + 'ETag': '0b4c12d7e0a73840c1c4f148fda3b037', + 'Content-Type': 'application/octet-stream', + 'name': '/a/c/o'}) + + def test_PUT_overwrite(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Length': '6', + 'Content-Type': 'application/octet-stream'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + sleep(.00001) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'Content-Encoding': 'gzip'}) + req.body = 'VERIFY TWO' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + objfile = os.path.join(self.testdir, 'sda1', + storage_directory(object_server.DATADIR, 'p', + hash_path('a', 'c', 'o')), + timestamp + '.data') + self.assert_(os.path.isfile(objfile)) + self.assertEquals(open(objfile).read(), 'VERIFY TWO') + self.assertEquals(pickle.loads(getxattr(objfile, object_server.METADATA_KEY)), + {'X-Timestamp': timestamp, + 'Content-Length': '10', + 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', + 'Content-Type': 'text/plain', + 'name': '/a/c/o', + 'Content-Encoding': 'gzip'}) + + def test_PUT_no_etag(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Type': 'text/plain'}) + req.body = 'test' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + def test_PUT_invalid_etag(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Type': 'text/plain', + 'ETag': 'invalid'}) + req.body = 'test' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 422) + + def test_PUT_user_metadata(self): + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'text/plain', + 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', + 'X-Object-Meta-1': 'One', + 'X-Object-Meta-Two': 'Two'}) + req.body = 'VERIFY THREE' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + objfile = os.path.join(self.testdir, 'sda1', + storage_directory(object_server.DATADIR, 'p', + hash_path('a', 'c', 'o')), + timestamp + '.data') + self.assert_(os.path.isfile(objfile)) + self.assertEquals(open(objfile).read(), 'VERIFY THREE') + self.assertEquals(pickle.loads(getxattr(objfile, object_server.METADATA_KEY)), + {'X-Timestamp': timestamp, + 'Content-Length': '12', + 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', + 'Content-Type': 'text/plain', + 'name': '/a/c/o', + 'X-Object-Meta-1': 'One', + 'X-Object-Meta-Two': 'Two'}) + + def test_PUT_container_connection(self): + def mock_http_connect(response, with_exc=False): + class FakeConn(object): + def __init__(self, status, with_exc): + self.status = status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = '1234' + self.with_exc = with_exc + def getresponse(self): + if self.with_exc: + raise Exception('test') + return self + def read(self, amt=None): + return '' + return lambda *args, **kwargs: FakeConn(response, with_exc) + old_http_connect = object_server.http_connect + try: + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Container-Host': '1.2.3.4:0', + 'X-Container-Partition': '3', + 'X-Container-Device': 'sda1', + 'X-Container-Timestamp': '1', + 'Content-Type': 'application/new1', + 'Content-Length': '0'}) + object_server.http_connect = mock_http_connect(201) + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Container-Host': '1.2.3.4:0', + 'X-Container-Partition': '3', + 'X-Container-Device': 'sda1', + 'X-Container-Timestamp': '1', + 'Content-Type': 'application/new1', + 'Content-Length': '0'}) + object_server.http_connect = mock_http_connect(500) + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Container-Host': '1.2.3.4:0', + 'X-Container-Partition': '3', + 'X-Container-Device': 'sda1', + 'X-Container-Timestamp': '1', + 'Content-Type': 'application/new1', + 'Content-Length': '0'}) + object_server.http_connect = mock_http_connect(500, with_exc=True) + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + finally: + object_server.http_connect = old_http_connect + + def test_HEAD(self): + """ Test swift.object_server.ObjectController.HEAD """ + req = Request.blank('/sda1/p/a/c') + resp = self.object_controller.HEAD(req) + self.assertEquals(resp.status_int, 400) + + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.HEAD(req) + self.assertEquals(resp.status_int, 404) + + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test', + 'X-Object-Meta-1': 'One', + 'X-Object-Meta-Two': 'Two'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.HEAD(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_length, 6) + self.assertEquals(resp.content_type, 'application/x-test') + self.assertEquals(resp.headers['content-type'], 'application/x-test') + self.assertEquals(resp.headers['last-modified'], + strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp)))) + self.assertEquals(resp.headers['etag'], + '"0b4c12d7e0a73840c1c4f148fda3b037"') + self.assertEquals(resp.headers['x-object-meta-1'], 'One') + self.assertEquals(resp.headers['x-object-meta-two'], 'Two') + + objfile = os.path.join(self.testdir, 'sda1', + storage_directory(object_server.DATADIR, 'p', + hash_path('a', 'c', 'o')), + timestamp + '.data') + os.unlink(objfile) + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.HEAD(req) + self.assertEquals(resp.status_int, 404) + + sleep(.00001) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': timestamp, + 'Content-Type': 'application/octet-stream', + 'Content-length': '6'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + sleep(.00001) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': timestamp}) + resp = self.object_controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.HEAD(req) + self.assertEquals(resp.status_int, 404) + + def test_GET(self): + """ Test swift.object_server.ObjectController.GET """ + req = Request.blank('/sda1/p/a/c') + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 400) + + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 404) + + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test', + 'X-Object-Meta-1': 'One', + 'X-Object-Meta-Two': 'Two'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body, 'VERIFY') + self.assertEquals(resp.content_length, 6) + self.assertEquals(resp.content_type, 'application/x-test') + self.assertEquals(resp.headers['content-length'], '6') + self.assertEquals(resp.headers['content-type'], 'application/x-test') + self.assertEquals(resp.headers['last-modified'], + strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp)))) + self.assertEquals(resp.headers['etag'], + '"0b4c12d7e0a73840c1c4f148fda3b037"') + self.assertEquals(resp.headers['x-object-meta-1'], 'One') + self.assertEquals(resp.headers['x-object-meta-two'], 'Two') + + req = Request.blank('/sda1/p/a/c/o') + req.range = 'bytes=1-3' + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 206) + self.assertEquals(resp.body, 'ERI') + self.assertEquals(resp.headers['content-length'], '3') + + req = Request.blank('/sda1/p/a/c/o') + req.range = 'bytes=1-' + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 206) + self.assertEquals(resp.body, 'ERIFY') + self.assertEquals(resp.headers['content-length'], '5') + + req = Request.blank('/sda1/p/a/c/o') + req.range = 'bytes=-2' + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 206) + self.assertEquals(resp.body, 'FY') + self.assertEquals(resp.headers['content-length'], '2') + + objfile = os.path.join(self.testdir, 'sda1', + storage_directory(object_server.DATADIR, 'p', + hash_path('a', 'c', 'o')), + timestamp + '.data') + os.unlink(objfile) + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 404) + + sleep(.00001) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': timestamp, + 'Content-Type': 'application:octet-stream', + 'Content-Length': '6'}) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + sleep(.00001) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': timestamp}) + resp = self.object_controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 404) + + def test_GET_if_match(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': normalize_timestamp(time()), + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4'}) + req.body = 'test' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + etag = resp.etag + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.etag, etag) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Match': '*'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.etag, etag) + + req = Request.blank('/sda1/p/a/c/o2', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Match': '*'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 412) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Match': '"%s"' % etag}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.etag, etag) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Match': + '"11111111111111111111111111111111"'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 412) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Match': + '"11111111111111111111111111111111", "%s"' % etag}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Match': + '"11111111111111111111111111111111", ' + '"22222222222222222222222222222222"'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 412) + + def test_GET_if_none_match(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': normalize_timestamp(time()), + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4'}) + req.body = 'test' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + etag = resp.etag + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.etag, etag) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-None-Match': '*'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 304) + self.assertEquals(resp.etag, etag) + + req = Request.blank('/sda1/p/a/c/o2', + environ={'REQUEST_METHOD': 'GET'}, + headers={'If-None-Match': '*'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 404) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-None-Match': '"%s"' % etag}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 304) + self.assertEquals(resp.etag, etag) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-None-Match': + '"11111111111111111111111111111111"'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.etag, etag) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-None-Match': + '"11111111111111111111111111111111", ' + '"%s"' % etag}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 304) + self.assertEquals(resp.etag, etag) + + def test_GET_if_modified_since(self): + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': timestamp, + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4'}) + req.body = 'test' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + + since = strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp))) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Modified-Since': since}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 304) + + since = \ + strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) - 1)) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Modified-Since': since}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + + since = \ + strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 1)) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Modified-Since': since}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 304) + + def test_GET_if_unmodified_since(self): + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': timestamp, + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4'}) + req.body = 'test' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + + since = strftime('%a, %d %b %Y %H:%M:%S GMT', + gmtime(float(timestamp) + 1)) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Unmodified-Since': since}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + + since = \ + strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) - 9)) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Unmodified-Since': since}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 412) + + since = \ + strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 9)) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Unmodified-Since': since}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + + def test_DELETE(self): + """ Test swift.object_server.ObjectController.DELETE """ + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}) + resp = self.object_controller.DELETE(req) + self.assertEquals(resp.status_int, 400) + + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}) + resp = self.object_controller.DELETE(req) + self.assertEquals(resp.status_int, 400) + # self.assertRaises(KeyError, self.object_controller.DELETE, req) + + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': timestamp}) + resp = self.object_controller.DELETE(req) + self.assertEquals(resp.status_int, 404) + + sleep(.00001) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': timestamp, + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4', + }) + req.body = 'test' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + timestamp = normalize_timestamp(float(timestamp) - 1) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': timestamp}) + resp = self.object_controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + objfile = os.path.join(self.testdir, 'sda1', + storage_directory(object_server.DATADIR, 'p', + hash_path('a', 'c', 'o')), + timestamp + '.ts') + self.assert_(os.path.isfile(objfile)) + + sleep(.00001) + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': timestamp}) + resp = self.object_controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + objfile = os.path.join(self.testdir, 'sda1', + storage_directory(object_server.DATADIR, 'p', + hash_path('a', 'c', 'o')), + timestamp + '.ts') + self.assert_(os.path.isfile(objfile)) + + def test_healthcheck(self): + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + + def start_response(*args): + """ Sends args to outbuf """ + outbuf.writelines(args) + + self.object_controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/healthcheck', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '200 ') + + def test_call(self): + """ Test swift.object_server.ObjectController.__call__ """ + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + + def start_response(*args): + """ Sends args to outbuf """ + outbuf.writelines(args) + + self.object_controller.__call__({'REQUEST_METHOD': 'PUT', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/sda1/p/a/c/o', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '400 ') + + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + self.object_controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/sda1/p/a/c/o', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '404 ') + + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + self.object_controller.__call__({'REQUEST_METHOD': 'INVALID', + 'SCRIPT_NAME': '', + 'PATH_INFO': '/sda1/p/a/c/o', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '405 ') + + def test_chunked_put(self): + listener = listen(('localhost', 0)) + port = listener.getsockname()[1] + killer = spawn(wsgi.server, listener, self.object_controller, + NullLogger()) + sock = connect_tcp(('localhost', port)) + fd = sock.makefile() + fd.write('PUT /sda1/p/a/c/o HTTP/1.1\r\nHost: localhost\r\n' + 'Content-Type: text/plain\r\n' + 'Connection: close\r\nX-Timestamp: 1.0\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + '2\r\noh\r\n4\r\n hai\r\n0\r\n\r\n') + fd.flush() + readuntil2crlfs(fd) + sock = connect_tcp(('localhost', port)) + fd = sock.makefile() + fd.write('GET /sda1/p/a/c/o HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\n\r\n') + fd.flush() + readuntil2crlfs(fd) + response = fd.read() + self.assertEquals(response, 'oh hai') + killer.kill() + + def test_max_object_name_length(self): + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/' + ('1' * 1024), + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = 'DATA' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/' + ('2' * 1025), + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Length': '4', + 'Content-Type': 'application/octet-stream'}) + req.body = 'DATA' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_disk_file_app_iter_corners(self): + df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') + mkdirs(df.datadir) + f = open(os.path.join(df.datadir, + normalize_timestamp(time()) + '.data'), 'wb') + f.write('1234567890') + setxattr(f.fileno(), object_server.METADATA_KEY, + pickle.dumps({}, object_server.PICKLE_PROTOCOL)) + f.close() + df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', + keep_data_fp=True) + it = df.app_iter_range(0, None) + sio = StringIO() + for chunk in it: + sio.write(chunk) + self.assertEquals(sio.getvalue(), '1234567890') + + df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', + keep_data_fp=True) + it = df.app_iter_range(5, None) + sio = StringIO() + for chunk in it: + sio.write(chunk) + self.assertEquals(sio.getvalue(), '67890') + + def test_disk_file_mkstemp_creates_dir(self): + tmpdir = os.path.join(self.testdir, 'sda1', 'tmp') + os.rmdir(tmpdir) + with object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o').mkstemp(): + self.assert_(os.path.exists(tmpdir)) + + def test_max_upload_time(self): + class SlowBody(): + def __init__(self): + self.sent = 0 + def read(self, size=-1): + if self.sent < 4: + sleep(0.1) + self.sent += 1 + return ' ' + return '' + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Length': '4', 'Content-Type': 'text/plain'}) + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.object_controller.max_upload_time = 0.1 + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Length': '4', 'Content-Type': 'text/plain'}) + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 408) + + def test_short_body(self): + class ShortBody(): + def __init__(self): + self.sent = False + def read(self, size=-1): + if not self.sent: + self.sent = True + return ' ' + return '' + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': ShortBody()}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Length': '4', 'Content-Type': 'text/plain'}) + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 499) + + def test_bad_sinces(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Length': '4', 'Content-Type': 'text/plain'}, + body=' ') + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Unmodified-Since': 'Not a valid date'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Modified-Since': 'Not a valid date'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Unmodified-Since': 'Sat, 29 Oct 1000 19:43:31 GMT'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 412) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, + headers={'If-Modified-Since': 'Sat, 29 Oct 1000 19:43:31 GMT'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 412) + + def test_content_encoding(self): + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(time()), + 'Content-Length': '4', 'Content-Type': 'text/plain', + 'Content-Encoding': 'gzip'}, + body=' ') + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.headers['content-encoding'], 'gzip') + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.object_controller.HEAD(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.headers['content-encoding'], 'gzip') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py new file mode 100644 index 0000000000..e5de8df4fd --- /dev/null +++ b/test/unit/obj/test_updater.py @@ -0,0 +1,131 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cPickle as pickle +import json +import os +import unittest +from gzip import GzipFile +from shutil import rmtree +from time import time + +from eventlet import spawn, TimeoutError, listen +from eventlet.timeout import Timeout + +from swift.obj import updater as object_updater, server as object_server +from swift.common.ring import RingData +from swift.common import utils +from swift.common.utils import hash_path, normalize_timestamp, mkdirs + + +class TestObjectUpdater(unittest.TestCase): + + def setUp(self): + utils.HASH_PATH_SUFFIX = 'endcap' + self.testdir = os.path.join(os.path.dirname(__file__), + 'object_updater') + rmtree(self.testdir, ignore_errors=1) + os.mkdir(self.testdir) + pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', + 'zone': 0}, + {'id': 1, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', + 'zone': 2}], 30), + GzipFile(os.path.join(self.testdir, 'container.ring.gz'), 'wb')) + self.devices_dir = os.path.join(self.testdir, 'devices') + os.mkdir(self.devices_dir) + self.sda1 = os.path.join(self.devices_dir, 'sda1') + os.mkdir(self.sda1) + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def test_creation(self): + cu = object_updater.ObjectUpdater( + {'devices': self.devices_dir, 'mount_check': 'false', + 'swift_dir': self.testdir}, + {'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) + self.assert_(hasattr(cu, 'logger')) + self.assert_(cu.logger is not None) + self.assertEquals(cu.devices, self.devices_dir) + self.assertEquals(cu.interval, 1) + self.assertEquals(cu.concurrency, 2) + self.assertEquals(cu.node_timeout, 5) + self.assert_(cu.get_container_ring() is not None) + + def test_update_once_single_threaded(self): + cu = object_updater.ObjectUpdater( + {'devices': self.devices_dir, 'mount_check': 'false', + 'swift_dir': self.testdir}, + {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + cu.update_once_single_threaded() + async_dir = os.path.join(self.sda1, object_server.ASYNCDIR) + os.mkdir(async_dir) + cu.update_once_single_threaded() + self.assert_(os.path.exists(async_dir)) + + odd_dir = os.path.join(async_dir, 'not really supposed to be here') + os.mkdir(odd_dir) + cu.update_once_single_threaded() + self.assert_(os.path.exists(async_dir)) + self.assert_(not os.path.exists(odd_dir)) + + ohash = hash_path('a', 'c', 'o') + odir = os.path.join(async_dir, ohash[-3:]) + mkdirs(odir) + op_path = os.path.join(odir, + '%s-%s' % (ohash, normalize_timestamp(time()))) + pickle.dump({'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', + 'headers': {'X-Container-Timestamp': normalize_timestamp(0)}}, + open(op_path, 'wb')) + cu.update_once_single_threaded() + self.assert_(os.path.exists(op_path)) + + bindsock = listen(('127.0.0.1', 0)) + def accept(return_code): + try: + with Timeout(3): + sock, addr = bindsock.accept() + inc = sock.makefile('rb') + out = sock.makefile('wb') + out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % + return_code) + out.flush() + self.assertEquals(inc.readline(), + 'PUT /sda1/0/a/c/o HTTP/1.1\r\n') + headers = {} + line = inc.readline() + while line and line != '\r\n': + headers[line.split(':')[0].lower()] = \ + line.split(':')[1].strip() + line = inc.readline() + self.assert_('x-container-timestamp' in headers) + except BaseException, err: + return err + return None + events = [spawn(accept, 201), spawn(accept, 201)] + for dev in cu.get_container_ring().devs: + if dev is not None: + dev['port'] = bindsock.getsockname()[1] + cu.update_once_single_threaded() + for event in events: + err = event.wait() + if err: + raise err + self.assert_(not os.path.exists(op_path)) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/proxy/__init__.py b/test/unit/proxy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py new file mode 100644 index 0000000000..c7180e0a41 --- /dev/null +++ b/test/unit/proxy/test_server.py @@ -0,0 +1,1719 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import cPickle as pickle +import logging +import os +import sys +import unittest +from ConfigParser import ConfigParser +from contextlib import contextmanager +from cStringIO import StringIO +from gzip import GzipFile +from httplib import HTTPException +from shutil import rmtree +from time import time +from urllib import unquote, quote + +import eventlet +from eventlet import sleep, spawn, TimeoutError, util, wsgi, listen +from eventlet.timeout import Timeout +import simplejson +from webob import Request + +from test.unit import connect_tcp, readuntil2crlfs +from swift.proxy import server as proxy_server +from swift.account import server as account_server +from swift.container import server as container_server +from swift.obj import server as object_server +from swift.common import ring +from swift.common.constraints import MAX_META_NAME_LENGTH, \ + MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, MAX_FILE_SIZE +from swift.common.utils import mkdirs, normalize_timestamp, NullLogger + + +# mocks +logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) + +def fake_http_connect(*code_iter, **kwargs): + class FakeConn(object): + def __init__(self, status, etag=None): + self.status = status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = '1234' + self.sent = 0 + self.received = 0 + self.etag = etag + def getresponse(self): + if 'raise_exc' in kwargs: + raise Exception('test') + return self + def getexpect(self): + return FakeConn(100) + def getheaders(self): + headers = {'content-length': 0, + 'content-type': 'x-application/test', + 'x-timestamp': '1', + 'x-object-meta-test': 'testing', + 'etag': + self.etag or '"68b329da9893e34099c7d8ad5cb9c940"', + 'x-works': 'yes', + } + try: + if container_ts_iter.next() is False: + headers['x-container-timestamp'] = '1' + except StopIteration: + pass + if 'slow' in kwargs: + headers['content-length'] = '4' + return headers + def read(self, amt=None): + if 'slow' in kwargs: + if self.sent < 4: + self.sent += 1 + sleep(0.1) + return ' ' + return '' + def send(self, amt=None): + if 'slow' in kwargs: + if self.received < 4: + self.received += 1 + sleep(0.1) + def getheader(self, name, default=None): + return self.getheaders().get(name.lower(), default) + etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) + x = kwargs.get('missing_container', [False] * len(code_iter)) + if not isinstance(x, (tuple, list)): + x = [x] * len(code_iter) + container_ts_iter = iter(x) + code_iter = iter(code_iter) + def connect(*args, **ckwargs): + if 'give_content_type' in kwargs: + if len(args) >= 7 and 'content_type' in args[6]: + kwargs['give_content_type'](args[6]['content-type']) + else: + kwargs['give_content_type']('') + status = code_iter.next() + etag = etag_iter.next() + if status == -1: + raise HTTPException() + return FakeConn(status, etag) + return connect + + +class FakeRing(object): + + def __init__(self): + # 9 total nodes (6 more past the initial 3) is the cap, no matter if + # this is set higher. + self.max_more_nodes = 0 + self.devs = {} + self.replica_count = 3 + + def get_nodes(self, account, container=None, obj=None): + devs = [] + for x in xrange(3): + devs.append(self.devs.get(x)) + if devs[x] is None: + self.devs[x] = devs[x] = \ + {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'} + return 1, devs + + def get_more_nodes(self, nodes): + # 9 is the true cap + for x in xrange(3, min(3 + self.max_more_nodes, 9)): + yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'} + + +class FakeMemcache(object): + def __init__(self): + self.store = {} + + def get(self, key): + return self.store.get(key) + + def set(self, key, value, timeout=0): + self.store[key] = value + return True + + def incr(self, key, timeout=0): + self.store[key] = self.store.setdefault(key, 0) + 1 + return self.store[key] + + @contextmanager + def soft_lock(self, key, timeout=0, retries=5): + yield True + + def delete(self, key): + try: + del self.store[key] + except: + pass + return True + + +class FakeMemcacheReturnsNone(FakeMemcache): + + def get(self, key): + # Returns None as the timestamp of the container; assumes we're only + # using the FakeMemcache for container existence checks. + return None + +class NullLoggingHandler(logging.Handler): + + def emit(self, record): + pass + +@contextmanager +def save_globals(): + orig_http_connect = getattr(proxy_server, 'http_connect', None) + try: + yield True + finally: + proxy_server.http_connect = orig_http_connect + +# tests + +class TestObjectController(unittest.TestCase): + + def setUp(self): + self.app = proxy_server.Application(None, FakeMemcache()) + self.app.object_ring = FakeRing() + self.app.container_ring = FakeRing() + self.app.account_ring = FakeRing() + + def assert_status_map(self, method, statuses, expected, raise_exc=False): + with save_globals(): + kwargs = {} + if raise_exc: + kwargs['raise_exc'] = raise_exc + proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', headers={'Content-Length': '0', + 'Content-Type': 'text/plain'}) + req.account = 'a' + res = method(req) + self.assertEquals(res.status_int, expected) + proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', headers={'Content-Length': '0', + 'Content-Type': 'text/plain'}) + req.account = 'a' + res = method(req) + self.assertEquals(res.status_int, expected) + + def test_PUT_auto_content_type(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + def test_content_type(filename, expected): + proxy_server.http_connect = fake_http_connect(201, 201, 201, + give_content_type=lambda content_type: + self.assertEquals(content_type, expected.next())) + req = Request.blank('/a/c/%s' % filename, {}) + req.account = 'a' + res = controller.PUT(req) + test_content_type('test.jpg', + iter(['', '', '', 'image/jpeg', 'image/jpeg', 'image/jpeg'])) + test_content_type('test.html', + iter(['', '', '', 'text/html', 'text/html', 'text/html'])) + test_content_type('test.css', + iter(['', '', '', 'text/css', 'text/css', 'text/css'])) + + def test_PUT(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + def test_status_map(statuses, expected): + proxy_server.http_connect = fake_http_connect(*statuses) + req = Request.blank('/a/c/o.jpg', {}) + req.content_length = 0 + req.account = 'a' + self.app.memcache.store = {} + res = controller.PUT(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + test_status_map((200, 200, 201, 201, 201), 201) + test_status_map((200, 200, 201, 201, 500), 201) + test_status_map((200, 200, 204, 404, 404), 404) + test_status_map((200, 200, 204, 500, 404), 503) + + def test_PUT_connect_exceptions(self): + def mock_http_connect(*code_iter, **kwargs): + class FakeConn(object): + def __init__(self, status): + self.status = status + self.reason = 'Fake' + def getresponse(self): return self + def read(self, amt=None): return '' + def getheader(self, name): return '' + def getexpect(self): return FakeConn(100) + code_iter = iter(code_iter) + def connect(*args, **ckwargs): + status = code_iter.next() + if status == -1: + raise HTTPException() + return FakeConn(status) + return connect + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + def test_status_map(statuses, expected): + proxy_server.http_connect = mock_http_connect(*statuses) + self.app.memcache.store = {} + req = Request.blank('/a/c/o.jpg', {}) + req.content_length = 0 + req.account = 'a' + res = controller.PUT(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + test_status_map((200, 200, 201, 201, -1), 201) + test_status_map((200, 200, 201, -1, -1), 503) + test_status_map((200, 200, 503, 503, -1), 503) + + def test_PUT_send_exceptions(self): + def mock_http_connect(*code_iter, **kwargs): + class FakeConn(object): + def __init__(self, status): + self.status = status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = 1024 + def getresponse(self): return self + def read(self, amt=None): return '' + def send(self, amt=None): + if self.status == -1: + raise HTTPException() + def getheader(self, name): return '' + def getexpect(self): return FakeConn(100) + code_iter = iter(code_iter) + def connect(*args, **ckwargs): + return FakeConn(code_iter.next()) + return connect + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + def test_status_map(statuses, expected): + self.app.memcache.store = {} + proxy_server.http_connect = mock_http_connect(*statuses) + req = Request.blank('/a/c/o.jpg', {}) + req.account = 'a' + req.body_file = StringIO('some data') + res = controller.PUT(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + test_status_map((200, 200, 201, 201, -1), 201) + test_status_map((200, 200, 201, -1, -1), 503) + test_status_map((200, 200, 503, 503, -1), 503) + + def test_PUT_max_size(self): + with save_globals(): + proxy_server.http_connect = fake_http_connect(201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Length': str(MAX_FILE_SIZE + 1), + 'Content-Type': 'foo/bar'}) + req.account = 'a' + res = controller.PUT(req) + self.assertEquals(res.status_int, 413) + + def test_PUT_getresponse_exceptions(self): + def mock_http_connect(*code_iter, **kwargs): + class FakeConn(object): + def __init__(self, status): + self.status = status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = 1024 + def getresponse(self): + if self.status == -1: + raise HTTPException() + return self + def read(self, amt=None): return '' + def send(self, amt=None): pass + def getheader(self, name): return '' + def getexpect(self): return FakeConn(100) + code_iter = iter(code_iter) + def connect(*args, **ckwargs): + return FakeConn(code_iter.next()) + return connect + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + def test_status_map(statuses, expected): + self.app.memcache.store = {} + proxy_server.http_connect = mock_http_connect(*statuses) + req = Request.blank('/a/c/o.jpg', {}) + req.content_length = 0 + req.account = 'a' + res = controller.PUT(req) + expected = str(expected) + self.assertEquals(res.status[:len(str(expected))], + str(expected)) + test_status_map((200, 200, 201, 201, -1), 201) + test_status_map((200, 200, 201, -1, -1), 503) + test_status_map((200, 200, 503, 503, -1), 503) + + def test_POST(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + def test_status_map(statuses, expected): + proxy_server.http_connect = fake_http_connect(*statuses) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar'}) + req.account = 'a' + res = controller.POST(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + test_status_map((200, 200, 202, 202, 202), 202) + test_status_map((200, 200, 202, 202, 500), 202) + test_status_map((200, 200, 202, 500, 500), 503) + test_status_map((200, 200, 202, 404, 500), 503) + test_status_map((200, 200, 202, 404, 404), 404) + test_status_map((200, 200, 404, 500, 500), 503) + test_status_map((200, 200, 404, 404, 404), 404) + + def test_DELETE(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + def test_status_map(statuses, expected): + proxy_server.http_connect = fake_http_connect(*statuses) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', {}) + req.account = 'a' + res = controller.DELETE(req) + self.assertEquals(res.status[:len(str(expected))], + str(expected)) + test_status_map((200, 200, 204, 204, 204), 204) + test_status_map((200, 200, 204, 204, 500), 204) + test_status_map((200, 200, 204, 404, 404), 404) + test_status_map((200, 200, 204, 500, 404), 503) + test_status_map((200, 200, 404, 404, 404), 404) + test_status_map((200, 200, 404, 404, 500), 404) + + def test_HEAD(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + def test_status_map(statuses, expected): + proxy_server.http_connect = fake_http_connect(*statuses) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', {}) + req.account = 'a' + res = controller.HEAD(req) + self.assertEquals(res.status[:len(str(expected))], + str(expected)) + if expected < 400: + self.assert_('x-works' in res.headers) + self.assertEquals(res.headers['x-works'], 'yes') + test_status_map((200, 404, 404), 200) + test_status_map((200, 500, 404), 200) + test_status_map((304, 500, 404), 304) + test_status_map((404, 404, 404), 404) + test_status_map((404, 404, 500), 404) + test_status_map((500, 500, 500), 503) + + def test_POST_meta_val_len(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 202, 202, 202) + # acct cont obj obj obj + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar', + 'X-Object-Meta-Foo': 'x'*256}) + req.account = 'a' + res = controller.POST(req) + self.assertEquals(res.status_int, 202) + proxy_server.http_connect = fake_http_connect(202, 202, 202) + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar', + 'X-Object-Meta-Foo': 'x'*257}) + req.account = 'a' + res = controller.POST(req) + self.assertEquals(res.status_int, 400) + + def test_POST_meta_key_len(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 202, 202, 202) + # acct cont obj obj obj + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar', + ('X-Object-Meta-'+'x'*128): 'x'}) + req.account = 'a' + res = controller.POST(req) + self.assertEquals(res.status_int, 202) + proxy_server.http_connect = fake_http_connect(202, 202, 202) + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar', + ('X-Object-Meta-'+'x'*129): 'x'}) + req.account = 'a' + res = controller.POST(req) + self.assertEquals(res.status_int, 400) + + def test_POST_meta_count(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + headers = dict((('X-Object-Meta-'+str(i), 'a') for i in xrange(91))) + headers.update({'Content-Type': 'foo/bar'}) + proxy_server.http_connect = fake_http_connect(202, 202, 202) + req = Request.blank('/a/c/o', {}, headers=headers) + req.account = 'a' + res = controller.POST(req) + self.assertEquals(res.status_int, 400) + + def test_POST_meta_size(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + headers = dict((('X-Object-Meta-'+str(i), 'a'*256) for i in xrange(1000))) + headers.update({'Content-Type': 'foo/bar'}) + proxy_server.http_connect = fake_http_connect(202, 202, 202) + req = Request.blank('/a/c/o', {}, headers=headers) + req.account = 'a' + res = controller.POST(req) + self.assertEquals(res.status_int, 400) + + def test_client_timeout(self): + with save_globals(): + self.app.account_ring.get_nodes('account') + for dev in self.app.account_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.container_ring.get_nodes('account') + for dev in self.app.container_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.object_ring.get_nodes('account') + for dev in self.app.object_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + class SlowBody(): + def __init__(self): + self.sent = 0 + def read(self, size=-1): + if self.sent < 4: + sleep(0.1) + self.sent += 1 + return ' ' + return '' + req = Request.blank('/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, + headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) + req.account = 'account' + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + # acct cont obj obj obj + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.app.client_timeout = 0.1 + req = Request.blank('/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, + headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) + req.account = 'account' + proxy_server.http_connect = \ + fake_http_connect(201, 201, 201) + # obj obj obj + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 408) + + def test_client_disconnect(self): + with save_globals(): + self.app.account_ring.get_nodes('account') + for dev in self.app.account_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.container_ring.get_nodes('account') + for dev in self.app.container_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.object_ring.get_nodes('account') + for dev in self.app.object_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + class SlowBody(): + def __init__(self): + self.sent = 0 + def read(self, size=-1): + raise Exception('Disconnected') + req = Request.blank('/a/c/o', + environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, + headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) + req.account = 'account' + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + # acct cont obj obj obj + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 499) + + def test_node_read_timeout(self): + with save_globals(): + self.app.account_ring.get_nodes('account') + for dev in self.app.account_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.container_ring.get_nodes('account') + for dev in self.app.container_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.object_ring.get_nodes('account') + for dev in self.app.object_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + req.account = 'account' + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, slow=True) + req.sent_size = 0 + resp = controller.GET(req) + got_exc = False + try: + resp.body + except proxy_server.ChunkReadTimeout: + got_exc = True + self.assert_(not got_exc) + self.app.node_timeout=0.1 + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, slow=True) + resp = controller.GET(req) + got_exc = False + try: + resp.body + except proxy_server.ChunkReadTimeout: + got_exc = True + self.assert_(got_exc) + + def test_node_write_timeout(self): + with save_globals(): + self.app.account_ring.get_nodes('account') + for dev in self.app.account_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.container_ring.get_nodes('account') + for dev in self.app.container_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + self.app.object_ring.get_nodes('account') + for dev in self.app.object_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 + req = Request.blank('/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, + body=' ') + req.account = 'account' + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201, slow=True) + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.app.node_timeout=0.1 + proxy_server.http_connect = \ + fake_http_connect(201, 201, 201, slow=True) + req = Request.blank('/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, + body=' ') + req.account = 'account' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 503) + + def test_iter_nodes(self): + with save_globals(): + try: + self.app.object_ring.max_more_nodes = 2 + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + partition, nodes = self.app.object_ring.get_nodes('account', + 'container', 'object') + collected_nodes = [] + for node in controller.iter_nodes(partition, nodes, + self.app.object_ring): + collected_nodes.append(node) + self.assertEquals(len(collected_nodes), 5) + + self.app.object_ring.max_more_nodes = 20 + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + partition, nodes = self.app.object_ring.get_nodes('account', + 'container', 'object') + collected_nodes = [] + for node in controller.iter_nodes(partition, nodes, + self.app.object_ring): + collected_nodes.append(node) + self.assertEquals(len(collected_nodes), 9) + finally: + self.app.object_ring.max_more_nodes = 0 + + def test_best_response_sets_etag(self): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, + 'Object') + self.assertEquals(resp.etag, None) + resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, + 'Object', etag='68b329da9893e34099c7d8ad5cb9c940') + self.assertEquals(resp.etag, '68b329da9893e34099c7d8ad5cb9c940') + + def test_proxy_passes_content_type(self): + with save_globals(): + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + req.account = 'account' + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = fake_http_connect(200, 200, 200) + resp = controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'x-application/test') + proxy_server.http_connect = fake_http_connect(200, 200, 200) + resp = controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_length, 0) + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, slow=True) + resp = controller.GET(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_length, 4) + + def test_proxy_passes_content_length_on_head(self): + with save_globals(): + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) + req.account = 'account' + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = fake_http_connect(200, 200, 200) + resp = controller.HEAD(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_length, 0) + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, slow=True) + resp = controller.HEAD(req) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_length, 4) + + def test_error_limiting(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + self.assert_status_map(controller.HEAD, (503, 200, 200), 200) + self.assertEquals(controller.app.object_ring.devs[0]['errors'], 2) + self.assert_('last_error' in controller.app.object_ring.devs[0]) + for _ in xrange(self.app.error_suppression_limit): + self.assert_status_map(controller.HEAD, (503, 503, 503), 503) + self.assertEquals(controller.app.object_ring.devs[0]['errors'], + self.app.error_suppression_limit + 1) + self.assert_status_map(controller.HEAD, (200, 200, 200), 503) + self.assert_('last_error' in controller.app.object_ring.devs[0]) + self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503) + self.assert_status_map(controller.POST, (200, 202, 202, 202), 503) + self.assert_status_map(controller.DELETE, (200, 204, 204, 204), 503) + self.app.error_suppression_interval = -300 + self.assert_status_map(controller.HEAD, (200, 200, 200), 200) + self.assertRaises(BaseException, + self.assert_status_map, controller.DELETE, + (200, 204, 204, 204), 503, raise_exc=True) + + def test_acc_or_con_missing_returns_404(self): + with save_globals(): + self.app.memcache = FakeMemcacheReturnsNone() + for dev in self.app.account_ring.devs.values(): + del dev['errors'] + del dev['last_error'] + for dev in self.app.container_ring.devs.values(): + del dev['errors'] + del dev['last_error'] + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 200) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) + req.account = 'a' + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 200) + + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404) + # acct acct acct + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(503, 404, 404) + # acct acct acct + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(503, 503, 404) + # acct acct acct + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(503, 503, 503) + # acct acct acct + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(200, 200, 204, 204, 204) + # acct cont obj obj obj + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 204) + + proxy_server.http_connect = \ + fake_http_connect(200, 404, 404, 404) + # acct cont cont cont + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(200, 503, 503, 503) + # acct cont cont cont + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 404) + + for dev in self.app.account_ring.devs.values(): + dev['errors'] = self.app.error_suppression_limit + 1 + dev['last_error'] = time() + proxy_server.http_connect = \ + fake_http_connect(200) + # acct [isn't actually called since everything + # is error limited] + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 404) + + for dev in self.app.account_ring.devs.values(): + dev['errors'] = 0 + for dev in self.app.container_ring.devs.values(): + dev['errors'] = self.app.error_suppression_limit + 1 + dev['last_error'] = time() + proxy_server.http_connect = \ + fake_http_connect(200, 200) + # acct cont [isn't actually called since + # everything is error limited] + resp = getattr(controller, 'DELETE')(req) + self.assertEquals(resp.status_int, 404) + + def test_PUT_POST_requires_container_exist(self): + with save_globals(): + self.app.memcache = FakeMemcacheReturnsNone() + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404, 200, 200, 200) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404, 200, 200, 200) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'text/plain'}) + req.account = 'a' + resp = controller.POST(req) + self.assertEquals(resp.status_int, 404) + + def test_bad_metadata(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + # acct cont obj obj obj + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0'}) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + proxy_server.http_connect = fake_http_connect(201, 201, 201) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Object-Meta-' + ('a' * + MAX_META_NAME_LENGTH) : 'v'}) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + proxy_server.http_connect = fake_http_connect(201, 201, 201) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Object-Meta-' + ('a' * + (MAX_META_NAME_LENGTH + 1)) : 'v'}) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + proxy_server.http_connect = fake_http_connect(201, 201, 201) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Object-Meta-Too-Long': 'a' * + MAX_META_VALUE_LENGTH}) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + proxy_server.http_connect = fake_http_connect(201, 201, 201) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Object-Meta-Too-Long': 'a' * + (MAX_META_VALUE_LENGTH + 1)}) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + proxy_server.http_connect = fake_http_connect(201, 201, 201) + headers = {'Content-Length': '0'} + for x in xrange(MAX_META_COUNT): + headers['X-Object-Meta-%d' % x] = 'v' + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers=headers) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + proxy_server.http_connect = fake_http_connect(201, 201, 201) + headers = {'Content-Length': '0'} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Object-Meta-%d' % x] = 'v' + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers=headers) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + proxy_server.http_connect = fake_http_connect(201, 201, 201) + headers = {'Content-Length': '0'} + header_value = 'a' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - \ + MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Object-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Object-Meta-a'] = \ + 'a' * (MAX_META_OVERALL_SIZE - size - 1) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers=headers) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + proxy_server.http_connect = fake_http_connect(201, 201, 201) + headers['X-Object-Meta-a'] = \ + 'a' * (MAX_META_OVERALL_SIZE - size) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers=headers) + req.account = 'a' + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_copy_from(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0'}) + req.account = 'a' + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + # acct cont obj obj obj + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o'}) + req.account = 'a' + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acct cont objc obj obj obj + self.app.memcache.store = {} + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + req.account = 'a' + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) + # acct cont acct cont objc obj obj obj + self.app.memcache.store = {} + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + req.account = 'a' + proxy_server.http_connect = \ + fake_http_connect(200, 200, 503, 503, 503) + # acct cont objc objc objc + self.app.memcache.store = {} + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 503) + + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + req.account = 'a' + proxy_server.http_connect = \ + fake_http_connect(200, 200, 404, 404, 404) + # acct cont objc objc objc + self.app.memcache.store = {} + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 404) + + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + req.account = 'a' + proxy_server.http_connect = \ + fake_http_connect(200, 200, 404, 404, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + self.app.memcache.store = {} + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + req.account = 'a' + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 201, 201, 201) + # acct cont objc obj obj obj + self.app.memcache.store = {} + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers.get('x-object-meta-test'), 'testing') + self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') + + def test_chunked_put_and_a_bit_more(self): + # Since we're starting up a lot here, we're going to test more than + # just chunked puts; we're also going to test parts of + # proxy_server.Application we couldn't get to easily otherwise. + path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS') + if not path_to_test_xfs or not os.path.exists(path_to_test_xfs): + raise Exception('PATH_TO_TEST_XFS not set or not pointing to ' + 'a valid directory.\nPlease set PATH_TO_TEST_XFS to a ' + 'directory on an XFS file system for testing.') + testdir = \ + os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked') + mkdirs(testdir) + rmtree(testdir) + mkdirs(os.path.join(testdir, 'sda1')) + mkdirs(os.path.join(testdir, 'sda1', 'tmp')) + mkdirs(os.path.join(testdir, 'sdb1')) + mkdirs(os.path.join(testdir, 'sdb1', 'tmp')) + try: + conf = {'devices': testdir, 'swift_dir': testdir, + 'mount_check': 'false'} + prolis = listen(('localhost', 0)) + acc1lis = listen(('localhost', 0)) + acc2lis = listen(('localhost', 0)) + con1lis = listen(('localhost', 0)) + con2lis = listen(('localhost', 0)) + obj1lis = listen(('localhost', 0)) + obj2lis = listen(('localhost', 0)) + pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': acc1lis.getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': acc2lis.getsockname()[1]}], 30), + GzipFile(os.path.join(testdir, 'account.ring.gz'), 'wb')) + pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': con1lis.getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': con2lis.getsockname()[1]}], 30), + GzipFile(os.path.join(testdir, 'container.ring.gz'), 'wb')) + pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': obj1lis.getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': obj2lis.getsockname()[1]}], 30), + GzipFile(os.path.join(testdir, 'object.ring.gz'), 'wb')) + prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone()) + acc1srv = account_server.AccountController(conf) + acc2srv = account_server.AccountController(conf) + con1srv = container_server.ContainerController(conf) + con2srv = container_server.ContainerController(conf) + obj1srv = object_server.ObjectController(conf) + obj2srv = object_server.ObjectController(conf) + nl = NullLogger() + prospa = spawn(wsgi.server, prolis, prosrv, nl) + acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl) + acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl) + con1spa = spawn(wsgi.server, con1lis, con1srv, nl) + con2spa = spawn(wsgi.server, con2lis, con2srv, nl) + obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) + obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) + try: + # healthcheck test + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /healthcheck HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nContent-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + body = fd.read() + self.assertEquals(body, 'OK') + # Check bad version + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v0 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nContent-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 412' + self.assertEquals(headers[:len(exp)], exp) + # Check bad path + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET invalid HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nContent-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 404' + self.assertEquals(headers[:len(exp)], exp) + # Check bad method + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('LICK /healthcheck HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nContent-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 405' + self.assertEquals(headers[:len(exp)], exp) + # Check blacklist + prosrv.rate_limit_blacklist = ['a'] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nContent-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 497' + self.assertEquals(headers[:len(exp)], exp) + prosrv.rate_limit_blacklist = [] + # Check invalid utf-8 + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a%80 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 412' + self.assertEquals(headers[:len(exp)], exp) + # Check bad path, no controller + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1 HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 412' + self.assertEquals(headers[:len(exp)], exp) + # Check rate limiting + orig_rate_limit = prosrv.rate_limit + prosrv.rate_limit = 0 + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 498' + self.assertEquals(headers[:len(exp)], exp) + prosrv.rate_limit = orig_rate_limit + orig_rate_limit = prosrv.account_rate_limit + prosrv.account_rate_limit = 0 + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 498' + self.assertEquals(headers[:len(exp)], exp) + prosrv.account_rate_limit = orig_rate_limit + # Check bad method + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('LICK /v1/a HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 405' + self.assertEquals(headers[:len(exp)], exp) + # Check unhandled exception + orig_rate_limit = prosrv.rate_limit + del prosrv.rate_limit + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 503' + self.assertEquals(headers[:len(exp)], exp) + prosrv.rate_limit = orig_rate_limit + # Okay, back to chunked put testing; Create account + ts = normalize_timestamp(time()) + partition, nodes = prosrv.account_ring.get_nodes('a') + for node in nodes: + conn = proxy_server.http_connect(node['ip'], node['port'], + node['device'], partition, 'PUT', '/a', + {'X-Timestamp': ts, 'X-CF-Trans-Id': 'test'}) + resp = conn.getresponse() + self.assertEquals(resp.status, 201) + # Head account, just a double check and really is here to test + # the part Application.log_request that 'enforces' a + # content_length on the response. + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 204' + self.assertEquals(headers[:len(exp)], exp) + self.assert_('\r\nContent-Length: 0\r\n' in headers) + # Create container + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + # GET account with a query string to test that + # Application.log_request logs the query string. Also, throws + # in a test for logging x-forwarded-for (first entry only). + class Logger(object): + def info(self, msg): + self.msg = msg + orig_logger = prosrv.logger + prosrv.logger = Logger() + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a?format=json HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\nX-Forwarded-For: host1, host2\r\n' + '\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + self.assert_('/v1/a%3Fformat%3Djson' in prosrv.logger.msg, + prosrv.logger.msg) + exp = 'host1' + self.assertEquals(prosrv.logger.msg[:len(exp)], exp) + prosrv.logger = orig_logger + # Turn on header logging. + class Logger(object): + def info(self, msg): + self.msg = msg + orig_logger = prosrv.logger + prosrv.logger = Logger() + prosrv.log_headers = True + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n' + 'Content-Length: 0\r\nGoofy-Header: True\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + self.assert_('Goofy-Header%3A%20True' in prosrv.logger.msg, + prosrv.logger.msg) + prosrv.log_headers = False + prosrv.logger = orig_logger + # Test UTF-8 Unicode all the way through the system + ustr = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xba \xe1\xbc\xb0\xce' \ + '\xbf\xe1\xbd\xbb\xce\x87 \xcf\x84\xe1\xbd\xb0 \xcf' \ + '\x80\xe1\xbd\xb1\xce\xbd\xcf\x84\xca\xbc \xe1\xbc' \ + '\x82\xce\xbd \xe1\xbc\x90\xce\xbe\xe1\xbd\xb5\xce' \ + '\xba\xce\xbf\xce\xb9 \xcf\x83\xce\xb1\xcf\x86\xe1' \ + '\xbf\x86.Test' + ustr_short = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xbatest' + # Create ustr container + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n' % quote(ustr)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + # List account with ustr container (test plain) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + containers = fd.read().split('\n') + self.assert_(ustr in containers) + # List account with ustr container (test json) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a?format=json HTTP/1.1\r\n' + 'Host: localhost\r\nConnection: close\r\n' + 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + listing = simplejson.loads(fd.read()) + self.assertEquals(listing[1]['name'], ustr.decode('utf8')) + # List account with ustr container (test xml) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a?format=xml HTTP/1.1\r\n' + 'Host: localhost\r\nConnection: close\r\n' + 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + self.assert_('%s' % ustr in fd.read()) + # Create ustr object with ustr metadata in ustr container + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'X-Object-Meta-%s: %s\r\nContent-Length: 0\r\n\r\n' % + (quote(ustr), quote(ustr), quote(ustr_short), + quote(ustr))) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + # List ustr container with ustr object (test plain) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n' % quote(ustr)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + objects = fd.read().split('\n') + self.assert_(ustr in objects) + # List ustr container with ustr object (test json) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s?format=json HTTP/1.1\r\n' + 'Host: localhost\r\nConnection: close\r\n' + 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' % + quote(ustr)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + listing = simplejson.loads(fd.read()) + self.assertEquals(listing[0]['name'], ustr.decode('utf8')) + # List ustr container with ustr object (test xml) + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s?format=xml HTTP/1.1\r\n' + 'Host: localhost\r\nConnection: close\r\n' + 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' % + quote(ustr)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + self.assert_('%s' % ustr in fd.read()) + # Retrieve ustr object with ustr metadata + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n' % + (quote(ustr), quote(ustr))) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + self.assert_('\r\nX-Object-Meta-%s: %s\r\n' % + (quote(ustr_short).lower(), quote(ustr)) in headers) + # Do chunked object put + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + # Also happens to assert that x-storage-token is taken as a + # replacement for x-auth-token. + fd.write('PUT /v1/a/c/o/chunky HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + '2\r\noh\r\n4\r\n hai\r\nf\r\n123456789abcdef\r\n' + '0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) + # Ensure we get what we put + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/c/o/chunky HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEquals(headers[:len(exp)], exp) + body = fd.read() + self.assertEquals(body, 'oh hai123456789abcdef') + finally: + prospa.kill() + acc1spa.kill() + acc2spa.kill() + con1spa.kill() + con2spa.kill() + obj1spa.kill() + obj2spa.kill() + finally: + rmtree(testdir) + + def test_mismatched_etags(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0'}) + req.account = 'a' + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201, + etags=[None, + '68b329da9893e34099c7d8ad5cb9c940', + '68b329da9893e34099c7d8ad5cb9c940', + '68b329da9893e34099c7d8ad5cb9c941']) + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 422) + + +class TestContainerController(unittest.TestCase): + "Test swift.proxy_server.ContainerController" + + def setUp(self): + self.app = proxy_server.Application(None, FakeMemcache()) + self.app.object_ring = FakeRing() + self.app.container_ring = FakeRing() + self.app.account_ring = FakeRing() + + def assert_status_map(self, method, statuses, expected, raise_exc=False, missing_container=False): + with save_globals(): + kwargs = {} + if raise_exc: + kwargs['raise_exc'] = raise_exc + kwargs['missing_container'] = missing_container + proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/a/c', headers={'Content-Length': '0', + 'Content-Type': 'text/plain'}) + req.account = 'a' + res = method(req) + self.assertEquals(res.status_int, expected) + proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/a/c/', headers={'Content-Length': '0', + 'Content-Type': 'text/plain'}) + req.account = 'a' + res = method(req) + self.assertEquals(res.status_int, expected) + + def test_HEAD(self): + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + def test_status_map(statuses, expected, **kwargs): + proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/a/c', {}) + req.account = 'a' + res = controller.HEAD(req) + self.assertEquals(res.status[:len(str(expected))], + str(expected)) + if expected < 400: + self.assert_('x-works' in res.headers) + self.assertEquals(res.headers['x-works'], 'yes') + test_status_map((200, 200, 404, 404), 200) + test_status_map((200, 200, 500, 404), 200) + test_status_map((200, 304, 500, 404), 304) + test_status_map((200, 404, 404, 404), 404) + test_status_map((200, 404, 404, 500), 404) + test_status_map((200, 500, 500, 500), 503) + + def test_PUT(self): + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + def test_status_map(statuses, expected, **kwargs): + proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/a/c', {}) + req.content_length = 0 + req.account = 'a' + res = controller.PUT(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + test_status_map((200, 201, 201, 201), 201, missing_container=True) + test_status_map((200, 201, 201, 500), 201, missing_container=True) + test_status_map((200, 204, 404, 404), 404, missing_container=True) + test_status_map((200, 204, 500, 404), 503, missing_container=True) + + def test_PUT_max_container_name_length(self): + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'account', + '1'*256) + self.assert_status_map(controller.PUT, (200, 200, 200, 201, 201, 201), 201, missing_container=True) + controller = proxy_server.ContainerController(self.app, 'account', + '2'*257) + self.assert_status_map(controller.PUT, (201, 201, 201), 400, missing_container=True) + + def test_PUT_connect_exceptions(self): + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + self.assert_status_map(controller.PUT, (200, 201, 201, -1), 201, missing_container=True) + self.assert_status_map(controller.PUT, (200, 201, -1, -1), 503, missing_container=True) + self.assert_status_map(controller.PUT, (200, 503, 503, -1), 503, missing_container=True) + + def test_acc_missing_returns_404(self): + for meth in ('DELETE', 'PUT'): + with save_globals(): + self.app.memcache = FakeMemcacheReturnsNone() + for dev in self.app.account_ring.devs.values(): + del dev['errors'] + del dev['last_error'] + controller = proxy_server.ContainerController(self.app, + 'account', 'container') + if meth == 'PUT': + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 200, missing_container=True) + else: + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200) + self.app.memcache.store = {} + req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth}) + req.account = 'a' + resp = getattr(controller, meth)(req) + self.assertEquals(resp.status_int, 200) + + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404, 200, 200, 200) + resp = getattr(controller, meth)(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(503, 404, 404) + resp = getattr(controller, meth)(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(503, 404, raise_exc=True) + resp = getattr(controller, meth)(req) + self.assertEquals(resp.status_int, 404) + + for dev in self.app.account_ring.devs.values(): + dev['errors'] = self.app.error_suppression_limit + 1 + dev['last_error'] = time() + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 200) + resp = getattr(controller, meth)(req) + self.assertEquals(resp.status_int, 404) + + def test_put_locking(self): + class MockMemcache(FakeMemcache): + def __init__(self, allow_lock=None): + self.allow_lock = allow_lock + super(MockMemcache, self).__init__() + @contextmanager + def soft_lock(self, key, timeout=0, retries=5): + if self.allow_lock: + yield True + else: + raise MemcacheLockError() + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + self.app.memcache = MockMemcache(allow_lock=True) + proxy_server.http_connect = fake_http_connect(200, 200, 200, 201, 201, 201, missing_container=True) + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}) + req.account = 'a' + res = controller.PUT(req) + self.assertEquals(res.status_int, 201) + + def test_error_limiting(self): + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200, missing_container=False) + self.assertEquals( + controller.app.container_ring.devs[0]['errors'], 2) + self.assert_('last_error' in controller.app.container_ring.devs[0]) + for _ in xrange(self.app.error_suppression_limit): + self.assert_status_map(controller.HEAD, (200, 503, 503, 503), 503) + self.assertEquals(controller.app.container_ring.devs[0]['errors'], + self.app.error_suppression_limit + 1) + self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 503) + self.assert_('last_error' in controller.app.container_ring.devs[0]) + self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503, missing_container=True) + self.assert_status_map(controller.DELETE, (200, 204, 204, 204), 503) + self.app.error_suppression_interval = -300 + self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 200) + self.assert_status_map(controller.DELETE, (200, 204, 204, 204), 404, + raise_exc=True) + + def test_DELETE(self): + with save_globals(): + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + self.assert_status_map(controller.DELETE, (200, 204, 204, 204), 204) + self.assert_status_map(controller.DELETE, (200, 204, 204, 503), 503) + self.assert_status_map(controller.DELETE, (200, 204, 503, 503), 503) + self.assert_status_map(controller.DELETE, (200, 204, 404, 404), 404) + self.assert_status_map(controller.DELETE, (200, 404, 404, 404), 404) + self.assert_status_map(controller.DELETE, (200, 204, 503, 404), 503) + + self.app.memcache = FakeMemcacheReturnsNone() + # 200: Account check, 404x3: Container check + self.assert_status_map(controller.DELETE, (200, 404, 404, 404), 404) + + +class TestAccountController(unittest.TestCase): + + def setUp(self): + self.app = proxy_server.Application(None, FakeMemcache()) + self.app.object_ring = FakeRing() + self.app.container_ring = FakeRing() + self.app.account_ring = FakeRing() + + def assert_status_map(self, method, statuses, expected): + with save_globals(): + proxy_server.http_connect = fake_http_connect(*statuses) + req = Request.blank('/a', {}) + req.account = 'a' + res = method(req) + self.assertEquals(res.status_int, expected) + proxy_server.http_connect = fake_http_connect(*statuses) + req = Request.blank('/a/', {}) + req.account = 'a' + res = method(req) + self.assertEquals(res.status_int, expected) + + def test_GET(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.assert_status_map(controller.GET, (200, 200, 200), 200) + self.assert_status_map(controller.GET, (200, 200, 503), 200) + self.assert_status_map(controller.GET, (200, 503, 503), 200) + self.assert_status_map(controller.GET, (204, 204, 204), 204) + self.assert_status_map(controller.GET, (204, 204, 503), 204) + self.assert_status_map(controller.GET, (204, 503, 503), 204) + self.assert_status_map(controller.GET, (204, 204, 200), 204) + self.assert_status_map(controller.GET, (204, 200, 200), 204) + self.assert_status_map(controller.GET, (404, 404, 404), 404) + self.assert_status_map(controller.GET, (404, 404, 200), 200) + self.assert_status_map(controller.GET, (404, 200, 200), 200) + self.assert_status_map(controller.GET, (404, 404, 503), 404) + self.assert_status_map(controller.GET, (404, 503, 503), 503) + self.assert_status_map(controller.GET, (404, 204, 503), 204) + + self.app.memcache = FakeMemcacheReturnsNone() + self.assert_status_map(controller.GET, (404, 404, 404), 404) + + def test_HEAD(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.assert_status_map(controller.HEAD, (200, 200, 200), 200) + self.assert_status_map(controller.HEAD, (200, 200, 503), 200) + self.assert_status_map(controller.HEAD, (200, 503, 503), 200) + self.assert_status_map(controller.HEAD, (204, 204, 204), 204) + self.assert_status_map(controller.HEAD, (204, 204, 503), 204) + self.assert_status_map(controller.HEAD, (204, 503, 503), 204) + self.assert_status_map(controller.HEAD, (204, 204, 200), 204) + self.assert_status_map(controller.HEAD, (204, 200, 200), 204) + self.assert_status_map(controller.HEAD, (404, 404, 404), 404) + self.assert_status_map(controller.HEAD, (404, 404, 200), 200) + self.assert_status_map(controller.HEAD, (404, 200, 200), 200) + self.assert_status_map(controller.HEAD, (404, 404, 503), 404) + self.assert_status_map(controller.HEAD, (404, 503, 503), 503) + self.assert_status_map(controller.HEAD, (404, 204, 503), 204) + + def test_connection_refused(self): + self.app.account_ring.get_nodes('account') + for dev in self.app.account_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = 1 ## can't connect on this port + controller = proxy_server.AccountController(self.app, 'account') + req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) + req.account = 'account' + resp = controller.HEAD(req) + self.assertEquals(resp.status_int, 503) + + def test_other_socket_error(self): + self.app.account_ring.get_nodes('account') + for dev in self.app.account_ring.devs.values(): + dev['ip'] = '127.0.0.1' + dev['port'] = -1 ## invalid port number + controller = proxy_server.AccountController(self.app, 'account') + req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) + req.account = 'account' + resp = controller.HEAD(req) + self.assertEquals(resp.status_int, 503) + + +if __name__ == '__main__': + unittest.main()