From 4ce861c8ad570ad65d73bee61c73a9ae70fcb64f Mon Sep 17 00:00:00 2001 From: MORITA Kazutaka Date: Wed, 25 Jun 2014 08:57:16 +0900 Subject: [PATCH] etree: add support for xml validation This patch adds support for XML validation with RelaxNG. The original S3 schema is available at http://doc.s3.amazonaws.com/2006-03-01/AmazonS3.xsd, but I write a schema for swift3 from scratch. It is because: - The original schema does not support all of the latest S3 API. Even if we use it, we have to update and maintain it in either way. - The original schema is written with XML Schema, but the language is not enough to describe some of the latest S3 API. For example, the Multi-Object Delete operation sends an XML document, which can interleave a Quiet element with Object elements. XML Schema cannot define such kinds of XML documents. This patch includes both RelaxNG (rng) and RelaxNG compact syntax files (rnc) for validation. What I wrote are only rnc files, and rng files are automatically generated from rnc with trang, the most well-known schema converter. If possible, I'd like to use only rnc schemas since rng files are complicated and not so human-readable. However, lxml doesn't support RelaxNG compact syntax currently, unfortunately. After lxml supports compact syntax, let's remove all of the rng files. This patch also fixes some XML errors detected by RelaxNG validator, and some helper methods to convert strings between camel-case name and snake-case name. Change-Id: I4513345a6b981efc5d5f5e8cf528aba84ac1bdad --- doc/rnc/access_control_policy.rnc | 7 ++ doc/rnc/bucket_logging_status.rnc | 10 ++ doc/rnc/common.rnc | 26 +++++ doc/rnc/complete_multipart_upload.rnc | 7 ++ doc/rnc/complete_multipart_upload_result.rnc | 7 ++ doc/rnc/copy_object_result.rnc | 5 + doc/rnc/create_bucket_configuration.rnc | 4 + doc/rnc/delete.rnc | 8 ++ doc/rnc/delete_result.rnc | 17 +++ doc/rnc/error.rnc | 11 ++ doc/rnc/initiate_multipart_upload_result.rnc | 6 + doc/rnc/lifecycle_configuration.rnc | 20 ++++ doc/rnc/list_all_my_buckets_result.rnc | 12 ++ doc/rnc/list_bucket_result.rnc | 24 ++++ doc/rnc/list_multipart_uploads_result.rnc | 26 +++++ doc/rnc/list_parts_result.rnc | 22 ++++ doc/rnc/list_versions_result.rnc | 37 +++++++ doc/rnc/location_constraint.rnc | 1 + doc/rnc/versioning_configuration.rnc | 5 + swift3/controllers/acl.py | 8 +- swift3/controllers/bucket.py | 5 +- swift3/controllers/multi_delete.py | 12 +- swift3/controllers/service.py | 5 + swift3/etree.py | 28 ++++- swift3/response.py | 4 +- swift3/schema/access_control_policy.rng | 16 +++ swift3/schema/bucket_logging_status.rng | 25 +++++ swift3/schema/common.rng | 66 +++++++++++ swift3/schema/complete_multipart_upload.rng | 19 ++++ .../complete_multipart_upload_result.rng | 19 ++++ swift3/schema/copy_object_result.rng | 13 +++ swift3/schema/create_bucket_configuration.rng | 10 ++ swift3/schema/delete.rng | 28 +++++ swift3/schema/delete_result.rng | 47 ++++++++ swift3/schema/error.rng | 30 +++++ .../initiate_multipart_upload_result.rng | 16 +++ swift3/schema/lifecycle_configuration.rng | 56 ++++++++++ swift3/schema/list_all_my_buckets_result.rng | 23 ++++ swift3/schema/list_bucket_result.rng | 69 ++++++++++++ .../schema/list_multipart_uploads_result.rng | 73 ++++++++++++ swift3/schema/list_parts_result.rng | 59 ++++++++++ swift3/schema/list_versions_result.rng | 104 ++++++++++++++++++ swift3/schema/location_constraint.rng | 8 ++ swift3/schema/versioning_configuration.rng | 25 +++++ swift3/test/functional/001.out | 4 + swift3/test/functional/003.out | 4 +- swift3/test/unit/__init__.py | 3 +- swift3/test/unit/test_acl.py | 3 +- swift3/test/unit/test_bucket.py | 13 +-- swift3/test/unit/test_middleware.py | 3 +- swift3/test/unit/test_service.py | 3 +- swift3/test/unit/test_utils.py | 34 ++++++ swift3/test/unit/test_versioning.py | 3 +- swift3/utils.py | 23 ++++ 54 files changed, 1087 insertions(+), 29 deletions(-) create mode 100644 doc/rnc/access_control_policy.rnc create mode 100644 doc/rnc/bucket_logging_status.rnc create mode 100644 doc/rnc/common.rnc create mode 100644 doc/rnc/complete_multipart_upload.rnc create mode 100644 doc/rnc/complete_multipart_upload_result.rnc create mode 100644 doc/rnc/copy_object_result.rnc create mode 100644 doc/rnc/create_bucket_configuration.rnc create mode 100644 doc/rnc/delete.rnc create mode 100644 doc/rnc/delete_result.rnc create mode 100644 doc/rnc/error.rnc create mode 100644 doc/rnc/initiate_multipart_upload_result.rnc create mode 100644 doc/rnc/lifecycle_configuration.rnc create mode 100644 doc/rnc/list_all_my_buckets_result.rnc create mode 100644 doc/rnc/list_bucket_result.rnc create mode 100644 doc/rnc/list_multipart_uploads_result.rnc create mode 100644 doc/rnc/list_parts_result.rnc create mode 100644 doc/rnc/list_versions_result.rnc create mode 100644 doc/rnc/location_constraint.rnc create mode 100644 doc/rnc/versioning_configuration.rnc create mode 100644 swift3/schema/access_control_policy.rng create mode 100644 swift3/schema/bucket_logging_status.rng create mode 100644 swift3/schema/common.rng create mode 100644 swift3/schema/complete_multipart_upload.rng create mode 100644 swift3/schema/complete_multipart_upload_result.rng create mode 100644 swift3/schema/copy_object_result.rng create mode 100644 swift3/schema/create_bucket_configuration.rng create mode 100644 swift3/schema/delete.rng create mode 100644 swift3/schema/delete_result.rng create mode 100644 swift3/schema/error.rng create mode 100644 swift3/schema/initiate_multipart_upload_result.rng create mode 100644 swift3/schema/lifecycle_configuration.rng create mode 100644 swift3/schema/list_all_my_buckets_result.rng create mode 100644 swift3/schema/list_bucket_result.rng create mode 100644 swift3/schema/list_multipart_uploads_result.rng create mode 100644 swift3/schema/list_parts_result.rng create mode 100644 swift3/schema/list_versions_result.rng create mode 100644 swift3/schema/location_constraint.rng create mode 100644 swift3/schema/versioning_configuration.rng create mode 100644 swift3/test/unit/test_utils.py create mode 100644 swift3/utils.py diff --git a/doc/rnc/access_control_policy.rnc b/doc/rnc/access_control_policy.rnc new file mode 100644 index 00000000..c857359e --- /dev/null +++ b/doc/rnc/access_control_policy.rnc @@ -0,0 +1,7 @@ +include "common.rnc" + +start = + element AccessControlPolicy { + element Owner { CanonicalUser } & + element AccessControlList { AccessControlList } + } diff --git a/doc/rnc/bucket_logging_status.rnc b/doc/rnc/bucket_logging_status.rnc new file mode 100644 index 00000000..a7d9a1ef --- /dev/null +++ b/doc/rnc/bucket_logging_status.rnc @@ -0,0 +1,10 @@ +include "common.rnc" + +start = + element BucketLoggingStatus { + element LoggingEnabled { + element TargetBucket { xsd:string } & + element TargetPrefix { xsd:string } & + element TargetGrants { AccessControlList }? + }? + } diff --git a/doc/rnc/common.rnc b/doc/rnc/common.rnc new file mode 100644 index 00000000..79dddbb5 --- /dev/null +++ b/doc/rnc/common.rnc @@ -0,0 +1,26 @@ +namespace xsi = "http://www.w3.org/2001/XMLSchema-instance" + +CanonicalUser = + element ID { xsd:string } & + element DisplayName { xsd:string }? + +StorageClass = "STANDARD" | "REDUCED_REDUNDANCY" | "GLACIER" | "UNKNOWN" + +AccessControlList = + element Grant { + element Grantee { + ( + attribute xsi:type { "AmazonCustomerByEmail" }, + element EmailAddress { xsd:string } + ) | ( + attribute xsi:type { "CanonicalUser" }, + CanonicalUser + ) | ( + attribute xsi:type { "Group" }, + element URI { xsd:string } + ) + } & + element Permission { + "READ" | "WRITE" | "READ_ACP" | "WRITE_ACP" | "FULL_CONTROL" + } + }* diff --git a/doc/rnc/complete_multipart_upload.rnc b/doc/rnc/complete_multipart_upload.rnc new file mode 100644 index 00000000..dee60e54 --- /dev/null +++ b/doc/rnc/complete_multipart_upload.rnc @@ -0,0 +1,7 @@ +start = + element CompleteMultipartUpload { + element Part { + element PartNumber { xsd:int } & + element ETag { xsd:string } + }+ + } diff --git a/doc/rnc/complete_multipart_upload_result.rnc b/doc/rnc/complete_multipart_upload_result.rnc new file mode 100644 index 00000000..6dd9cbeb --- /dev/null +++ b/doc/rnc/complete_multipart_upload_result.rnc @@ -0,0 +1,7 @@ +start = + element CompleteMultipartUploadResult { + element Location { xsd:anyURI }, + element Bucket { xsd:string }, + element Key { xsd:string }, + element ETag { xsd:string } + } diff --git a/doc/rnc/copy_object_result.rnc b/doc/rnc/copy_object_result.rnc new file mode 100644 index 00000000..bf96a8a9 --- /dev/null +++ b/doc/rnc/copy_object_result.rnc @@ -0,0 +1,5 @@ +start = + element CopyObjectResult { + element LastModified { xsd:dateTime }, + element ETag { xsd:string } + } diff --git a/doc/rnc/create_bucket_configuration.rnc b/doc/rnc/create_bucket_configuration.rnc new file mode 100644 index 00000000..53f07d8a --- /dev/null +++ b/doc/rnc/create_bucket_configuration.rnc @@ -0,0 +1,4 @@ +start = + element CreateBucketConfiguration { + element LocationConstraint { xsd:string } + } diff --git a/doc/rnc/delete.rnc b/doc/rnc/delete.rnc new file mode 100644 index 00000000..95214f01 --- /dev/null +++ b/doc/rnc/delete.rnc @@ -0,0 +1,8 @@ +start = + element Delete { + element Quiet { xsd:boolean }? & + element Object { + element Key { xsd:string } & + element VersionId { xsd:string }? + }+ + } diff --git a/doc/rnc/delete_result.rnc b/doc/rnc/delete_result.rnc new file mode 100644 index 00000000..3a63bf78 --- /dev/null +++ b/doc/rnc/delete_result.rnc @@ -0,0 +1,17 @@ +start = + element DeleteResult { + ( + element Deleted { + element Key { xsd:string }, + element VersionId { xsd:string }?, + element DeleteMarker { xsd:boolean }?, + element DeleteMarkerVersionId { xsd:string }? + } | + element Error { + element Key { xsd:string }, + element VersionId { xsd:string }?, + element Code { xsd:string }, + element Message { xsd:string } + } + )* + } diff --git a/doc/rnc/error.rnc b/doc/rnc/error.rnc new file mode 100644 index 00000000..0e352c71 --- /dev/null +++ b/doc/rnc/error.rnc @@ -0,0 +1,11 @@ +start = + element Error { + element Code { xsd:string }, + element Message { xsd:string }, + DebugInfo* + } + +DebugInfo = + element * { + (attribute * { text } | text | DebugInfo)* + } diff --git a/doc/rnc/initiate_multipart_upload_result.rnc b/doc/rnc/initiate_multipart_upload_result.rnc new file mode 100644 index 00000000..8830121f --- /dev/null +++ b/doc/rnc/initiate_multipart_upload_result.rnc @@ -0,0 +1,6 @@ +start = + element InitiateMultipartUploadResult { + element Bucket { xsd:string }, + element Key { xsd:string }, + element UploadId { xsd:string } + } diff --git a/doc/rnc/lifecycle_configuration.rnc b/doc/rnc/lifecycle_configuration.rnc new file mode 100644 index 00000000..b21fc07b --- /dev/null +++ b/doc/rnc/lifecycle_configuration.rnc @@ -0,0 +1,20 @@ +include "common.rnc" + +start = + element LifecycleConfiguration { + element Rule { + element ID { xsd:string }? & + element Prefix { xsd:string } & + element Status { "Enabled" | "Disabled" } & + element Transition { Transition }? & + element Expiration { Expiration }? + }+ + } + +Expiration = + element Days { xsd:int } | + element Date { xsd:dateTime } + +Transition = + Expiration & + element StorageClass { StorageClass } diff --git a/doc/rnc/list_all_my_buckets_result.rnc b/doc/rnc/list_all_my_buckets_result.rnc new file mode 100644 index 00000000..220a34aa --- /dev/null +++ b/doc/rnc/list_all_my_buckets_result.rnc @@ -0,0 +1,12 @@ +include "common.rnc" + +start = + element ListAllMyBucketsResult { + element Owner { CanonicalUser }, + element Buckets { + element Bucket { + element Name { xsd:string }, + element CreationDate { xsd:dateTime } + }* + } + } diff --git a/doc/rnc/list_bucket_result.rnc b/doc/rnc/list_bucket_result.rnc new file mode 100644 index 00000000..1dac2ac8 --- /dev/null +++ b/doc/rnc/list_bucket_result.rnc @@ -0,0 +1,24 @@ +include "common.rnc" + +start = + element ListBucketResult { + element Name { xsd:string }, + element Prefix { xsd:string }, + element Marker { xsd:string }, + element NextMarker { xsd:string }?, + element MaxKeys { xsd:int }, + element EncodingType { xsd:string }?, + element Delimiter { xsd:string }?, + element IsTruncated { xsd:boolean }, + element Contents { + element Key { xsd:string }, + element LastModified { xsd:dateTime }, + element ETag { xsd:string }, + element Size { xsd:long }, + element Owner { CanonicalUser }?, + element StorageClass { StorageClass } + }*, + element CommonPrefixes { + element Prefix { xsd:string } + }* + } diff --git a/doc/rnc/list_multipart_uploads_result.rnc b/doc/rnc/list_multipart_uploads_result.rnc new file mode 100644 index 00000000..6ac1e123 --- /dev/null +++ b/doc/rnc/list_multipart_uploads_result.rnc @@ -0,0 +1,26 @@ +include "common.rnc" + +start = + element ListMultipartUploadsResult { + element Bucket { xsd:string }, + element KeyMarker { xsd:string }, + element UploadIdMarker { xsd:string }, + element NextKeyMarker { xsd:string }, + element NextUploadIdMarker { xsd:string }, + element Delimiter { xsd:string }?, + element Prefix { xsd:string }?, + element MaxUploads { xsd:int }, + element EncodingType { xsd:string }?, + element IsTruncated { xsd:boolean }, + element Upload { + element Key { xsd:string }, + element UploadId { xsd:string }, + element Initiator { CanonicalUser }, + element Owner { CanonicalUser }, + element StorageClass { StorageClass }, + element Initiated { xsd:dateTime } + }*, + element CommonPrefixes { + element Prefix { xsd:string } + }* + } diff --git a/doc/rnc/list_parts_result.rnc b/doc/rnc/list_parts_result.rnc new file mode 100644 index 00000000..21433154 --- /dev/null +++ b/doc/rnc/list_parts_result.rnc @@ -0,0 +1,22 @@ +include "common.rnc" + +start = + element ListPartsResult { + element Bucket { xsd:string }, + element Key { xsd:string }, + element UploadId { xsd:string }, + element Initiator { CanonicalUser }, + element Owner { CanonicalUser }, + element StorageClass { StorageClass }, + element PartNumberMarker { xsd:int }, + element NextPartNumberMarker { xsd:int }, + element MaxParts { xsd:int }, + element EncodingType { xsd:string }?, + element IsTruncated { xsd:boolean }, + element Part { + element PartNumber { xsd:int }, + element LastModified { xsd:dateTime }, + element ETag { xsd:string }, + element Size { xsd:long } + }* + } diff --git a/doc/rnc/list_versions_result.rnc b/doc/rnc/list_versions_result.rnc new file mode 100644 index 00000000..969073f3 --- /dev/null +++ b/doc/rnc/list_versions_result.rnc @@ -0,0 +1,37 @@ +include "common.rnc" + +start = + element ListVersionsResult { + element Name { xsd:string }, + element Prefix { xsd:string }, + element KeyMarker { xsd:string }, + element VersionIdMarker { xsd:string }, + element NextKeyMarker { xsd:string }?, + element NextVersionIdMarker { xsd:string }?, + element MaxKeys { xsd:int }, + element EncodingType { xsd:string }?, + element Delimiter { xsd:string }?, + element IsTruncated { xsd:boolean }, + ( + element Version { + element Key { xsd:string }, + element VersionId { xsd:string }, + element IsLatest { xsd:boolean }, + element LastModified { xsd:dateTime }, + element ETag { xsd:string }, + element Size { xsd:long }, + element Owner { CanonicalUser }?, + element StorageClass { StorageClass } + } | + element DeleteMarker { + element Key { xsd:string }, + element VersionId { xsd:string }, + element IsLatest { xsd:boolean }, + element LastModified { xsd:dateTime }, + element Owner { CanonicalUser }? + } + )*, + element CommonPrefixes { + element Prefix { xsd:string } + }* + } diff --git a/doc/rnc/location_constraint.rnc b/doc/rnc/location_constraint.rnc new file mode 100644 index 00000000..829176ff --- /dev/null +++ b/doc/rnc/location_constraint.rnc @@ -0,0 +1 @@ +start = element LocationConstraint { xsd:string } diff --git a/doc/rnc/versioning_configuration.rnc b/doc/rnc/versioning_configuration.rnc new file mode 100644 index 00000000..87e5d15a --- /dev/null +++ b/doc/rnc/versioning_configuration.rnc @@ -0,0 +1,5 @@ +start = + element VersioningConfiguration { + element Status { "Enabled" | "Suspended" }? & + element MfaDelete { "Enabled" | "Disabled" }? + } diff --git a/swift3/controllers/acl.py b/swift3/controllers/acl.py index 6d41cd6e..471a35c0 100644 --- a/swift3/controllers/acl.py +++ b/swift3/controllers/acl.py @@ -18,7 +18,8 @@ from swift.common.middleware.acl import parse_acl, referrer_allowed from swift3.controllers.base import Controller from swift3.response import HTTPOk, S3NotImplemented, MalformedACLError -from swift3.etree import Element, SubElement, fromstring, tostring +from swift3.etree import Element, SubElement, fromstring, tostring, \ + DocumentInvalid XMLNS_XSI = 'http://www.w3.org/2001/XMLSchema-instance' @@ -97,7 +98,10 @@ def swift_acl_translate(acl, group='', user='', xml=False): ['HTTP_X_CONTAINER_READ', '.']] if xml: # We are working with XML and need to parse it - elem = fromstring(acl) + try: + elem = fromstring(acl, 'AccessControlPolicy') + except DocumentInvalid: + raise MalformedACLError() acl = 'unknown' for grant in elem.findall('./AccessControlList/Grant'): permission = grant.find('./Permission').text diff --git a/swift3/controllers/bucket.py b/swift3/controllers/bucket.py index e5ad92dd..61094448 100644 --- a/swift3/controllers/bucket.py +++ b/swift3/controllers/bucket.py @@ -62,16 +62,16 @@ class BucketController(Controller): objects = loads(resp.body) elem = Element('ListBucketResult') + SubElement(elem, 'Name').text = req.container_name SubElement(elem, 'Prefix').text = req.params.get('prefix') SubElement(elem, 'Marker').text = req.params.get('marker') + SubElement(elem, 'MaxKeys').text = str(max_keys) SubElement(elem, 'Delimiter').text = req.params.get('delimiter') if max_keys > 0 and len(objects) == max_keys + 1: is_truncated = 'true' else: is_truncated = 'false' SubElement(elem, 'IsTruncated').text = is_truncated - SubElement(elem, 'MaxKeys').text = str(max_keys) - SubElement(elem, 'Name').text = req.container_name for o in objects[:max_keys]: if 'subdir' not in o: @@ -82,6 +82,7 @@ class BucketController(Controller): SubElement(contents, 'ETag').text = o['hash'] SubElement(contents, 'Size').text = str(o['bytes']) add_canonical_user(contents, 'Owner', req.user_id) + SubElement(contents, 'StorageClass').text = 'STANDARD' for o in objects[:max_keys]: if 'subdir' in o: diff --git a/swift3/controllers/multi_delete.py b/swift3/controllers/multi_delete.py index 9e155965..85bcc76d 100644 --- a/swift3/controllers/multi_delete.py +++ b/swift3/controllers/multi_delete.py @@ -14,8 +14,10 @@ # limitations under the License. from swift3.controllers.base import Controller -from swift3.etree import Element, SubElement, fromstring, tostring -from swift3.response import HTTPOk, S3NotImplemented, NoSuchKey, ErrorResponse +from swift3.etree import Element, SubElement, fromstring, tostring, \ + DocumentInvalid +from swift3.response import HTTPOk, S3NotImplemented, NoSuchKey, \ + ErrorResponse, MalformedXML class MultiObjectDeleteController(Controller): @@ -28,7 +30,11 @@ class MultiObjectDeleteController(Controller): Handles Delete Multiple Objects. """ def object_key_iter(xml): - elem = fromstring(xml) + try: + elem = fromstring(xml, 'Delete') + except DocumentInvalid: + raise MalformedXML() + for obj in elem.iterchildren('Object'): key = obj.find('./Key').text version = obj.find('./VersionId') diff --git a/swift3/controllers/service.py b/swift3/controllers/service.py index bfcb29e6..1f57149f 100644 --- a/swift3/controllers/service.py +++ b/swift3/controllers/service.py @@ -34,6 +34,11 @@ class ServiceController(Controller): # we don't keep the creation time of a backet (s3cmd doesn't # work without that) so we use something bogus. elem = Element('ListAllMyBucketsResult') + + owner = SubElement(elem, 'Owner') + SubElement(owner, 'ID').text = req.user_id + SubElement(owner, 'DisplayName').text = req.user_id + buckets = SubElement(elem, 'Buckets') for c in containers: bucket = SubElement(buckets, 'Bucket') diff --git a/swift3/etree.py b/swift3/etree.py index e2a2b944..e33fa7c0 100644 --- a/swift3/etree.py +++ b/swift3/etree.py @@ -15,9 +15,21 @@ import lxml.etree from copy import deepcopy +from pkg_resources import resource_stream + +from swift.common.utils import get_logger + +from swift3.exception import S3Exception +from swift3.utils import camel_to_snake +from swift3.cfg import CONF XMLNS_S3 = 'http://s3.amazonaws.com/doc/2006-03-01/' +LOGGER = get_logger(CONF, log_route='swift3') + +class DocumentInvalid(S3Exception, lxml.etree.DocumentInvalid): + pass + def cleanup_namespaces(elem): def remove_ns(tag, ns): @@ -36,10 +48,24 @@ def cleanup_namespaces(elem): cleanup_namespaces(e) -def fromstring(text): +def fromstring(text, root_tag=None): elem = lxml.etree.fromstring(text) cleanup_namespaces(elem) + if root_tag is not None: + # validate XML + try: + path = 'schema/%s.rng' % camel_to_snake(root_tag) + rng = resource_stream(__name__, path) + lxml.etree.RelaxNG(file=rng).assertValid(elem) + except IOError as e: + # Probably, the schema file doesn't exist. + LOGGER.error(e) + raise + except lxml.etree.DocumentInvalid as e: + LOGGER.debug(e) + raise DocumentInvalid(e) + return elem diff --git a/swift3/response.py b/swift3/response.py index baf7bcdd..141ff6dc 100644 --- a/swift3/response.py +++ b/swift3/response.py @@ -19,6 +19,7 @@ from functools import partial from swift.common import swob +from swift3.utils import snake_to_camel from swift3.etree import Element, SubElement, tostring @@ -157,8 +158,7 @@ class ErrorResponse(swob.HTTPException): def _dict_to_etree(self, parent, d): for key, value in d.items(): - tag = key.title().replace('_', '') - tag = re.sub('\W', '', tag) + tag = re.sub('\W', '', snake_to_camel(key)) elem = SubElement(parent, tag) if isinstance(value, (dict, DictMixin)): diff --git a/swift3/schema/access_control_policy.rng b/swift3/schema/access_control_policy.rng new file mode 100644 index 00000000..5308a12f --- /dev/null +++ b/swift3/schema/access_control_policy.rng @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/swift3/schema/bucket_logging_status.rng b/swift3/schema/bucket_logging_status.rng new file mode 100644 index 00000000..27ea1e1d --- /dev/null +++ b/swift3/schema/bucket_logging_status.rng @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/common.rng b/swift3/schema/common.rng new file mode 100644 index 00000000..22319c0e --- /dev/null +++ b/swift3/schema/common.rng @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + STANDARD + REDUCED_REDUNDANCY + GLACIER + UNKNOWN + + + + + + + + + + + AmazonCustomerByEmail + + + + + + + + CanonicalUser + + + + + + Group + + + + + + + + + + READ + WRITE + READ_ACP + WRITE_ACP + FULL_CONTROL + + + + + + + diff --git a/swift3/schema/complete_multipart_upload.rng b/swift3/schema/complete_multipart_upload.rng new file mode 100644 index 00000000..d7ba2569 --- /dev/null +++ b/swift3/schema/complete_multipart_upload.rng @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/complete_multipart_upload_result.rng b/swift3/schema/complete_multipart_upload_result.rng new file mode 100644 index 00000000..47406e1c --- /dev/null +++ b/swift3/schema/complete_multipart_upload_result.rng @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/copy_object_result.rng b/swift3/schema/copy_object_result.rng new file mode 100644 index 00000000..ec0ac95f --- /dev/null +++ b/swift3/schema/copy_object_result.rng @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/swift3/schema/create_bucket_configuration.rng b/swift3/schema/create_bucket_configuration.rng new file mode 100644 index 00000000..0e22a298 --- /dev/null +++ b/swift3/schema/create_bucket_configuration.rng @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/swift3/schema/delete.rng b/swift3/schema/delete.rng new file mode 100644 index 00000000..3417a394 --- /dev/null +++ b/swift3/schema/delete.rng @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/delete_result.rng b/swift3/schema/delete_result.rng new file mode 100644 index 00000000..1e28b3ce --- /dev/null +++ b/swift3/schema/delete_result.rng @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/error.rng b/swift3/schema/error.rng new file mode 100644 index 00000000..a0d61d48 --- /dev/null +++ b/swift3/schema/error.rng @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/initiate_multipart_upload_result.rng b/swift3/schema/initiate_multipart_upload_result.rng new file mode 100644 index 00000000..67d03016 --- /dev/null +++ b/swift3/schema/initiate_multipart_upload_result.rng @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/swift3/schema/lifecycle_configuration.rng b/swift3/schema/lifecycle_configuration.rng new file mode 100644 index 00000000..dd0816e2 --- /dev/null +++ b/swift3/schema/lifecycle_configuration.rng @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + Enabled + Disabled + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/list_all_my_buckets_result.rng b/swift3/schema/list_all_my_buckets_result.rng new file mode 100644 index 00000000..76959d7b --- /dev/null +++ b/swift3/schema/list_all_my_buckets_result.rng @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/list_bucket_result.rng b/swift3/schema/list_bucket_result.rng new file mode 100644 index 00000000..f463fe38 --- /dev/null +++ b/swift3/schema/list_bucket_result.rng @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/list_multipart_uploads_result.rng b/swift3/schema/list_multipart_uploads_result.rng new file mode 100644 index 00000000..2e20c840 --- /dev/null +++ b/swift3/schema/list_multipart_uploads_result.rng @@ -0,0 +1,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/list_parts_result.rng b/swift3/schema/list_parts_result.rng new file mode 100644 index 00000000..4cf5a0ce --- /dev/null +++ b/swift3/schema/list_parts_result.rng @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/list_versions_result.rng b/swift3/schema/list_versions_result.rng new file mode 100644 index 00000000..464cfbcc --- /dev/null +++ b/swift3/schema/list_versions_result.rng @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/swift3/schema/location_constraint.rng b/swift3/schema/location_constraint.rng new file mode 100644 index 00000000..2f3a143b --- /dev/null +++ b/swift3/schema/location_constraint.rng @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/swift3/schema/versioning_configuration.rng b/swift3/schema/versioning_configuration.rng new file mode 100644 index 00000000..3d6d3d12 --- /dev/null +++ b/swift3/schema/versioning_configuration.rng @@ -0,0 +1,25 @@ + + + + + + + + + Enabled + Suspended + + + + + + + Enabled + Disabled + + + + + + + diff --git a/swift3/test/functional/001.out b/swift3/test/functional/001.out index ed8923b2..5ee2c59f 100644 --- a/swift3/test/functional/001.out +++ b/swift3/test/functional/001.out @@ -10,6 +10,10 @@ X-Trans-Id: TXID + + TESTER + TESTER + 5ABCDEFGHIJKLMNOPQRSTUVWXYZ.5-6789 diff --git a/swift3/test/functional/003.out b/swift3/test/functional/003.out index 11ddcad6..54f1a538 100644 --- a/swift3/test/functional/003.out +++ b/swift3/test/functional/003.out @@ -14,12 +14,12 @@ X-Trans-Id: TXID + bucket photos/2006/ + 1000 / false - 1000 - bucket photos/2006/February/ diff --git a/swift3/test/unit/__init__.py b/swift3/test/unit/__init__.py index 64066f6a..e21a45b9 100644 --- a/swift3/test/unit/__init__.py +++ b/swift3/test/unit/__init__.py @@ -76,8 +76,7 @@ class Swift3TestCase(unittest.TestCase): swob.HTTPNoContent, {}, None) def _get_error_code(self, body): - elem = fromstring(body) - self.assertEquals(elem.tag, 'Error') + elem = fromstring(body, 'Error') return elem.find('./Code').text def _test_method_error(self, method, path, response_class, headers={}): diff --git a/swift3/test/unit/test_acl.py b/swift3/test/unit/test_acl.py index 94ec527c..da9e1f39 100644 --- a/swift3/test/unit/test_acl.py +++ b/swift3/test/unit/test_acl.py @@ -29,8 +29,7 @@ class TestSwift3Acl(Swift3TestCase): super(TestSwift3Acl, self).setUp() def _check_acl(self, owner, body): - elem = fromstring(body) - self.assertEquals(elem.tag, 'AccessControlPolicy') + elem = fromstring(body, 'AccessControlPolicy') permission = elem.find('./AccessControlList/Grant/Permission').text self.assertEquals(permission, 'FULL_CONTROL') name = elem.find('./AccessControlList/Grant/Grantee/ID').text diff --git a/swift3/test/unit/test_bucket.py b/swift3/test/unit/test_bucket.py index 3297a777..9c8a2d11 100644 --- a/swift3/test/unit/test_bucket.py +++ b/swift3/test/unit/test_bucket.py @@ -66,8 +66,7 @@ class TestSwift3Bucket(Swift3TestCase): status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200') - elem = fromstring(body) - self.assertEquals(elem.tag, 'ListBucketResult') + elem = fromstring(body, 'ListBucketResult') name = elem.find('./Name').text self.assertEquals(name, bucket_name) @@ -90,7 +89,7 @@ class TestSwift3Bucket(Swift3TestCase): 'QUERY_STRING': 'max-keys=5'}, headers={'Authorization': 'AWS test:tester:hmac'}) status, headers, body = self.call_swift3(req) - elem = fromstring(body) + elem = fromstring(body, 'ListBucketResult') self.assertEquals(elem.find('./IsTruncated').text, 'false') req = Request.blank('/%s' % bucket_name, @@ -98,7 +97,7 @@ class TestSwift3Bucket(Swift3TestCase): 'QUERY_STRING': 'max-keys=4'}, headers={'Authorization': 'AWS test:tester:hmac'}) status, headers, body = self.call_swift3(req) - elem = fromstring(body) + elem = fromstring(body, 'ListBucketResult') self.assertEquals(elem.find('./IsTruncated').text, 'true') def test_bucket_GET_max_keys(self): @@ -109,7 +108,7 @@ class TestSwift3Bucket(Swift3TestCase): 'QUERY_STRING': 'max-keys=5'}, headers={'Authorization': 'AWS test:tester:hmac'}) status, headers, body = self.call_swift3(req) - elem = fromstring(body) + elem = fromstring(body, 'ListBucketResult') self.assertEquals(elem.find('./MaxKeys').text, '5') _, path = self.swift.calls[-1] _, query_string = path.split('?') @@ -121,7 +120,7 @@ class TestSwift3Bucket(Swift3TestCase): 'QUERY_STRING': 'max-keys=5000'}, headers={'Authorization': 'AWS test:tester:hmac'}) status, headers, body = self.call_swift3(req) - elem = fromstring(body) + elem = fromstring(body, 'ListBucketResult') self.assertEquals(elem.find('./MaxKeys').text, '1000') _, path = self.swift.calls[-1] _, query_string = path.split('?') @@ -135,7 +134,7 @@ class TestSwift3Bucket(Swift3TestCase): 'delimiter=a&marker=b&prefix=c'}, headers={'Authorization': 'AWS test:tester:hmac'}) status, headers, body = self.call_swift3(req) - elem = fromstring(body) + elem = fromstring(body, 'ListBucketResult') self.assertEquals(elem.find('./Prefix').text, 'c') self.assertEquals(elem.find('./Marker').text, 'b') self.assertEquals(elem.find('./Delimiter').text, 'a') diff --git a/swift3/test/unit/test_middleware.py b/swift3/test/unit/test_middleware.py index 526bd838..2b1c83a9 100644 --- a/swift3/test/unit/test_middleware.py +++ b/swift3/test/unit/test_middleware.py @@ -256,9 +256,8 @@ class TestSwift3Middleware(Swift3TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac'}, body=xml) - # FIXME: swift3 should handle invalid xml file status, headers, body = self.call_swift3(req) - self.assertEquals(self._get_error_code(body), 'InternalError') + self.assertEquals(self._get_error_code(body), 'MalformedACLError') def _test_unsupported_resource(self, resource): req = Request.blank('/error?' + resource, diff --git a/swift3/test/unit/test_service.py b/swift3/test/unit/test_service.py index 2aab6107..4b192cb1 100644 --- a/swift3/test/unit/test_service.py +++ b/swift3/test/unit/test_service.py @@ -59,8 +59,7 @@ class TestSwift3Service(Swift3TestCase): status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200') - elem = fromstring(body) - self.assertEquals(elem.tag, 'ListAllMyBucketsResult') + elem = fromstring(body, 'ListAllMyBucketsResult') all_buckets = elem.find('./Buckets') buckets = all_buckets.iterchildren('Bucket') diff --git a/swift3/test/unit/test_utils.py b/swift3/test/unit/test_utils.py new file mode 100644 index 00000000..297e678f --- /dev/null +++ b/swift3/test/unit/test_utils.py @@ -0,0 +1,34 @@ +# Copyright (c) 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from swift3 import utils + +strs = [ + ('Owner', 'owner'), + ('DisplayName', 'display_name'), + ('AccessControlPolicy', 'access_control_policy'), +] + + +class TestSwift3Utils(unittest.TestCase): + def test_camel_to_snake(self): + for s1, s2 in strs: + self.assertEquals(utils.camel_to_snake(s1), s2) + + def test_snake_to_camel(self): + for s1, s2 in strs: + self.assertEquals(s1, utils.snake_to_camel(s2)) diff --git a/swift3/test/unit/test_versioning.py b/swift3/test/unit/test_versioning.py index 36110b78..4999e757 100644 --- a/swift3/test/unit/test_versioning.py +++ b/swift3/test/unit/test_versioning.py @@ -31,8 +31,7 @@ class TestSwift3Versioning(Swift3TestCase): environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac'}) status, headers, body = self.call_swift3(req) - elem = fromstring(body) - self.assertEquals(elem.tag, 'VersioningConfiguration') + fromstring(body, 'VersioningConfiguration') if __name__ == '__main__': unittest.main() diff --git a/swift3/utils.py b/swift3/utils.py new file mode 100644 index 00000000..b09ef252 --- /dev/null +++ b/swift3/utils.py @@ -0,0 +1,23 @@ +# Copyright (c) 2014 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +def camel_to_snake(camel): + return re.sub('(.)([A-Z])', r'\1_\2', camel).lower() + + +def snake_to_camel(snake): + return snake.title().replace('_', '')