Merge "Import swift3 into swift repo as s3api middleware"

.zuul.yaml

@@ -105,6 +105,18 @@
     vars:
       tox_envlist: func-domain-remap-staticweb
 
+- job:
+    name: swift-tox-func-s3api
+    parent: swift-tox-base
+    description: |
+      Run functional tests for swift under cPython version 2.7.
+
+      Uses tox with the ``func-s3api`` environment.
+      It sets TMPDIR to an XFS mount point created via
+      tools/test-setup.sh.
+    vars:
+      tox_envlist: func-s3api
+
 - job:
     name: swift-probetests-centos-7
     parent: unittests
@@ -128,6 +140,7 @@
         - swift-tox-func-encryption
         - swift-tox-func-domain-remap-staticweb
         - swift-tox-func-ec
+        - swift-tox-func-s3api
         - swift-probetests-centos-7
     gate:
       jobs:
@@ -137,6 +150,7 @@
         - swift-tox-func-encryption
         - swift-tox-func-domain-remap-staticweb
         - swift-tox-func-ec
+        - swift-tox-func-s3api
     experimental:
       jobs:
         - swift-tox-py27-centos-7

AUTHORS

@@ -72,6 +72,7 @@ Brian Ober (bober@us.ibm.com)
 Brian Reitz (brian.reitz@oracle.com)
 Bryan Keller (kellerbr@us.ibm.com)
 Béla Vancsics (vancsics@inf.u-szeged.hu)
+Виль Суркин (vills@vills-pro.local)
 Caleb Tennis (caleb.tennis@gmail.com)
 Cao Xuan Hoang (hoangcx@vn.fujitsu.com)
 Carlos Cavanna (ccavanna@ca.ibm.com)
@@ -111,6 +112,7 @@ Dan Prince (dprince@redhat.com)
 dangming (dangming@unitedstack.com)
 Daniele Valeriani (daniele@dvaleriani.net)
 Darrell Bishop (darrell@swiftstack.com)
+Darryl Tam (dtam@swiftstack.com)
 David Goetz (david.goetz@rackspace.com)
 David Hadas (davidh@il.ibm.com)
 David Liu (david.liu@cn.ibm.com)
@@ -253,6 +255,7 @@ Martin Geisler (martin@geisler.net)
 Martin Kletzander (mkletzan@redhat.com)
 Maru Newby (mnewby@internap.com)
 Mathias Bjoerkqvist (mbj@zurich.ibm.com)
+Masaki Tsukuda (tsukuda.masaki@po.ntts.co.jp)
 Matt Kassawara (mkassawara@gmail.com)
 Matt Riedemann (mriedem@us.ibm.com)
 Matthew Oliver (matt@oliver.net.au)
@@ -274,6 +277,8 @@ Nakagawa Masaaki (nakagawamsa@nttdata.co.jp)
 Nakul Dahiwade (nakul.dahiwade@intel.com)
 Nam Nguyen Hoai (namnh@vn.fujitsu.com)
 Nandini Tata (nandini.tata@intel.com)
+Naoto Nishizono (nishizono.naoto@po.ntts.co.jp)
+Nassim Babaci (nassim.babaci@cloudwatt.com)
 Nathan Kinder (nkinder@redhat.com)
 Nelson Almeida (nelsonmarcos@gmail.com)
 Newptone (xingchao@unitedstack.com)
@@ -365,11 +370,13 @@ Victor Lowther (victor.lowther@gmail.com)
 Victor Rodionov (victor.rodionov@nexenta.com)
 Victor Stinner (vstinner@redhat.com)
 Viktor Varga (vvarga@inf.u-szeged.hu)
+Vil Surkin (mail@vills.me)
 Vincent Untz (vuntz@suse.com)
 Vladimir Vechkanov (vvechkanov@mirantis.com)
 Vu Cong Tuan (tuanvc@vn.fujitsu.com)
 vxlinux (yan.wei7@zte.com.cn)
 wanghongtaozz (wanghongtaozz@inspur.com)
+Wyllys Ingersoll (wyllys.ingersoll@evault.com)
 Wu Wenxiang (wu.wenxiang@99cloud.net)
 xhancar (pavel.hancar@gmail.com)
 XieYingYun (smokony@sina.com)

bindep.txt

@@ -10,6 +10,10 @@ liberasurecode-dev [platform:dpkg]
 liberasurecode-devel [platform:rpm !platform:centos]
 libffi-dev [platform:dpkg]
 libffi-devel [platform:rpm]
+libxml2-dev [platform:dpkg]
+libxml2-devel [platform:rpm]
+libxslt-devel [platform:rpm]
+libxslt1-dev [platform:dpkg]
 memcached
 python-dev [platform:dpkg]
 python-devel [platform:rpm]

doc/s3api/conf/ceph-known-failures-keystone.yaml

@@ -0,0 +1,209 @@
+ceph_s3:
+  <nose.suite.ContextSuite context=s3tests.functional>:teardown: {status: KNOWN}
+  <nose.suite.ContextSuite context=test_routing_generator>:setup: {status: KNOWN}
+  s3tests.functional.test_headers.test_bucket_create_bad_authorization_invalid_aws2: {status: KNOWN}
+  s3tests.functional.test_headers.test_bucket_create_bad_authorization_none: {status: KNOWN}
+  s3tests.functional.test_headers.test_object_create_bad_authorization_invalid_aws2: {status: KNOWN}
+  s3tests.functional.test_headers.test_object_create_bad_authorization_none: {status: KNOWN}
+  s3tests.functional.test_s3.test_100_continue: {status: KNOWN}
+  s3tests.functional.test_s3.test_atomic_conditional_write_1mb: {status: KNOWN}
+  s3tests.functional.test_s3.test_atomic_dual_conditional_write_1mb: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_default: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_email: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_email_notexist: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_nonexist_user: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_userid_fullcontrol: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_userid_read: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_userid_readacp: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_userid_write: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_userid_writeacp: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_no_grants: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acls_changes_persistent: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_xml_fullcontrol: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_xml_read: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_xml_readacp: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_xml_write: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_xml_writeacp: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_create_exists: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_header_acl_grants: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_list_objects_anonymous: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_list_objects_anonymous_fail: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_recreate_not_overriding: {status: KNOWN}
+  s3tests.functional.test_s3.test_cors_origin_response: {status: KNOWN}
+  s3tests.functional.test_s3.test_cors_origin_wildcard: {status: KNOWN}
+  s3tests.functional.test_s3.test_list_buckets_anonymous: {status: KNOWN}
+  s3tests.functional.test_s3.test_list_buckets_invalid_auth: {status: KNOWN}
+  s3tests.functional.test_s3.test_logging_toggle: {status: KNOWN}
+  s3tests.functional.test_s3.test_multipart_resend_first_finishes_last: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_acl_full_control_verify_owner: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_acl_xml: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_acl_xml_read: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_acl_xml_readacp: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_acl_xml_write: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_acl_xml_writeacp: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_canned_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_not_owned_object_bucket: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_replacing_metadata: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_giveaway: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_header_acl_grants: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get_bucket_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get_bucket_gone: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get_object_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get_object_gone: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_put: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_put_write_access: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_set_valid_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_anonymous_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_authenticated_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_authenticated_request_bad_access_key: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_case_insensitive_condition_fields: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_condition_is_case_sensitive: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_escaped_field_values: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_expired_policy: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_expires_is_case_sensitive: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_ignored_header: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_access_key: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_content_length_argument: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_date_format: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_request_field_value: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_signature: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_conditions_list: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_content_length_argument: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_expires_condition: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_policy_condition: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_signature: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_no_key_specified: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_request_missing_policy_specified_field: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_set_invalid_success_code: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_set_key_from_filename: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_set_success_code: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_success_redirect_action: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_upload_larger_than_chunk: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_upload_size_below_minimum: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_upload_size_limit_exceeded: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_user_specified_header: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifmatch_failed: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifmatch_good: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifmatch_nonexisted_failed: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifnonmatch_failed: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifnonmatch_good: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed: {status: KNOWN}
+  s3tests.functional.test_s3.test_set_cors: {status: KNOWN}
+  s3tests.functional.test_s3.test_stress_bucket_acls_changes: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioned_concurrent_object_create_and_remove: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioned_concurrent_object_create_concurrent_remove: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioned_object_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_bucket_create_suspend: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_copy_obj_version: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_multi_object_delete: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_multi_object_delete_with_marker: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_multi_object_delete_with_marker_create: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_overwrite_multipart: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_read_remove: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_read_remove_head: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_versions_remove_all: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_versions_remove_special_names: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_list_marker: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_plain_null_version_overwrite: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_plain_null_version_overwrite_suspended: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_plain_null_version_removal: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_suspend_versions: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_suspend_versions_simple: {status: KNOWN}
+  s3tests.functional.test_s3_website.check_can_test_website: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_bucket_private_redirectall_base: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_bucket_private_redirectall_path: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_bucket_private_redirectall_path_upgrade: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_nonexistant_bucket_rgw: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_nonexistant_bucket_s3: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_empty: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_empty_blockederrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_empty_gooderrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_empty_missingerrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_private_index: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_private_index_blockederrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_private_index_gooderrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_private_index_missingerrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_public_index: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_empty: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_empty_blockederrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_empty_gooderrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_empty_missingerrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_private_index: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_private_index_blockederrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_private_index_gooderrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_private_index_missingerrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_public_index: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_nonwebsite: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_private_abs: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_private_relative: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_public_abs: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_public_relative: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_configure_recreate: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_list_return_data_versioning: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy_another_bucket: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy_different_tenant: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy_set_condition_operator_end_with_IfExists: {status: KNOWN}
+  s3tests.functional.test_s3.test_delete_tags_obj_public: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_invalid_md5: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_method_head: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_multipart_bad_download: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_multipart_invalid_chunks_1: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_multipart_invalid_chunks_2: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_no_key: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_no_md5: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_other_key: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_post_object_authenticated_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_present: {status: KNOWN}
+  s3tests.functional.test_s3.test_get_obj_head_tagging: {status: KNOWN}
+  s3tests.functional.test_s3.test_get_obj_tagging: {status: KNOWN}
+  s3tests.functional.test_s3.test_get_tags_acl_public: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_deletemarker_expiration: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_expiration: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_expiration_date: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_get: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_get_no_id: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_id_too_long: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_multipart_expiration: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_noncur_expiration: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_rules_conflicted: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_same_id: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_date: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_deletemarker: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_empty_filter: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_filter: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_multipart: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_noncurrent: {status: KNOWN}
+  s3tests.functional.test_s3.test_multipart_copy_invalid_range: {status: KNOWN}
+  s3tests.functional.test_s3.test_multipart_copy_versioned: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_versioned_bucket: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_versioning_multipart_upload: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_empty_conditions: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_tags_anonymous_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_tags_authenticated_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_delete_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_excess_key_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_excess_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_excess_val_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_max_kvsize_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_max_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_modify_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_obj_with_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_tags_acl_public: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_method_head: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_multipart_invalid_chunks_1: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_multipart_invalid_chunks_2: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_multipart_upload: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_post_object_authenticated_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_present: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_read_declare: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_transfer_13b: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_transfer_1MB: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_transfer_1b: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_transfer_1kb: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioned_object_acl_no_version_specified: {status: KNOWN}

doc/s3api/conf/ceph-known-failures-tempauth.yaml

@@ -0,0 +1,187 @@
+ceph_s3:
+  <nose.suite.ContextSuite context=s3tests.functional>:teardown: {status: KNOWN}
+  <nose.suite.ContextSuite context=test_routing_generator>:setup: {status: KNOWN}
+  s3tests.functional.test_headers.test_bucket_create_bad_authorization_invalid_aws2: {status: KNOWN}
+  s3tests.functional.test_headers.test_bucket_create_bad_authorization_none: {status: KNOWN}
+  s3tests.functional.test_headers.test_object_create_bad_authorization_invalid_aws2: {status: KNOWN}
+  s3tests.functional.test_headers.test_object_create_bad_authorization_none: {status: KNOWN}
+  s3tests.functional.test_s3.test_100_continue: {status: KNOWN}
+  s3tests.functional.test_s3.test_atomic_conditional_write_1mb: {status: KNOWN}
+  s3tests.functional.test_s3.test_atomic_dual_conditional_write_1mb: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_email: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_email_notexist: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_grant_nonexist_user: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_acl_no_grants: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_create_exists: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_header_acl_grants: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_list_objects_anonymous: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_list_objects_anonymous_fail: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_recreate_not_overriding: {status: KNOWN}
+  s3tests.functional.test_s3.test_cors_origin_response: {status: KNOWN}
+  s3tests.functional.test_s3.test_cors_origin_wildcard: {status: KNOWN}
+  s3tests.functional.test_s3.test_list_buckets_anonymous: {status: KNOWN}
+  s3tests.functional.test_s3.test_list_buckets_invalid_auth: {status: KNOWN}
+  s3tests.functional.test_s3.test_logging_toggle: {status: KNOWN}
+  s3tests.functional.test_s3.test_multipart_resend_first_finishes_last: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_canned_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_replacing_metadata: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_header_acl_grants: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get_bucket_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get_bucket_gone: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get_object_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_get_object_gone: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_put: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_raw_put_write_access: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_anonymous_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_authenticated_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_authenticated_request_bad_access_key: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_case_insensitive_condition_fields: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_condition_is_case_sensitive: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_escaped_field_values: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_expired_policy: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_expires_is_case_sensitive: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_ignored_header: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_access_key: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_content_length_argument: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_date_format: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_request_field_value: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_invalid_signature: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_conditions_list: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_content_length_argument: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_expires_condition: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_policy_condition: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_missing_signature: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_no_key_specified: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_request_missing_policy_specified_field: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_set_invalid_success_code: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_set_key_from_filename: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_set_success_code: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_success_redirect_action: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_upload_larger_than_chunk: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_upload_size_below_minimum: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_upload_size_limit_exceeded: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_user_specified_header: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifmatch_failed: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifmatch_good: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifmatch_nonexisted_failed: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifnonmatch_failed: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifnonmatch_good: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed: {status: KNOWN}
+  s3tests.functional.test_s3.test_set_cors: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioned_concurrent_object_create_and_remove: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioned_concurrent_object_create_concurrent_remove: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioned_object_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_bucket_create_suspend: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_copy_obj_version: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_multi_object_delete: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_multi_object_delete_with_marker: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_multi_object_delete_with_marker_create: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_overwrite_multipart: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_read_remove: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_read_remove_head: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_versions_remove_all: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_create_versions_remove_special_names: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_list_marker: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_plain_null_version_overwrite: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_plain_null_version_overwrite_suspended: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_plain_null_version_removal: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_suspend_versions: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioning_obj_suspend_versions_simple: {status: KNOWN}
+  s3tests.functional.test_s3_website.check_can_test_website: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_bucket_private_redirectall_base: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_bucket_private_redirectall_path: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_bucket_private_redirectall_path_upgrade: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_nonexistant_bucket_rgw: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_nonexistant_bucket_s3: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_empty: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_empty_blockederrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_empty_gooderrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_empty_missingerrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_private_index: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_private_index_blockederrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_private_index_gooderrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_private_index_missingerrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_private_bucket_list_public_index: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_empty: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_empty_blockederrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_empty_gooderrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_empty_missingerrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_private_index: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_private_index_blockederrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_private_index_gooderrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_private_index_missingerrordoc: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_public_bucket_list_public_index: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_nonwebsite: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_private_abs: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_private_relative: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_public_abs: {status: KNOWN}
+  s3tests.functional.test_s3_website.test_website_xredirect_public_relative: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_configure_recreate: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_list_return_data_versioning: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy_acl: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy_another_bucket: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy_different_tenant: {status: KNOWN}
+  s3tests.functional.test_s3.test_bucket_policy_set_condition_operator_end_with_IfExists: {status: KNOWN}
+  s3tests.functional.test_s3.test_delete_tags_obj_public: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_invalid_md5: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_method_head: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_multipart_bad_download: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_multipart_invalid_chunks_1: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_multipart_invalid_chunks_2: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_no_key: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_no_md5: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_other_key: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_post_object_authenticated_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_encryption_sse_c_present: {status: KNOWN}
+  s3tests.functional.test_s3.test_get_obj_head_tagging: {status: KNOWN}
+  s3tests.functional.test_s3.test_get_obj_tagging: {status: KNOWN}
+  s3tests.functional.test_s3.test_get_tags_acl_public: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_deletemarker_expiration: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_expiration: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_expiration_date: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_get: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_get_no_id: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_id_too_long: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_multipart_expiration: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_noncur_expiration: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_rules_conflicted: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_same_id: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_date: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_deletemarker: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_empty_filter: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_filter: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_multipart: {status: KNOWN}
+  s3tests.functional.test_s3.test_lifecycle_set_noncurrent: {status: KNOWN}
+  s3tests.functional.test_s3.test_multipart_copy_invalid_range: {status: KNOWN}
+  s3tests.functional.test_s3.test_multipart_copy_versioned: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_versioned_bucket: {status: KNOWN}
+  s3tests.functional.test_s3.test_object_copy_versioning_multipart_upload: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_empty_conditions: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_tags_anonymous_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_post_object_tags_authenticated_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_delete_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_excess_key_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_excess_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_excess_val_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_max_kvsize_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_max_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_modify_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_obj_with_tags: {status: KNOWN}
+  s3tests.functional.test_s3.test_put_tags_acl_public: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_method_head: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_multipart_invalid_chunks_1: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_multipart_invalid_chunks_2: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_multipart_upload: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_post_object_authenticated_request: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_present: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_read_declare: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_transfer_13b: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_transfer_1MB: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_transfer_1b: {status: KNOWN}
+  s3tests.functional.test_s3.test_sse_kms_transfer_1kb: {status: KNOWN}
+  s3tests.functional.test_s3.test_versioned_object_acl_no_version_specified: {status: KNOWN}

doc/s3api/conf/ceph-s3.conf.in

@@ -0,0 +1,18 @@
+[DEFAULT]
+host = localhost
+port = 8080
+is_secure = no
+
+[s3 main]
+user_id = %ADMIN_ACCESS_KEY%
+display_name = %ADMIN_ACCESS_KEY%
+email = %ADMIN_ACCESS_KEY%
+access_key = %ADMIN_ACCESS_KEY%
+secret_key = %ADMIN_SECRET_KEY%
+
+[s3 alt]
+user_id = %TESTER_ACCESS_KEY%
+display_name = %TESTER_ACCESS_KEY%
+email = %TESTER_ACCESS_KEY%
+access_key = %TESTER_ACCESS_KEY%
+secret_key = %TESTER_SECRET_KEY%

doc/s3api/conf/object-server.conf.in

@@ -0,0 +1,17 @@
+[DEFAULT]
+user = %USER%
+bind_port = 6000
+swift_dir = %TEST_DIR%/etc
+devices = %TEST_DIR%
+mount_check = false
+workers = 1
+log_level = DEBUG
+
+[pipeline:main]
+pipeline = object-server
+
+[app:object-server]
+use = egg:swift#object
+allowed_headers = Cache-Control, Content-Disposition, Content-Encoding,
+ Content-Language, Expires, X-Delete-At, X-Object-Manifest, X-Robots-Tag,
+ X-Static-Large-Object

doc/s3api/rnc/access_control_policy.rnc

@@ -0,0 +1,7 @@
+include "common.rnc"
+
+start =
+  element AccessControlPolicy {
+    element Owner { CanonicalUser } &
+    element AccessControlList { AccessControlList }
+  }

doc/s3api/rnc/bucket_logging_status.rnc

@@ -0,0 +1,10 @@
+include "common.rnc"
+
+start =
+  element BucketLoggingStatus {
+    element LoggingEnabled {
+      element TargetBucket { xsd:string } &
+      element TargetPrefix { xsd:string } &
+      element TargetGrants { AccessControlList }?
+    }?
+  }

doc/s3api/rnc/common.rnc

@@ -0,0 +1,26 @@
+namespace xsi = "http://www.w3.org/2001/XMLSchema-instance"
+
+CanonicalUser =
+  element ID { xsd:string } &
+  element DisplayName { xsd:string }?
+
+StorageClass = "STANDARD" | "REDUCED_REDUNDANCY" | "GLACIER" | "UNKNOWN"
+
+AccessControlList =
+  element Grant {
+    element Grantee {
+      (
+        attribute xsi:type { "AmazonCustomerByEmail" },
+        element EmailAddress { xsd:string }
+      ) | (
+        attribute xsi:type { "CanonicalUser" },
+        CanonicalUser
+      ) | (
+        attribute xsi:type { "Group" },
+        element URI { xsd:string }
+      )
+    } &
+    element Permission {
+      "READ" | "WRITE" | "READ_ACP" | "WRITE_ACP" | "FULL_CONTROL"
+    }
+  }*

doc/s3api/rnc/complete_multipart_upload.rnc

@@ -0,0 +1,7 @@
+start =
+  element CompleteMultipartUpload {
+    element Part {
+      element PartNumber { xsd:int } &
+      element ETag { xsd:string }
+    }+
+  }

doc/s3api/rnc/complete_multipart_upload_result.rnc

@@ -0,0 +1,7 @@
+start =
+  element CompleteMultipartUploadResult {
+    element Location { xsd:anyURI },
+    element Bucket { xsd:string },
+    element Key { xsd:string },
+    element ETag { xsd:string }
+  }

doc/s3api/rnc/copy_object_result.rnc

@@ -0,0 +1,5 @@
+start =
+  element CopyObjectResult {
+    element LastModified { xsd:dateTime },
+    element ETag { xsd:string }
+  }

doc/s3api/rnc/copy_part_result.rnc

@@ -0,0 +1,5 @@
+start =
+  element CopyPartResult {
+    element LastModified { xsd:dateTime },
+    element ETag { xsd:string }
+  }

doc/s3api/rnc/create_bucket_configuration.rnc

@@ -0,0 +1,4 @@
+start =
+  element * {
+    element LocationConstraint { xsd:string }
+  }

doc/s3api/rnc/delete.rnc

@@ -0,0 +1,8 @@
+start =
+  element Delete {
+    element Quiet { xsd:boolean }? &
+    element Object {
+      element Key { xsd:string } &
+      element VersionId { xsd:string }?
+    }+
+  }

doc/s3api/rnc/delete_result.rnc

@@ -0,0 +1,17 @@
+start =
+  element DeleteResult {
+    (
+      element Deleted {
+        element Key { xsd:string },
+        element VersionId { xsd:string }?,
+        element DeleteMarker { xsd:boolean }?,
+        element DeleteMarkerVersionId { xsd:string }?
+      } |
+      element Error {
+        element Key { xsd:string },
+        element VersionId { xsd:string }?,
+        element Code { xsd:string },
+        element Message { xsd:string }
+      }
+    )*
+  }

doc/s3api/rnc/error.rnc

@@ -0,0 +1,11 @@
+start =
+  element Error {
+    element Code { xsd:string },
+    element Message { xsd:string },
+    DebugInfo*
+  }
+
+DebugInfo =
+  element * {
+    (attribute * { text } | text | DebugInfo)*
+  }

doc/s3api/rnc/initiate_multipart_upload_result.rnc

@@ -0,0 +1,6 @@
+start =
+  element InitiateMultipartUploadResult {
+    element Bucket { xsd:string },
+    element Key { xsd:string },
+    element UploadId { xsd:string }
+  }

doc/s3api/rnc/lifecycle_configuration.rnc

@@ -0,0 +1,20 @@
+include "common.rnc"
+
+start =
+  element LifecycleConfiguration {
+    element Rule {
+      element ID { xsd:string }? &
+      element Prefix { xsd:string } &
+      element Status { "Enabled" | "Disabled" } &
+      element Transition { Transition }? &
+      element Expiration { Expiration }?
+    }+
+  }
+
+Expiration =
+  element Days { xsd:int } |
+  element Date { xsd:dateTime }
+
+Transition =
+  Expiration &
+  element StorageClass { StorageClass }

doc/s3api/rnc/list_all_my_buckets_result.rnc

@@ -0,0 +1,12 @@
+include "common.rnc"
+
+start =
+  element ListAllMyBucketsResult {
+    element Owner { CanonicalUser },
+    element Buckets {
+      element Bucket {
+        element Name { xsd:string },
+        element CreationDate { xsd:dateTime }
+      }*
+    }
+  }

doc/s3api/rnc/list_bucket_result.rnc

@@ -0,0 +1,33 @@
+include "common.rnc"
+
+start =
+  element ListBucketResult {
+    element Name { xsd:string },
+    element Prefix { xsd:string },
+    (
+      (
+        element Marker { xsd:string },
+        element NextMarker { xsd:string }?
+      ) | (
+        element NextContinuationToken { xsd:string }?,
+        element ContinuationToken { xsd:string }?,
+        element StartAfter { xsd:string }?,
+        element KeyCount { xsd:int }
+      )
+    ),
+    element MaxKeys { xsd:int },
+    element EncodingType { xsd:string }?,
+    element Delimiter { xsd:string }?,
+    element IsTruncated { xsd:boolean },
+    element Contents {
+      element Key { xsd:string },
+      element LastModified { xsd:dateTime },
+      element ETag { xsd:string },
+      element Size { xsd:long },
+      element Owner { CanonicalUser }?,
+      element StorageClass { StorageClass }
+    }*,
+    element CommonPrefixes {
+      element Prefix { xsd:string }
+    }*
+  }

doc/s3api/rnc/list_multipart_uploads_result.rnc

@@ -0,0 +1,26 @@
+include "common.rnc"
+
+start =
+  element ListMultipartUploadsResult {
+    element Bucket { xsd:string },
+    element KeyMarker { xsd:string },
+    element UploadIdMarker { xsd:string },
+    element NextKeyMarker { xsd:string },
+    element NextUploadIdMarker { xsd:string },
+    element Delimiter { xsd:string }?,
+    element Prefix { xsd:string }?,
+    element MaxUploads { xsd:int },
+    element EncodingType { xsd:string }?,
+    element IsTruncated { xsd:boolean },
+    element Upload {
+      element Key { xsd:string },
+      element UploadId { xsd:string },
+      element Initiator { CanonicalUser },
+      element Owner { CanonicalUser },
+      element StorageClass { StorageClass },
+      element Initiated { xsd:dateTime }
+    }*,
+    element CommonPrefixes {
+      element Prefix { xsd:string }
+    }*
+  }

doc/s3api/rnc/list_parts_result.rnc

@@ -0,0 +1,22 @@
+include "common.rnc"
+
+start =
+    element ListPartsResult {
+    element Bucket { xsd:string },
+    element Key { xsd:string },
+    element UploadId { xsd:string },
+    element Initiator { CanonicalUser },
+    element Owner { CanonicalUser },
+    element StorageClass { StorageClass },
+    element PartNumberMarker { xsd:int },
+    element NextPartNumberMarker { xsd:int },
+    element MaxParts { xsd:int },
+    element EncodingType { xsd:string }?,
+    element IsTruncated { xsd:boolean },
+    element Part {
+      element PartNumber { xsd:int },
+      element LastModified { xsd:dateTime },
+      element ETag { xsd:string },
+      element Size { xsd:long }
+    }*
+  }

doc/s3api/rnc/list_versions_result.rnc

@@ -0,0 +1,37 @@
+include "common.rnc"
+
+start =
+  element ListVersionsResult {
+    element Name { xsd:string },
+    element Prefix { xsd:string },
+    element KeyMarker { xsd:string },
+    element VersionIdMarker { xsd:string },
+    element NextKeyMarker { xsd:string }?,
+    element NextVersionIdMarker { xsd:string }?,
+    element MaxKeys { xsd:int },
+    element EncodingType { xsd:string }?,
+    element Delimiter { xsd:string }?,
+    element IsTruncated { xsd:boolean },
+    (
+      element Version {
+        element Key { xsd:string },
+        element VersionId { xsd:string },
+        element IsLatest { xsd:boolean },
+        element LastModified { xsd:dateTime },
+        element ETag { xsd:string },
+        element Size { xsd:long },
+        element Owner { CanonicalUser }?,
+        element StorageClass { StorageClass }
+      } |
+      element DeleteMarker {
+        element Key { xsd:string },
+        element VersionId { xsd:string },
+        element IsLatest { xsd:boolean },
+        element LastModified { xsd:dateTime },
+        element Owner { CanonicalUser }?
+      }
+    )*,
+    element CommonPrefixes {
+      element Prefix { xsd:string }
+    }*
+  }

doc/s3api/rnc/location_constraint.rnc

@@ -0,0 +1 @@
+start = element LocationConstraint { xsd:string }

doc/s3api/rnc/versioning_configuration.rnc

@@ -0,0 +1,5 @@
+start =
+  element VersioningConfiguration {
+    element Status { "Enabled" | "Suspended" }? &
+    element MfaDelete { "Enabled" | "Disabled" }?
+  }

doc/source/associated_projects.rst

@@ -108,7 +108,6 @@ Alternative API
 
 * `ProxyFS <https://github.com/swiftstack/ProxyFS>`_ - Integrated file and
   object access for Swift object storage
-* `Swift3 <https://github.com/openstack/swift3>`_ - Amazon S3 API emulation.
 * `SwiftHLM <https://github.com/ibm-research/SwiftHLM>`_ - a middleware for
   using OpenStack Swift with tape and other high latency media storage
   backends.
@@ -177,3 +176,4 @@ Other
   web browser
 * `swiftbackmeup <https://github.com/redhat-cip/swiftbackmeup>`_ -
   Utility that allows one to create backups and upload them to OpenStack Swift
+* `s3compat <https://github.com/swiftstack/s3compat>`_ - S3 API compatibility checker

doc/source/middleware.rst

@@ -11,6 +11,95 @@ Account Quotas
     :members:
     :show-inheritance:
 
+.. _s3api:
+
+AWS S3 Api
+==========
+
+.. automodule:: swift.common.middleware.s3api.s3api
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.s3token
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.s3request
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.s3response
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.exception
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.etree
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.utils
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.subresource
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.acl_handlers
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.acl_utils
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.base
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.service
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.bucket
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.obj
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.acl
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.s3_acl
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.multi_upload
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.multi_delete
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.versioning
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.location
+    :members:
+    :show-inheritance:
+
+.. automodule:: swift.common.middleware.s3api.controllers.logging
+    :members:
+    :show-inheritance:
+
 .. _bulk:
 
 Bulk Operations (Delete and Archive Auto Extraction)

etc/object-server.conf-sample

@@ -120,6 +120,8 @@ use = egg:swift#object
 # Comma separated list of headers that can be set in metadata on an object.
 # This list is in addition to X-Object-Meta-* headers and cannot include
 # Content-Type, etag, Content-Length, or deleted
+# Note that you may add some extra headers for better S3 compatibility, they are:
+# Cache-Control, Content-Language, Expires, and X-Robots-Tag
 # allowed_headers = Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object
 #
 # auto_create_account_prefix = .

etc/proxy-server.conf-sample

@@ -442,6 +442,145 @@ user_test5_tester5 = testing5 service
 # in ACLs by setting allow_names_in_acls to false:
 # allow_names_in_acls = true
 
+[filter:s3api]
+use = egg:swift#s3api
+
+# s3api setup:
+#
+# With either tempauth or your custom auth:
+# - Put s3api just before your auth filter(s) in the pipeline
+# With keystone:
+# - Put s3api and s3token before keystoneauth in the pipeline
+#
+# Swift has no concept of the S3's resource owner; the resources
+# (i.e. containers and objects) created via the Swift API have no owner
+# information. This option specifies how the s3api middleware handles them
+# with the S3 API.  If this option is 'false', such kinds of resources will be
+# invisible and no users can access them with the S3 API.  If set to 'true',
+# a resource without an owner belongs to everyone and everyone can access it
+# with the S3 API.  If you care about S3 compatibility, set 'false' here.  This
+# option makes sense only when the s3_acl option is set to 'true' and your
+# Swift cluster has the resources created via the Swift API.
+# allow_no_owner = false
+#
+# Set a region name of your Swift cluster.  Note that the s3api doesn't choose
+# a region of the newly created bucket.  This value is used for the
+# GET Bucket location API and v4 signatures calculation.
+# location = US
+#
+# Set whether to enforce DNS-compliant bucket names. Note that S3 enforces
+# these conventions in all regions except the US Standard region.
+# dns_compliant_bucket_names = True
+#
+# Set the default maximum number of objects returned in the GET Bucket
+# response.
+# max_bucket_listing = 1000
+#
+# Set the maximum number of parts returned in the List Parts operation.
+# (default: 1000 as well as S3 specification)
+# If setting it larger than 10000 (swift container_listing_limit default)
+# make sure you also increase the container_listing_limit in swift.conf.
+# max_parts_listing = 1000
+#
+# Set the maximum number of objects we can delete with the Multi-Object Delete
+# operation.
+# max_multi_delete_objects = 1000
+#
+# If set to 'true', s3api uses its own metadata for ACLs
+# (e.g. X-Container-Sysmeta-S3Api-Acl) to achieve the best S3 compatibility.
+# If set to 'false', s3api tries to use Swift ACLs (e.g. X-Container-Read)
+# instead of S3 ACLs as far as possible.
+# There are some caveats that one should know about this setting. Firstly,
+# if set to 'false' after being previously set to 'true' any new objects or
+# containers stored while 'true' setting will be accessible to all users
+# because the s3 ACLs will be ignored under s3_acl=False setting. Secondly,
+# s3_acl True mode don't keep ACL consistency between both the S3 and Swift
+# API. Meaning with s3_acl enabled S3 ACLs only effect objects and buckets
+# via the S3 API. As this ACL information wont be available via the Swift API
+# and so the ACL wont be applied.
+# Note that s3_acl currently supports only keystone and tempauth.
+# DON'T USE THIS for production before enough testing for your use cases.
+# This stuff is still under development and it might cause something
+# you don't expect.
+# s3_acl = false
+#
+# Specify a host name of your Swift cluster.  This enables virtual-hosted style
+# requests.
+# storage_domain =
+#
+# Enable pipeline order check for SLO, s3token, authtoken, keystoneauth
+# according to standard s3api/Swift construction using either tempauth or
+# keystoneauth. If the order is incorrect, it raises an exception to stop
+# proxy. Turn auth_pipeline_check off only when you want to bypass these
+# authenticate middlewares in order to use other 3rd party (or your
+# proprietary) authenticate middleware.
+# auth_pipeline_check = True
+#
+# Enable multi-part uploads. (default: true)
+# This is required to store files larger than Swift's max_file_size (by
+# default, 5GiB). Note that has performance implications when deleting objects,
+# as we now have to check for whether there are also segments to delete.
+# allow_multipart_uploads = True
+#
+# Set the maximum number of parts for Upload Part operation.(default: 1000)
+# When setting it to be larger than the default value in order to match the
+# specification of S3, set to be larger max_manifest_segments for slo
+# middleware.(specification of S3: 10000)
+# max_upload_part_num = 1000
+#
+# Enable returning only buckets which owner are the user who requested
+# GET Service operation. (default: false)
+# If you want to enable the above feature, set this and s3_acl to true.
+# That might cause significant performance degradation. So, only if your
+# service absolutely need this feature, set this setting to true.
+# If you set this to false, s3api returns all buckets.
+# check_bucket_owner = false
+#
+# By default, Swift reports only S3 style access log.
+# (e.g. PUT /bucket/object) If set force_swift_request_proxy_log
+# to be 'true', Swift will become to output Swift style log
+# (e.g. PUT /v1/account/container/object) in addition to S3 style log.
+# Note that they will be reported twice (i.e. s3api doesn't care about
+# the duplication) and Swift style log will includes also various subrequests
+# to achieve S3 compatibilities when force_swift_request_proxy_log is set to
+# 'true'
+# force_swift_request_proxy_log = false
+#
+# AWS S3 document says that each part must be at least 5 MB in a multipart
+# upload, except the last part.
+# min_segment_size = 5242880
+
+# You can override the default log routing for this filter here:
+# log_name = s3api
+
+[filter:s3token]
+# s3token middleware authenticates with keystone using the s3 credentials
+# provided in the request header. Please put s3token between s3api
+# and keystoneauth if you're using keystoneauth.
+use = egg:swift#s3token
+
+# Prefix that will be prepended to the tenant to form the account
+reseller_prefix = AUTH_
+
+# By default, s3token will reject all invalid S3-style requests. Set this to
+# True to delegate that decision to downstream WSGI components. This may be
+# useful if there are multiple auth systems in the proxy pipeline.
+delay_auth_decision = False
+
+# Keystone server details
+auth_uri = http://keystonehost:35357/v3
+
+# Connect/read timeout to use when communicating with Keystone
+http_timeout = 10.0
+
+# SSL-related options
+# insecure = False
+# certfile =
+# keyfile =
+
+# You can override the default log routing for this filter here:
+# log_name = s3token
+
 [filter:healthcheck]
 use = egg:swift#healthcheck
 # An optional filesystem path, which if present, will cause the healthcheck

requirements.txt

@@ -7,6 +7,8 @@ eventlet>=0.17.4  # MIT
 greenlet>=0.3.1
 netifaces>=0.5,!=0.10.0,!=0.10.1
 PasteDeploy>=1.3.3
+lxml
+requests>=2.14.2 # Apache-2.0
 six>=1.9.0
 xattr>=0.4
 PyECLib>=1.3.1                          # BSD

setup.cfg

@@ -110,6 +110,8 @@ paste.filter_factory =
     kms_keymaster = swift.common.middleware.crypto.kms_keymaster:filter_factory
     listing_formats = swift.common.middleware.listing_formats:filter_factory
     symlink = swift.common.middleware.symlink:filter_factory
+    s3api = swift.common.middleware.s3api.s3api:filter_factory
+    s3token = swift.common.middleware.s3api.s3token:filter_factory
 
 
 [egg_info]

swift/common/middleware/s3api/acl_handlers.py

@@ -0,0 +1,479 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+------------
+Acl Handlers
+------------
+
+Why do we need this
+^^^^^^^^^^^^^^^^^^^
+
+To make controller classes clean, we need these handlers.
+It is really useful for customizing acl checking algorithms for
+each controller.
+
+Basic Information
+^^^^^^^^^^^^^^^^^
+
+BaseAclHandler wraps basic Acl handling.
+(i.e. it will check acl from ACL_MAP by using HEAD)
+
+How to extend
+^^^^^^^^^^^^^
+
+Make a handler with the name of the controller.
+(e.g. BucketAclHandler is for BucketController)
+It consists of method(s) for actual S3 method on controllers as follows.
+
+Example::
+
+   class BucketAclHandler(BaseAclHandler):
+       def PUT:
+           << put acl handling algorithms here for PUT bucket >>
+
+.. note::
+  If the method DON'T need to recall _get_response in outside of
+  acl checking, the method have to return the response it needs at
+  the end of method.
+
+"""
+import sys
+
+from swift.common.middleware.s3api.subresource import ACL, Owner, encode_acl
+from swift.common.middleware.s3api.s3response import MissingSecurityHeader, \
+    MalformedACLError, UnexpectedContent
+from swift.common.middleware.s3api.etree import fromstring, XMLSyntaxError, \
+    DocumentInvalid
+from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX, \
+    sysmeta_header
+from contextlib import contextmanager
+
+
+def get_acl_handler(controller_name):
+    for base_klass in [BaseAclHandler, MultiUploadAclHandler]:
+        # pylint: disable-msg=E1101
+        for handler in base_klass.__subclasses__():
+            handler_suffix_len = len('AclHandler') \
+                if not handler.__name__ == 'S3AclHandler' else len('Handler')
+            if handler.__name__[:-handler_suffix_len] == controller_name:
+                return handler
+    return BaseAclHandler
+
+
+class BaseAclHandler(object):
+    """
+    BaseAclHandler: Handling ACL for basic requests mapped on ACL_MAP
+    """
+    def __init__(self, req, logger):
+        self.req = req
+        self.container = self.req.container_name
+        self.obj = self.req.object_name
+        self.method = req.environ['REQUEST_METHOD']
+        self.user_id = self.req.user_id
+        self.headers = self.req.headers
+        self.logger = logger
+
+    @contextmanager
+    def request_with(self, container=None, obj=None, headers=None):
+        try:
+            org_cont = self.container
+            org_obj = self.obj
+            org_headers = self.headers
+
+            self.container = container or org_cont
+            self.obj = obj or org_obj
+            self.headers = headers or org_headers
+            yield
+
+        finally:
+            self.container = org_cont
+            self.obj = org_obj
+            self.headers = org_headers
+
+    def handle_acl(self, app, method, container=None, obj=None, headers=None):
+        method = method or self.method
+
+        with self.request_with(container, obj, headers):
+            if hasattr(self, method):
+                return getattr(self, method)(app)
+            else:
+                return self._handle_acl(app, method)
+
+    def _handle_acl(self, app, sw_method, container=None, obj=None,
+                    permission=None, headers=None):
+        """
+        General acl handling method.
+        This method expects to call Request._get_response() in outside of
+        this method so that this method returns response only when sw_method
+        is HEAD.
+        """
+
+        container = self.container if container is None else container
+        obj = self.obj if obj is None else obj
+        sw_method = sw_method or self.req.environ['REQUEST_METHOD']
+        resource = 'object' if obj else 'container'
+        headers = self.headers if headers is None else headers
+
+        self.logger.debug(
+            'checking permission: %s %s %s %s' %
+            (container, obj, sw_method, dict(headers)))
+
+        if not container:
+            return
+
+        if not permission and (self.method, sw_method, resource) in ACL_MAP:
+            acl_check = ACL_MAP[(self.method, sw_method, resource)]
+            resource = acl_check.get('Resource') or resource
+            permission = acl_check['Permission']
+
+        if not permission:
+            self.logger.debug(
+                '%s %s %s %s' % (container, obj, sw_method, headers))
+            raise Exception('No permission to be checked exists')
+
+        if resource == 'object':
+            resp = self.req.get_acl_response(app, 'HEAD',
+                                             container, obj,
+                                             headers)
+            acl = resp.object_acl
+        elif resource == 'container':
+            resp = self.req.get_acl_response(app, 'HEAD',
+                                             container, '')
+            acl = resp.bucket_acl
+
+        try:
+            acl.check_permission(self.user_id, permission)
+        except Exception as e:
+            self.logger.debug(acl)
+            self.logger.debug('permission denined: %s %s %s' %
+                              (e, self.user_id, permission))
+            raise
+
+        if sw_method == 'HEAD':
+            return resp
+
+    def get_acl(self, headers, body, bucket_owner, object_owner=None):
+        """
+        Get ACL instance from S3 (e.g. x-amz-grant) headers or S3 acl xml body.
+        """
+        acl = ACL.from_headers(headers, bucket_owner, object_owner,
+                               as_private=False)
+
+        if acl is None:
+            # Get acl from request body if possible.
+            if not body:
+                raise MissingSecurityHeader(missing_header_name='x-amz-acl')
+            try:
+                elem = fromstring(body, ACL.root_tag)
+                acl = ACL.from_elem(
+                    elem, True, self.req.allow_no_owner)
+            except(XMLSyntaxError, DocumentInvalid):
+                raise MalformedACLError()
+            except Exception as e:
+                exc_type, exc_value, exc_traceback = sys.exc_info()
+                self.logger.error(e)
+                raise exc_type, exc_value, exc_traceback
+        else:
+            if body:
+                # Specifying grant with both header and xml is not allowed.
+                raise UnexpectedContent()
+
+        return acl
+
+
+class BucketAclHandler(BaseAclHandler):
+    """
+    BucketAclHandler: Handler for BucketController
+    """
+    def DELETE(self, app):
+        if self.container.endswith(MULTIUPLOAD_SUFFIX):
+            # anyways, delete multiupload container doesn't need acls
+            # because it depends on GET segment container result for
+            # cleanup
+            pass
+        else:
+            return self._handle_acl(app, 'DELETE')
+
+    def HEAD(self, app):
+        if self.method == 'DELETE':
+            return self._handle_acl(app, 'DELETE')
+        else:
+            return self._handle_acl(app, 'HEAD')
+
+    def GET(self, app):
+        if self.method == 'DELETE' and \
+                self.container.endswith(MULTIUPLOAD_SUFFIX):
+            pass
+        else:
+            return self._handle_acl(app, 'GET')
+
+    def PUT(self, app):
+        req_acl = ACL.from_headers(self.req.headers,
+                                   Owner(self.user_id, self.user_id))
+
+        # To avoid overwriting the existing bucket's ACL, we send PUT
+        # request first before setting the ACL to make sure that the target
+        # container does not exist.
+        self.req.get_acl_response(app, 'PUT')
+
+        # update metadata
+        self.req.bucket_acl = req_acl
+
+        # FIXME If this request is failed, there is a possibility that the
+        # bucket which has no ACL is left.
+        return self.req.get_acl_response(app, 'POST')
+
+
+class ObjectAclHandler(BaseAclHandler):
+    """
+    ObjectAclHandler: Handler for ObjectController
+    """
+    def HEAD(self, app):
+        # No check object permission needed at DELETE Object
+        if self.method != 'DELETE':
+            return self._handle_acl(app, 'HEAD')
+
+    def PUT(self, app):
+        b_resp = self._handle_acl(app, 'HEAD', obj='')
+        req_acl = ACL.from_headers(self.req.headers,
+                                   b_resp.bucket_acl.owner,
+                                   Owner(self.user_id, self.user_id))
+        self.req.object_acl = req_acl
+
+
+class S3AclHandler(BaseAclHandler):
+    """
+    S3AclHandler: Handler for S3AclController
+    """
+    def GET(self, app):
+        self._handle_acl(app, 'HEAD', permission='READ_ACP')
+
+    def PUT(self, app):
+        if self.req.is_object_request:
+            b_resp = self.req.get_acl_response(app, 'HEAD', obj='')
+            o_resp = self._handle_acl(app, 'HEAD', permission='WRITE_ACP')
+            req_acl = self.get_acl(self.req.headers,
+                                   self.req.xml(ACL.max_xml_length),
+                                   b_resp.bucket_acl.owner,
+                                   o_resp.object_acl.owner)
+
+            # Don't change the owner of the resource by PUT acl request.
+            o_resp.object_acl.check_owner(req_acl.owner.id)
+
+            for g in req_acl.grants:
+                self.logger.debug(
+                    'Grant %s %s permission on the object /%s/%s' %
+                    (g.grantee, g.permission, self.req.container_name,
+                     self.req.object_name))
+            self.req.object_acl = req_acl
+        else:
+            self._handle_acl(app, self.method)
+
+    def POST(self, app):
+        if self.req.is_bucket_request:
+            resp = self._handle_acl(app, 'HEAD', permission='WRITE_ACP')
+
+            req_acl = self.get_acl(self.req.headers,
+                                   self.req.xml(ACL.max_xml_length),
+                                   resp.bucket_acl.owner)
+
+            # Don't change the owner of the resource by PUT acl request.
+            resp.bucket_acl.check_owner(req_acl.owner.id)
+
+            for g in req_acl.grants:
+                self.logger.debug(
+                    'Grant %s %s permission on the bucket /%s' %
+                    (g.grantee, g.permission, self.req.container_name))
+            self.req.bucket_acl = req_acl
+        else:
+            self._handle_acl(app, self.method)
+
+
+class MultiObjectDeleteAclHandler(BaseAclHandler):
+    """
+    MultiObjectDeleteAclHandler: Handler for MultiObjectDeleteController
+    """
+    def HEAD(self, app):
+        # Only bucket write acl is required
+        if not self.obj:
+            return self._handle_acl(app, 'HEAD')
+
+    def DELETE(self, app):
+        # Only bucket write acl is required
+        pass
+
+
+class MultiUploadAclHandler(BaseAclHandler):
+    """
+    MultiUpload stuff requires acl checking just once for BASE container
+    so that MultiUploadAclHandler extends BaseAclHandler to check acl only
+    when the verb defined. We should define the verb as the first step to
+    request to backend Swift at incoming request.
+
+    Basic Rules:
+      - BASE container name is always w/o 'MULTIUPLOAD_SUFFIX'
+      - Any check timing is ok but we should check it as soon as possible.
+
+    ========== ====== ============= ==========
+    Controller Verb   CheckResource Permission
+    ========== ====== ============= ==========
+    Part       PUT    Container     WRITE
+    Uploads    GET    Container     READ
+    Uploads    POST   Container     WRITE
+    Upload     GET    Container     READ
+    Upload     DELETE Container     WRITE
+    Upload     POST   Container     WRITE
+    ========== ====== ============= ==========
+
+    """
+    def __init__(self, req, logger):
+        super(MultiUploadAclHandler, self).__init__(req, logger)
+        self.acl_checked = False
+
+    def handle_acl(self, app, method, container=None, obj=None, headers=None):
+        method = method or self.method
+        with self.request_with(container, obj, headers):
+            # MultiUpload stuffs don't need acl check basically.
+            if hasattr(self, method):
+                return getattr(self, method)(app)
+            else:
+                pass
+
+    def HEAD(self, app):
+        # For _check_upload_info
+        self._handle_acl(app, 'HEAD', self.container, '')
+
+
+class PartAclHandler(MultiUploadAclHandler):
+    """
+    PartAclHandler: Handler for PartController
+    """
+    def __init__(self, req, logger):
+        # pylint: disable-msg=E1003
+        super(MultiUploadAclHandler, self).__init__(req, logger)
+
+    def HEAD(self, app):
+        if self.container.endswith(MULTIUPLOAD_SUFFIX):
+            # For _check_upload_info
+            container = self.container[:-len(MULTIUPLOAD_SUFFIX)]
+            self._handle_acl(app, 'HEAD', container, '')
+        else:
+            # For check_copy_source
+            return self._handle_acl(app, 'HEAD', self.container, self.obj)
+
+
+class UploadsAclHandler(MultiUploadAclHandler):
+    """
+    UploadsAclHandler: Handler for UploadsController
+    """
+    def handle_acl(self, app, method, *args, **kwargs):
+        method = method or self.method
+        if hasattr(self, method):
+            return getattr(self, method)(app)
+        else:
+            pass
+
+    def GET(self, app):
+        # List Multipart Upload
+        self._handle_acl(app, 'GET', self.container, '')
+
+    def PUT(self, app):
+        if not self.acl_checked:
+            resp = self._handle_acl(app, 'HEAD', obj='')
+            req_acl = ACL.from_headers(self.req.headers,
+                                       resp.bucket_acl.owner,
+                                       Owner(self.user_id, self.user_id))
+            acl_headers = encode_acl('object', req_acl)
+            self.req.headers[sysmeta_header('object', 'tmpacl')] = \
+                acl_headers[sysmeta_header('object', 'acl')]
+            self.acl_checked = True
+
+
+class UploadAclHandler(MultiUploadAclHandler):
+    """
+    UploadAclHandler: Handler for UploadController
+    """
+    def handle_acl(self, app, method, *args, **kwargs):
+        method = method or self.method
+        if hasattr(self, method):
+            return getattr(self, method)(app)
+        else:
+            pass
+
+    def HEAD(self, app):
+        # FIXME: GET HEAD case conflicts with GET service
+        method = 'GET' if self.method == 'GET' else 'HEAD'
+        self._handle_acl(app, method, self.container, '')
+
+    def PUT(self, app):
+        container = self.req.container_name + MULTIUPLOAD_SUFFIX
+        obj = '%s/%s' % (self.obj, self.req.params['uploadId'])
+        resp = self.req._get_response(app, 'HEAD', container, obj)
+        self.req.headers[sysmeta_header('object', 'acl')] = \
+            resp.sysmeta_headers.get(sysmeta_header('object', 'tmpacl'))
+
+
+"""
+ACL_MAP =
+    {
+        ('<s3_method>', '<swift_method>', '<swift_resource>'):
+        {'Resource': '<check_resource>',
+         'Permission': '<check_permission>'},
+        ...
+    }
+
+s3_method: Method of S3 Request from user to s3api
+swift_method: Method of Swift Request from s3api to swift
+swift_resource: Resource of Swift Request from s3api to swift
+check_resource: <container/object>
+check_permission: <OWNER/READ/WRITE/READ_ACP/WRITE_ACP>
+"""
+ACL_MAP = {
+    # HEAD Bucket
+    ('HEAD', 'HEAD', 'container'):
+    {'Permission': 'READ'},
+    # GET Service
+    ('GET', 'HEAD', 'container'):
+    {'Permission': 'OWNER'},
+    # GET Bucket, List Parts, List Multipart Upload
+    ('GET', 'GET', 'container'):
+    {'Permission': 'READ'},
+    # PUT Object, PUT Object Copy
+    ('PUT', 'HEAD', 'container'):
+    {'Permission': 'WRITE'},
+    # DELETE Bucket
+    ('DELETE', 'DELETE', 'container'):
+    {'Permission': 'OWNER'},
+    # HEAD Object
+    ('HEAD', 'HEAD', 'object'):
+    {'Permission': 'READ'},
+    # GET Object
+    ('GET', 'GET', 'object'):
+    {'Permission': 'READ'},
+    # PUT Object Copy, Upload Part Copy
+    ('PUT', 'HEAD', 'object'):
+    {'Permission': 'READ'},
+    # Abort Multipart Upload
+    ('DELETE', 'HEAD', 'container'):
+    {'Permission': 'WRITE'},
+    # Delete Object
+    ('DELETE', 'DELETE', 'object'):
+    {'Resource': 'container',
+     'Permission': 'WRITE'},
+    # Complete Multipart Upload, DELETE Multiple Objects,
+    # Initiate Multipart Upload
+    ('POST', 'HEAD', 'container'):
+    {'Permission': 'WRITE'},
+}

swift/common/middleware/s3api/acl_utils.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.middleware.s3api.exception import ACLError
+from swift.common.middleware.s3api.etree import fromstring, XMLSyntaxError, \
+    DocumentInvalid, XMLNS_XSI
+from swift.common.middleware.s3api.s3response import S3NotImplemented, \
+    MalformedACLError, InvalidArgument
+
+
+def swift_acl_translate(acl, group='', user='', xml=False):
+    """
+    Takes an S3 style ACL and returns a list of header/value pairs that
+    implement that ACL in Swift, or "NotImplemented" if there isn't a way to do
+    that yet.
+    """
+    swift_acl = {}
+    swift_acl['public-read'] = [['X-Container-Read', '.r:*,.rlistings']]
+    # Swift does not support public write:
+    # https://answers.launchpad.net/swift/+question/169541
+    swift_acl['public-read-write'] = [['X-Container-Write', '.r:*'],
+                                      ['X-Container-Read',
+                                       '.r:*,.rlistings']]
+
+    # TODO: if there's a way to get group and user, this should work for
+    # private:
+    # swift_acl['private'] = \
+    #     [['HTTP_X_CONTAINER_WRITE',  group + ':' + user], \
+    #      ['HTTP_X_CONTAINER_READ', group + ':' + user]]
+    swift_acl['private'] = [['X-Container-Write', '.'],
+                            ['X-Container-Read', '.']]
+    if xml:
+        # We are working with XML and need to parse it
+        try:
+            elem = fromstring(acl, 'AccessControlPolicy')
+        except (XMLSyntaxError, DocumentInvalid):
+            raise MalformedACLError()
+        acl = 'unknown'
+        for grant in elem.findall('./AccessControlList/Grant'):
+            permission = grant.find('./Permission').text
+            grantee = grant.find('./Grantee').get('{%s}type' % XMLNS_XSI)
+            if permission == "FULL_CONTROL" and grantee == 'CanonicalUser' and\
+                    acl != 'public-read' and acl != 'public-read-write':
+                acl = 'private'
+            elif permission == "READ" and grantee == 'Group' and\
+                    acl != 'public-read-write':
+                acl = 'public-read'
+            elif permission == "WRITE" and grantee == 'Group':
+                acl = 'public-read-write'
+            else:
+                acl = 'unsupported'
+
+    if acl == 'authenticated-read':
+        raise S3NotImplemented()
+    elif acl not in swift_acl:
+        raise ACLError()
+
+    return swift_acl[acl]
+
+
+def handle_acl_header(req):
+    """
+    Handle the x-amz-acl header.
+    Note that this header currently used for only normal-acl
+    (not implemented) on s3acl.
+    TODO: add translation to swift acl like as x-container-read to s3acl
+    """
+
+    amz_acl = req.environ['HTTP_X_AMZ_ACL']
+    # Translate the Amazon ACL to something that can be
+    # implemented in Swift, 501 otherwise. Swift uses POST
+    # for ACLs, whereas S3 uses PUT.
+    del req.environ['HTTP_X_AMZ_ACL']
+    if req.query_string:
+        req.query_string = ''
+
+    try:
+        translated_acl = swift_acl_translate(amz_acl)
+    except ACLError:
+        raise InvalidArgument('x-amz-acl', amz_acl)
+
+    for header, acl in translated_acl:
+        req.headers[header] = acl

swift/common/middleware/s3api/controllers/__init__.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.middleware.s3api.controllers.base import Controller, \
+    UnsupportedController
+from swift.common.middleware.s3api.controllers.service import ServiceController
+from swift.common.middleware.s3api.controllers.bucket import BucketController
+from swift.common.middleware.s3api.controllers.obj import ObjectController
+
+from swift.common.middleware.s3api.controllers.acl import AclController
+from swift.common.middleware.s3api.controllers.s3_acl import S3AclController
+from swift.common.middleware.s3api.controllers.multi_delete import \
+    MultiObjectDeleteController
+from swift.common.middleware.s3api.controllers.multi_upload import \
+    UploadController, PartController, UploadsController
+from swift.common.middleware.s3api.controllers.location import \
+    LocationController
+from swift.common.middleware.s3api.controllers.logging import \
+    LoggingStatusController
+from swift.common.middleware.s3api.controllers.versioning import \
+    VersioningController
+
+__all__ = [
+    'Controller',
+    'ServiceController',
+    'BucketController',
+    'ObjectController',
+
+    'AclController',
+    'S3AclController',
+    'MultiObjectDeleteController',
+    'PartController',
+    'UploadsController',
+    'UploadController',
+    'LocationController',
+    'LoggingStatusController',
+    'VersioningController',
+
+    'UnsupportedController',
+]

swift/common/middleware/s3api/controllers/acl.py

@@ -0,0 +1,130 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.http import HTTP_OK
+from swift.common.middleware.acl import parse_acl, referrer_allowed
+from swift.common.utils import public
+
+from swift.common.middleware.s3api.exception import ACLError
+from swift.common.middleware.s3api.controllers.base import Controller
+from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
+    MalformedACLError, UnexpectedContent, MissingSecurityHeader
+from swift.common.middleware.s3api.etree import Element, SubElement, tostring
+from swift.common.middleware.s3api.acl_utils import swift_acl_translate, \
+    XMLNS_XSI
+
+
+MAX_ACL_BODY_SIZE = 200 * 1024
+
+
+def get_acl(account_name, headers):
+    """
+    Attempts to construct an S3 ACL based on what is found in the swift headers
+    """
+
+    elem = Element('AccessControlPolicy')
+    owner = SubElement(elem, 'Owner')
+    SubElement(owner, 'ID').text = account_name
+    SubElement(owner, 'DisplayName').text = account_name
+    access_control_list = SubElement(elem, 'AccessControlList')
+
+    # grant FULL_CONTROL to myself by default
+    grant = SubElement(access_control_list, 'Grant')
+    grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
+    grantee.set('{%s}type' % XMLNS_XSI, 'CanonicalUser')
+    SubElement(grantee, 'ID').text = account_name
+    SubElement(grantee, 'DisplayName').text = account_name
+    SubElement(grant, 'Permission').text = 'FULL_CONTROL'
+
+    referrers, _ = parse_acl(headers.get('x-container-read'))
+    if referrer_allowed('unknown', referrers):
+        # grant public-read access
+        grant = SubElement(access_control_list, 'Grant')
+        grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
+        grantee.set('{%s}type' % XMLNS_XSI, 'Group')
+        SubElement(grantee, 'URI').text = \
+            'http://acs.amazonaws.com/groups/global/AllUsers'
+        SubElement(grant, 'Permission').text = 'READ'
+
+    referrers, _ = parse_acl(headers.get('x-container-write'))
+    if referrer_allowed('unknown', referrers):
+        # grant public-write access
+        grant = SubElement(access_control_list, 'Grant')
+        grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
+        grantee.set('{%s}type' % XMLNS_XSI, 'Group')
+        SubElement(grantee, 'URI').text = \
+            'http://acs.amazonaws.com/groups/global/AllUsers'
+        SubElement(grant, 'Permission').text = 'WRITE'
+
+    body = tostring(elem)
+
+    return HTTPOk(body=body, content_type="text/plain")
+
+
+class AclController(Controller):
+    """
+    Handles the following APIs:
+
+    * GET Bucket acl
+    * PUT Bucket acl
+    * GET Object acl
+    * PUT Object acl
+
+    Those APIs are logged as ACL operations in the S3 server log.
+    """
+    @public
+    def GET(self, req):
+        """
+        Handles GET Bucket acl and GET Object acl.
+        """
+        resp = req.get_response(self.app, method='HEAD')
+
+        return get_acl(req.user_id, resp.headers)
+
+    @public
+    def PUT(self, req):
+        """
+        Handles PUT Bucket acl and PUT Object acl.
+        """
+        if req.is_object_request:
+            # Handle Object ACL
+            raise S3NotImplemented()
+        else:
+            # Handle Bucket ACL
+            xml = req.xml(MAX_ACL_BODY_SIZE)
+            if all(['HTTP_X_AMZ_ACL' in req.environ, xml]):
+                # S3 doesn't allow to give ACL with both ACL header and body.
+                raise UnexpectedContent()
+            elif not any(['HTTP_X_AMZ_ACL' in req.environ, xml]):
+                # Both canned ACL header and xml body are missing
+                raise MissingSecurityHeader(missing_header_name='x-amz-acl')
+            else:
+                # correct ACL exists in the request
+                if xml:
+                    # We very likely have an XML-based ACL request.
+                    # let's try to translate to the request header
+                    try:
+                        translated_acl = swift_acl_translate(xml, xml=True)
+                    except ACLError:
+                        raise MalformedACLError()
+
+                    for header, acl in translated_acl:
+                        req.headers[header] = acl
+
+            resp = req.get_response(self.app, 'POST')
+            resp.status = HTTP_OK
+            resp.headers.update({'Location': req.container_name})
+
+            return resp

swift/common/middleware/s3api/controllers/base.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+
+from swift.common.middleware.s3api.s3response import S3NotImplemented, \
+    InvalidRequest
+from swift.common.middleware.s3api.utils import camel_to_snake
+
+
+def bucket_operation(func=None, err_resp=None, err_msg=None):
+    """
+    A decorator to ensure that the request is a bucket operation.  If the
+    target resource is an object, this decorator updates the request by default
+    so that the controller handles it as a bucket operation.  If 'err_resp' is
+    specified, this raises it on error instead.
+    """
+    def _bucket_operation(func):
+        @functools.wraps(func)
+        def wrapped(self, req):
+            if not req.is_bucket_request:
+                if err_resp:
+                    raise err_resp(msg=err_msg)
+
+                self.logger.debug('A key is specified for bucket API.')
+                req.object_name = None
+
+            return func(self, req)
+
+        return wrapped
+
+    if func:
+        return _bucket_operation(func)
+    else:
+        return _bucket_operation
+
+
+def object_operation(func):
+    """
+    A decorator to ensure that the request is an object operation.  If the
+    target resource is not an object, this raises an error response.
+    """
+    @functools.wraps(func)
+    def wrapped(self, req):
+        if not req.is_object_request:
+            raise InvalidRequest('A key must be specified')
+
+        return func(self, req)
+
+    return wrapped
+
+
+def check_container_existence(func):
+    """
+    A decorator to ensure the container existence.
+    """
+    @functools.wraps(func)
+    def check_container(self, req):
+        req.get_container_info(self.app)
+        return func(self, req)
+
+    return check_container
+
+
+class Controller(object):
+    """
+    Base WSGI controller class for the middleware
+    """
+    def __init__(self, app, conf, logger, **kwargs):
+        self.app = app
+        self.conf = conf
+        self.logger = logger
+
+    @classmethod
+    def resource_type(cls):
+        """
+        Returns the target resource type of this controller.
+        """
+        name = cls.__name__[:-len('Controller')]
+        return camel_to_snake(name).upper()
+
+
+class UnsupportedController(Controller):
+    """
+    Handles unsupported requests.
+    """
+    def __init__(self, app, conf, logger, **kwargs):
+        raise S3NotImplemented('The requested resource is not implemented')

swift/common/middleware/s3api/controllers/bucket.py

@@ -0,0 +1,251 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from base64 import standard_b64encode as b64encode
+from base64 import standard_b64decode as b64decode
+
+from swift.common.http import HTTP_OK
+from swift.common.utils import json, public, config_true_value
+
+from swift.common.middleware.s3api.controllers.base import Controller
+from swift.common.middleware.s3api.etree import Element, SubElement, tostring, \
+    fromstring, XMLSyntaxError, DocumentInvalid
+from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
+    InvalidArgument, \
+    MalformedXML, InvalidLocationConstraint, NoSuchBucket, \
+    BucketNotEmpty, InternalError, ServiceUnavailable, NoSuchKey
+from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX
+
+MAX_PUT_BUCKET_BODY_SIZE = 10240
+
+
+class BucketController(Controller):
+    """
+    Handles bucket request.
+    """
+    def _delete_segments_bucket(self, req):
+        """
+        Before delete bucket, delete segments bucket if existing.
+        """
+        container = req.container_name + MULTIUPLOAD_SUFFIX
+        marker = ''
+        seg = ''
+
+        try:
+            resp = req.get_response(self.app, 'HEAD')
+            if int(resp.sw_headers['X-Container-Object-Count']) > 0:
+                raise BucketNotEmpty()
+            # FIXME: This extra HEAD saves unexpected segment deletion
+            # but if a complete multipart upload happen while cleanup
+            # segment container below, completed object may be missing its
+            # segments unfortunately. To be safer, it might be good
+            # to handle if the segments can be deleted for each object.
+        except NoSuchBucket:
+            pass
+
+        try:
+            while True:
+                # delete all segments
+                resp = req.get_response(self.app, 'GET', container,
+                                        query={'format': 'json',
+                                               'marker': marker})
+                segments = json.loads(resp.body)
+                for seg in segments:
+                    try:
+                        req.get_response(self.app, 'DELETE', container,
+                                         seg['name'])
+                    except NoSuchKey:
+                        pass
+                    except InternalError:
+                        raise ServiceUnavailable()
+                if segments:
+                    marker = seg['name']
+                else:
+                    break
+            req.get_response(self.app, 'DELETE', container)
+        except NoSuchBucket:
+            return
+        except (BucketNotEmpty, InternalError):
+            raise ServiceUnavailable()
+
+    @public
+    def HEAD(self, req):
+        """
+        Handle HEAD Bucket (Get Metadata) request
+        """
+        resp = req.get_response(self.app)
+
+        return HTTPOk(headers=resp.headers)
+
+    @public
+    def GET(self, req):
+        """
+        Handle GET Bucket (List Objects) request
+        """
+
+        max_keys = req.get_validated_param(
+            'max-keys', self.conf.max_bucket_listing)
+        # TODO: Separate max_bucket_listing and default_bucket_listing
+        tag_max_keys = max_keys
+        max_keys = min(max_keys, self.conf.max_bucket_listing)
+
+        encoding_type = req.params.get('encoding-type')
+        if encoding_type is not None and encoding_type != 'url':
+            err_msg = 'Invalid Encoding Method specified in Request'
+            raise InvalidArgument('encoding-type', encoding_type, err_msg)
+
+        query = {
+            'format': 'json',
+            'limit': max_keys + 1,
+        }
+        if 'marker' in req.params:
+            query.update({'marker': req.params['marker']})
+        if 'prefix' in req.params:
+            query.update({'prefix': req.params['prefix']})
+        if 'delimiter' in req.params:
+            query.update({'delimiter': req.params['delimiter']})
+
+        # GET Bucket (List Objects) Version 2 parameters
+        is_v2 = int(req.params.get('list-type', '1')) == 2
+        fetch_owner = False
+        if is_v2:
+            if 'start-after' in req.params:
+                query.update({'marker': req.params['start-after']})
+            # continuation-token overrides start-after
+            if 'continuation-token' in req.params:
+                decoded = b64decode(req.params['continuation-token'])
+                query.update({'marker': decoded})
+            if 'fetch-owner' in req.params:
+                fetch_owner = config_true_value(req.params['fetch-owner'])
+
+        resp = req.get_response(self.app, query=query)
+
+        objects = json.loads(resp.body)
+
+        elem = Element('ListBucketResult')
+        SubElement(elem, 'Name').text = req.container_name
+        SubElement(elem, 'Prefix').text = req.params.get('prefix')
+
+        # in order to judge that truncated is valid, check whether
+        # max_keys + 1 th element exists in swift.
+        is_truncated = max_keys > 0 and len(objects) > max_keys
+        objects = objects[:max_keys]
+
+        if not is_v2:
+            SubElement(elem, 'Marker').text = req.params.get('marker')
+            if is_truncated and 'delimiter' in req.params:
+                if 'name' in objects[-1]:
+                    SubElement(elem, 'NextMarker').text = \
+                        objects[-1]['name']
+                if 'subdir' in objects[-1]:
+                    SubElement(elem, 'NextMarker').text = \
+                        objects[-1]['subdir']
+        else:
+            if is_truncated:
+                if 'name' in objects[-1]:
+                    SubElement(elem, 'NextContinuationToken').text = \
+                        b64encode(objects[-1]['name'])
+                if 'subdir' in objects[-1]:
+                    SubElement(elem, 'NextContinuationToken').text = \
+                        b64encode(objects[-1]['subdir'])
+            if 'continuation-token' in req.params:
+                SubElement(elem, 'ContinuationToken').text = \
+                    req.params['continuation-token']
+            if 'start-after' in req.params:
+                SubElement(elem, 'StartAfter').text = \
+                    req.params['start-after']
+            SubElement(elem, 'KeyCount').text = str(len(objects))
+
+        SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
+
+        if 'delimiter' in req.params:
+            SubElement(elem, 'Delimiter').text = req.params['delimiter']
+
+        if encoding_type is not None:
+            SubElement(elem, 'EncodingType').text = encoding_type
+
+        SubElement(elem, 'IsTruncated').text = \
+            'true' if is_truncated else 'false'
+
+        for o in objects:
+            if 'subdir' not in o:
+                contents = SubElement(elem, 'Contents')
+                SubElement(contents, 'Key').text = o['name']
+                SubElement(contents, 'LastModified').text = \
+                    o['last_modified'][:-3] + 'Z'
+                SubElement(contents, 'ETag').text = '"%s"' % o['hash']
+                SubElement(contents, 'Size').text = str(o['bytes'])
+                if fetch_owner or not is_v2:
+                    owner = SubElement(contents, 'Owner')
+                    SubElement(owner, 'ID').text = req.user_id
+                    SubElement(owner, 'DisplayName').text = req.user_id
+                SubElement(contents, 'StorageClass').text = 'STANDARD'
+
+        for o in objects:
+            if 'subdir' in o:
+                common_prefixes = SubElement(elem, 'CommonPrefixes')
+                SubElement(common_prefixes, 'Prefix').text = o['subdir']
+
+        body = tostring(elem, encoding_type=encoding_type)
+
+        return HTTPOk(body=body, content_type='application/xml')
+
+    @public
+    def PUT(self, req):
+        """
+        Handle PUT Bucket request
+        """
+        xml = req.xml(MAX_PUT_BUCKET_BODY_SIZE)
+        if xml:
+            # check location
+            try:
+                elem = fromstring(
+                    xml, 'CreateBucketConfiguration', self.logger)
+                location = elem.find('./LocationConstraint').text
+            except (XMLSyntaxError, DocumentInvalid):
+                raise MalformedXML()
+            except Exception as e:
+                exc_type, exc_value, exc_traceback = sys.exc_info()
+                self.logger.error(e)
+                raise exc_type, exc_value, exc_traceback
+
+            if location != self.conf.location:
+                # s3api cannot support multiple regions currently.
+                raise InvalidLocationConstraint()
+
+        resp = req.get_response(self.app)
+
+        resp.status = HTTP_OK
+        resp.location = '/' + req.container_name
+
+        return resp
+
+    @public
+    def DELETE(self, req):
+        """
+        Handle DELETE Bucket request
+        """
+        if self.conf.allow_multipart_uploads:
+            self._delete_segments_bucket(req)
+        resp = req.get_response(self.app)
+        return resp
+
+    @public
+    def POST(self, req):
+        """
+        Handle POST Bucket request
+        """
+        raise S3NotImplemented()

swift/common/middleware/s3api/controllers/location.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.utils import public
+
+from swift.common.middleware.s3api.controllers.base import Controller, \
+    bucket_operation
+from swift.common.middleware.s3api.etree import Element, tostring
+from swift.common.middleware.s3api.s3response import HTTPOk
+
+
+class LocationController(Controller):
+    """
+    Handles GET Bucket location, which is logged as a LOCATION operation in the
+    S3 server log.
+    """
+    @public
+    @bucket_operation
+    def GET(self, req):
+        """
+        Handles GET Bucket location.
+        """
+        req.get_response(self.app, method='HEAD')
+
+        elem = Element('LocationConstraint')
+        if self.conf.location != 'US':
+            elem.text = self.conf.location
+        body = tostring(elem)
+
+        return HTTPOk(body=body, content_type='application/xml')

swift/common/middleware/s3api/controllers/logging.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.utils import public
+
+from swift.common.middleware.s3api.controllers.base import Controller, \
+    bucket_operation
+from swift.common.middleware.s3api.etree import Element, tostring
+from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
+    NoLoggingStatusForKey
+
+
+class LoggingStatusController(Controller):
+    """
+    Handles the following APIs:
+
+    * GET Bucket logging
+    * PUT Bucket logging
+
+    Those APIs are logged as LOGGING_STATUS operations in the S3 server log.
+    """
+    @public
+    @bucket_operation(err_resp=NoLoggingStatusForKey)
+    def GET(self, req):
+        """
+        Handles GET Bucket logging.
+        """
+        req.get_response(self.app, method='HEAD')
+
+        # logging disabled
+        elem = Element('BucketLoggingStatus')
+        body = tostring(elem)
+
+        return HTTPOk(body=body, content_type='application/xml')
+
+    @public
+    @bucket_operation(err_resp=NoLoggingStatusForKey)
+    def PUT(self, req):
+        """
+        Handles PUT Bucket logging.
+        """
+        raise S3NotImplemented()

swift/common/middleware/s3api/controllers/multi_delete.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from swift.common.utils import public
+
+from swift.common.middleware.s3api.controllers.base import Controller, \
+    bucket_operation
+from swift.common.middleware.s3api.etree import Element, SubElement, \
+    fromstring, tostring, XMLSyntaxError, DocumentInvalid
+from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
+    NoSuchKey, ErrorResponse, MalformedXML, UserKeyMustBeSpecified, \
+    AccessDenied, MissingRequestBodyError
+
+MAX_MULTI_DELETE_BODY_SIZE = 61365
+
+
+class MultiObjectDeleteController(Controller):
+    """
+    Handles Delete Multiple Objects, which is logged as a MULTI_OBJECT_DELETE
+    operation in the S3 server log.
+    """
+    def _gen_error_body(self, error, elem, delete_list):
+        for key, version in delete_list:
+            if version is not None:
+                # TODO: delete the specific version of the object
+                raise S3NotImplemented()
+
+            error_elem = SubElement(elem, 'Error')
+            SubElement(error_elem, 'Key').text = key
+            SubElement(error_elem, 'Code').text = error.__class__.__name__
+            SubElement(error_elem, 'Message').text = error._msg
+
+        return tostring(elem)
+
+    @public
+    @bucket_operation
+    def POST(self, req):
+        """
+        Handles Delete Multiple Objects.
+        """
+        def object_key_iter(elem):
+            for obj in elem.iterchildren('Object'):
+                key = obj.find('./Key').text
+                if not key:
+                    raise UserKeyMustBeSpecified()
+                version = obj.find('./VersionId')
+                if version is not None:
+                    version = version.text
+
+                yield key, version
+
+        try:
+            xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE)
+            if not xml:
+                raise MissingRequestBodyError()
+
+            req.check_md5(xml)
+            elem = fromstring(xml, 'Delete', self.logger)
+
+            quiet = elem.find('./Quiet')
+            if quiet is not None and quiet.text.lower() == 'true':
+                self.quiet = True
+            else:
+                self.quiet = False
+
+            delete_list = list(object_key_iter(elem))
+            if len(delete_list) > self.conf.max_multi_delete_objects:
+                raise MalformedXML()
+        except (XMLSyntaxError, DocumentInvalid):
+            raise MalformedXML()
+        except ErrorResponse:
+            raise
+        except Exception as e:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            self.logger.error(e)
+            raise exc_type, exc_value, exc_traceback
+
+        elem = Element('DeleteResult')
+
+        # check bucket existence
+        try:
+            req.get_response(self.app, 'HEAD')
+        except AccessDenied as error:
+            body = self._gen_error_body(error, elem, delete_list)
+            return HTTPOk(body=body)
+
+        for key, version in delete_list:
+            if version is not None:
+                # TODO: delete the specific version of the object
+                raise S3NotImplemented()
+
+            req.object_name = key
+
+            try:
+                query = req.gen_multipart_manifest_delete_query(self.app)
+                req.get_response(self.app, method='DELETE', query=query)
+            except NoSuchKey:
+                pass
+            except ErrorResponse as e:
+                error = SubElement(elem, 'Error')
+                SubElement(error, 'Key').text = key
+                SubElement(error, 'Code').text = e.__class__.__name__
+                SubElement(error, 'Message').text = e._msg
+                continue
+
+            if not self.quiet:
+                deleted = SubElement(elem, 'Deleted')
+                SubElement(deleted, 'Key').text = key
+
+        body = tostring(elem)
+
+        return HTTPOk(body=body)

swift/common/middleware/s3api/controllers/multi_upload.py

@@ -0,0 +1,671 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Implementation of S3 Multipart Upload.
+
+This module implements S3 Multipart Upload APIs with the Swift SLO feature.
+The following explains how S3api uses swift container and objects to store S3
+upload information:
+
+-----------------
+[bucket]+segments
+-----------------
+
+A container to store upload information. [bucket] is the original bucket
+where multipart upload is initiated.
+
+-----------------------------
+[bucket]+segments/[upload_id]
+-----------------------------
+
+A object of the ongoing upload id. The object is empty and used for
+checking the target upload status. If the object exists, it means that the
+upload is initiated but not either completed or aborted.
+
+-------------------------------------------
+[bucket]+segments/[upload_id]/[part_number]
+-------------------------------------------
+
+The last suffix is the part number under the upload id. When the client uploads
+the parts, they will be stored in the namespace with
+[bucket]+segments/[upload_id]/[part_number].
+
+Example listing result in the [bucket]+segments container::
+
+  [bucket]+segments/[upload_id1]  # upload id object for upload_id1
+  [bucket]+segments/[upload_id1]/1  # part object for upload_id1
+  [bucket]+segments/[upload_id1]/2  # part object for upload_id1
+  [bucket]+segments/[upload_id1]/3  # part object for upload_id1
+  [bucket]+segments/[upload_id2]  # upload id object for upload_id2
+  [bucket]+segments/[upload_id2]/1  # part object for upload_id2
+  [bucket]+segments/[upload_id2]/2  # part object for upload_id2
+     .
+     .
+
+Those part objects are directly used as segments of a Swift
+Static Large Object when the multipart upload is completed.
+
+"""
+
+import os
+import re
+import sys
+
+from swift.common.swob import Range
+from swift.common.utils import json, public
+from swift.common.db import utf8encode
+
+from six.moves.urllib.parse import urlparse  # pylint: disable=F0401
+
+from swift.common.middleware.s3api.controllers.base import Controller, \
+    bucket_operation, object_operation, check_container_existence
+from swift.common.middleware.s3api.s3response import InvalidArgument, \
+    ErrorResponse, MalformedXML, \
+    InvalidPart, BucketAlreadyExists, EntityTooSmall, InvalidPartOrder, \
+    InvalidRequest, HTTPOk, HTTPNoContent, NoSuchKey, NoSuchUpload, \
+    NoSuchBucket
+from swift.common.middleware.s3api.exception import BadSwiftRequest
+from swift.common.middleware.s3api.utils import unique_id, \
+    MULTIUPLOAD_SUFFIX, S3Timestamp, sysmeta_header
+from swift.common.middleware.s3api.etree import Element, SubElement, \
+    fromstring, tostring, XMLSyntaxError, DocumentInvalid
+
+DEFAULT_MAX_PARTS_LISTING = 1000
+DEFAULT_MAX_UPLOADS = 1000
+
+MAX_COMPLETE_UPLOAD_BODY_SIZE = 2048 * 1024
+
+
+def _get_upload_info(req, app, upload_id):
+
+    container = req.container_name + MULTIUPLOAD_SUFFIX
+    obj = '%s/%s' % (req.object_name, upload_id)
+
+    try:
+        return req.get_response(app, 'HEAD', container=container, obj=obj)
+    except NoSuchKey:
+        raise NoSuchUpload(upload_id=upload_id)
+
+
+def _check_upload_info(req, app, upload_id):
+
+    _get_upload_info(req, app, upload_id)
+
+
+class PartController(Controller):
+    """
+    Handles the following APIs:
+
+    * Upload Part
+    * Upload Part - Copy
+
+    Those APIs are logged as PART operations in the S3 server log.
+    """
+    @public
+    @object_operation
+    @check_container_existence
+    def PUT(self, req):
+        """
+        Handles Upload Part and Upload Part Copy.
+        """
+
+        if 'uploadId' not in req.params:
+            raise InvalidArgument('ResourceType', 'partNumber',
+                                  'Unexpected query string parameter')
+
+        try:
+            part_number = int(req.params['partNumber'])
+            if part_number < 1 or self.conf.max_upload_part_num < part_number:
+                raise Exception()
+        except Exception:
+            err_msg = 'Part number must be an integer between 1 and %d,' \
+                      ' inclusive' % self.conf.max_upload_part_num
+            raise InvalidArgument('partNumber', req.params['partNumber'],
+                                  err_msg)
+
+        upload_id = req.params['uploadId']
+        _check_upload_info(req, self.app, upload_id)
+
+        req.container_name += MULTIUPLOAD_SUFFIX
+        req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
+                                        part_number)
+
+        req_timestamp = S3Timestamp.now()
+        req.headers['X-Timestamp'] = req_timestamp.internal
+        source_resp = req.check_copy_source(self.app)
+        if 'X-Amz-Copy-Source' in req.headers and \
+                'X-Amz-Copy-Source-Range' in req.headers:
+            rng = req.headers['X-Amz-Copy-Source-Range']
+
+            header_valid = True
+            try:
+                rng_obj = Range(rng)
+                if len(rng_obj.ranges) != 1:
+                    header_valid = False
+            except ValueError:
+                header_valid = False
+            if not header_valid:
+                err_msg = ('The x-amz-copy-source-range value must be of the '
+                           'form bytes=first-last where first and last are '
+                           'the zero-based offsets of the first and last '
+                           'bytes to copy')
+                raise InvalidArgument('x-amz-source-range', rng, err_msg)
+
+            source_size = int(source_resp.headers['Content-Length'])
+            if not rng_obj.ranges_for_length(source_size):
+                err_msg = ('Range specified is not valid for source object '
+                           'of size: %s' % source_size)
+                raise InvalidArgument('x-amz-source-range', rng, err_msg)
+
+            req.headers['Range'] = rng
+            del req.headers['X-Amz-Copy-Source-Range']
+        resp = req.get_response(self.app)
+
+        if 'X-Amz-Copy-Source' in req.headers:
+            resp.append_copy_resp_body(req.controller_name,
+                                       req_timestamp.s3xmlformat)
+
+        resp.status = 200
+        return resp
+
+
+class UploadsController(Controller):
+    """
+    Handles the following APIs:
+
+    * List Multipart Uploads
+    * Initiate Multipart Upload
+
+    Those APIs are logged as UPLOADS operations in the S3 server log.
+    """
+    @public
+    @bucket_operation(err_resp=InvalidRequest,
+                      err_msg="Key is not expected for the GET method "
+                              "?uploads subresource")
+    @check_container_existence
+    def GET(self, req):
+        """
+        Handles List Multipart Uploads
+        """
+
+        def separate_uploads(uploads, prefix, delimiter):
+            """
+            separate_uploads will separate uploads into non_delimited_uploads
+            (a subset of uploads) and common_prefixes according to the
+            specified delimiter. non_delimited_uploads is a list of uploads
+            which exclude the delimiter. common_prefixes is a set of prefixes
+            prior to the specified delimiter. Note that the prefix in the
+            common_prefixes includes the delimiter itself.
+
+            i.e. if '/' delimiter specified and then the uploads is consists of
+            ['foo', 'foo/bar'], this function will return (['foo'], ['foo/']).
+
+            :param uploads: A list of uploads dictionary
+            :param prefix: A string of prefix reserved on the upload path.
+                           (i.e. the delimiter must be searched behind the
+                            prefix)
+            :param delimiter: A string of delimiter to split the path in each
+                              upload
+
+            :return (non_delimited_uploads, common_prefixes)
+            """
+            (prefix, delimiter) = \
+                utf8encode(prefix, delimiter)
+            non_delimited_uploads = []
+            common_prefixes = set()
+            for upload in uploads:
+                key = upload['key']
+                end = key.find(delimiter, len(prefix))
+                if end >= 0:
+                    common_prefix = key[:end + len(delimiter)]
+                    common_prefixes.add(common_prefix)
+                else:
+                    non_delimited_uploads.append(upload)
+            return non_delimited_uploads, sorted(common_prefixes)
+
+        encoding_type = req.params.get('encoding-type')
+        if encoding_type is not None and encoding_type != 'url':
+            err_msg = 'Invalid Encoding Method specified in Request'
+            raise InvalidArgument('encoding-type', encoding_type, err_msg)
+
+        keymarker = req.params.get('key-marker', '')
+        uploadid = req.params.get('upload-id-marker', '')
+        maxuploads = req.get_validated_param(
+            'max-uploads', DEFAULT_MAX_UPLOADS, DEFAULT_MAX_UPLOADS)
+
+        query = {
+            'format': 'json',
+            'limit': maxuploads + 1,
+        }
+
+        if uploadid and keymarker:
+            query.update({'marker': '%s/%s' % (keymarker, uploadid)})
+        elif keymarker:
+            query.update({'marker': '%s/~' % (keymarker)})
+        if 'prefix' in req.params:
+            query.update({'prefix': req.params['prefix']})
+
+        container = req.container_name + MULTIUPLOAD_SUFFIX
+        try:
+            resp = req.get_response(self.app, container=container, query=query)
+            objects = json.loads(resp.body)
+        except NoSuchBucket:
+            # Assume NoSuchBucket as no uploads
+            objects = []
+
+        def object_to_upload(object_info):
+            obj, upid = object_info['name'].rsplit('/', 1)
+            obj_dict = {'key': obj,
+                        'upload_id': upid,
+                        'last_modified': object_info['last_modified']}
+            return obj_dict
+
+        # uploads is a list consists of dict, {key, upload_id, last_modified}
+        # Note that pattern matcher will drop whole segments objects like as
+        # object_name/upload_id/1.
+        pattern = re.compile('/[0-9]+$')
+        uploads = [object_to_upload(obj) for obj in objects if
+                   pattern.search(obj.get('name', '')) is None]
+
+        prefixes = []
+        if 'delimiter' in req.params:
+            prefix = req.params.get('prefix', '')
+            delimiter = req.params['delimiter']
+            uploads, prefixes = \
+                separate_uploads(uploads, prefix, delimiter)
+
+        if len(uploads) > maxuploads:
+            uploads = uploads[:maxuploads]
+            truncated = True
+        else:
+            truncated = False
+
+        nextkeymarker = ''
+        nextuploadmarker = ''
+        if len(uploads) > 1:
+            nextuploadmarker = uploads[-1]['upload_id']
+            nextkeymarker = uploads[-1]['key']
+
+        result_elem = Element('ListMultipartUploadsResult')
+        SubElement(result_elem, 'Bucket').text = req.container_name
+        SubElement(result_elem, 'KeyMarker').text = keymarker
+        SubElement(result_elem, 'UploadIdMarker').text = uploadid
+        SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker
+        SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker
+        if 'delimiter' in req.params:
+            SubElement(result_elem, 'Delimiter').text = \
+                req.params['delimiter']
+        if 'prefix' in req.params:
+            SubElement(result_elem, 'Prefix').text = req.params['prefix']
+        SubElement(result_elem, 'MaxUploads').text = str(maxuploads)
+        if encoding_type is not None:
+            SubElement(result_elem, 'EncodingType').text = encoding_type
+        SubElement(result_elem, 'IsTruncated').text = \
+            'true' if truncated else 'false'
+
+        # TODO: don't show uploads which are initiated before this bucket is
+        # created.
+        for u in uploads:
+            upload_elem = SubElement(result_elem, 'Upload')
+            SubElement(upload_elem, 'Key').text = u['key']
+            SubElement(upload_elem, 'UploadId').text = u['upload_id']
+            initiator_elem = SubElement(upload_elem, 'Initiator')
+            SubElement(initiator_elem, 'ID').text = req.user_id
+            SubElement(initiator_elem, 'DisplayName').text = req.user_id
+            owner_elem = SubElement(upload_elem, 'Owner')
+            SubElement(owner_elem, 'ID').text = req.user_id
+            SubElement(owner_elem, 'DisplayName').text = req.user_id
+            SubElement(upload_elem, 'StorageClass').text = 'STANDARD'
+            SubElement(upload_elem, 'Initiated').text = \
+                u['last_modified'][:-3] + 'Z'
+
+        for p in prefixes:
+            elem = SubElement(result_elem, 'CommonPrefixes')
+            SubElement(elem, 'Prefix').text = p
+
+        body = tostring(result_elem, encoding_type=encoding_type)
+
+        return HTTPOk(body=body, content_type='application/xml')
+
+    @public
+    @object_operation
+    @check_container_existence
+    def POST(self, req):
+        """
+        Handles Initiate Multipart Upload.
+        """
+
+        # Create a unique S3 upload id from UUID to avoid duplicates.
+        upload_id = unique_id()
+
+        container = req.container_name + MULTIUPLOAD_SUFFIX
+        content_type = req.headers.get('Content-Type')
+        if content_type:
+            req.headers[sysmeta_header('object', 'has-content-type')] = 'yes'
+            req.headers[
+                sysmeta_header('object', 'content-type')] = content_type
+        else:
+            req.headers[sysmeta_header('object', 'has-content-type')] = 'no'
+        req.headers['Content-Type'] = 'application/directory'
+
+        try:
+            req.get_response(self.app, 'PUT', container, '')
+        except BucketAlreadyExists:
+            pass
+
+        obj = '%s/%s' % (req.object_name, upload_id)
+
+        req.headers.pop('Etag', None)
+        req.headers.pop('Content-Md5', None)
+
+        req.get_response(self.app, 'PUT', container, obj, body='')
+
+        result_elem = Element('InitiateMultipartUploadResult')
+        SubElement(result_elem, 'Bucket').text = req.container_name
+        SubElement(result_elem, 'Key').text = req.object_name
+        SubElement(result_elem, 'UploadId').text = upload_id
+
+        body = tostring(result_elem)
+
+        return HTTPOk(body=body, content_type='application/xml')
+
+
+class UploadController(Controller):
+    """
+    Handles the following APIs:
+
+    * List Parts
+    * Abort Multipart Upload
+    * Complete Multipart Upload
+
+    Those APIs are logged as UPLOAD operations in the S3 server log.
+    """
+    @public
+    @object_operation
+    @check_container_existence
+    def GET(self, req):
+        """
+        Handles List Parts.
+        """
+        def filter_part_num_marker(o):
+            try:
+                num = int(os.path.basename(o['name']))
+                return num > part_num_marker
+            except ValueError:
+                return False
+
+        encoding_type = req.params.get('encoding-type')
+        if encoding_type is not None and encoding_type != 'url':
+            err_msg = 'Invalid Encoding Method specified in Request'
+            raise InvalidArgument('encoding-type', encoding_type, err_msg)
+
+        upload_id = req.params['uploadId']
+        _check_upload_info(req, self.app, upload_id)
+
+        maxparts = req.get_validated_param(
+            'max-parts', DEFAULT_MAX_PARTS_LISTING,
+            self.conf.max_parts_listing)
+        part_num_marker = req.get_validated_param(
+            'part-number-marker', 0)
+
+        query = {
+            'format': 'json',
+            'limit': maxparts + 1,
+            'prefix': '%s/%s/' % (req.object_name, upload_id),
+            'delimiter': '/'
+        }
+
+        container = req.container_name + MULTIUPLOAD_SUFFIX
+        resp = req.get_response(self.app, container=container, obj='',
+                                query=query)
+        objects = json.loads(resp.body)
+
+        last_part = 0
+
+        # If the caller requested a list starting at a specific part number,
+        # construct a sub-set of the object list.
+        objList = filter(filter_part_num_marker, objects)
+
+        # pylint: disable-msg=E1103
+        objList.sort(key=lambda o: int(o['name'].split('/')[-1]))
+
+        if len(objList) > maxparts:
+            objList = objList[:maxparts]
+            truncated = True
+        else:
+            truncated = False
+        # TODO: We have to retrieve object list again when truncated is True
+        # and some objects filtered by invalid name because there could be no
+        # enough objects for limit defined by maxparts.
+
+        if objList:
+            o = objList[-1]
+            last_part = os.path.basename(o['name'])
+
+        result_elem = Element('ListPartsResult')
+        SubElement(result_elem, 'Bucket').text = req.container_name
+        SubElement(result_elem, 'Key').text = req.object_name
+        SubElement(result_elem, 'UploadId').text = upload_id
+
+        initiator_elem = SubElement(result_elem, 'Initiator')
+        SubElement(initiator_elem, 'ID').text = req.user_id
+        SubElement(initiator_elem, 'DisplayName').text = req.user_id
+        owner_elem = SubElement(result_elem, 'Owner')
+        SubElement(owner_elem, 'ID').text = req.user_id
+        SubElement(owner_elem, 'DisplayName').text = req.user_id
+
+        SubElement(result_elem, 'StorageClass').text = 'STANDARD'
+        SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker)
+        SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part)
+        SubElement(result_elem, 'MaxParts').text = str(maxparts)
+        if 'encoding-type' in req.params:
+            SubElement(result_elem, 'EncodingType').text = \
+                req.params['encoding-type']
+        SubElement(result_elem, 'IsTruncated').text = \
+            'true' if truncated else 'false'
+
+        for i in objList:
+            part_elem = SubElement(result_elem, 'Part')
+            SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1]
+            SubElement(part_elem, 'LastModified').text = \
+                i['last_modified'][:-3] + 'Z'
+            SubElement(part_elem, 'ETag').text = '"%s"' % i['hash']
+            SubElement(part_elem, 'Size').text = str(i['bytes'])
+
+        body = tostring(result_elem, encoding_type=encoding_type)
+
+        return HTTPOk(body=body, content_type='application/xml')
+
+    @public
+    @object_operation
+    @check_container_existence
+    def DELETE(self, req):
+        """
+        Handles Abort Multipart Upload.
+        """
+        upload_id = req.params['uploadId']
+        _check_upload_info(req, self.app, upload_id)
+
+        # First check to see if this multi-part upload was already
+        # completed.  Look in the primary container, if the object exists,
+        # then it was completed and we return an error here.
+        container = req.container_name + MULTIUPLOAD_SUFFIX
+        obj = '%s/%s' % (req.object_name, upload_id)
+        req.get_response(self.app, container=container, obj=obj)
+
+        # The completed object was not found so this
+        # must be a multipart upload abort.
+        # We must delete any uploaded segments for this UploadID and then
+        # delete the object in the main container as well
+        query = {
+            'format': 'json',
+            'prefix': '%s/%s/' % (req.object_name, upload_id),
+            'delimiter': '/',
+        }
+
+        resp = req.get_response(self.app, 'GET', container, '', query=query)
+
+        #  Iterate over the segment objects and delete them individually
+        objects = json.loads(resp.body)
+        for o in objects:
+            container = req.container_name + MULTIUPLOAD_SUFFIX
+            req.get_response(self.app, container=container, obj=o['name'])
+
+        return HTTPNoContent()
+
+    @public
+    @object_operation
+    @check_container_existence
+    def POST(self, req):
+        """
+        Handles Complete Multipart Upload.
+        """
+        upload_id = req.params['uploadId']
+        resp = _get_upload_info(req, self.app, upload_id)
+        headers = {}
+        for key, val in resp.headers.iteritems():
+            _key = key.lower()
+            if _key.startswith('x-amz-meta-'):
+                headers['x-object-meta-' + _key[11:]] = val
+
+        hct_header = sysmeta_header('object', 'has-content-type')
+        if resp.sysmeta_headers.get(hct_header) == 'yes':
+            content_type = resp.sysmeta_headers.get(
+                sysmeta_header('object', 'content-type'))
+        elif hct_header in resp.sysmeta_headers:
+            # has-content-type is present but false, so no content type was
+            # set on initial upload. In that case, we won't set one on our
+            # PUT request. Swift will end up guessing one based on the
+            # object name.
+            content_type = None
+        else:
+            content_type = resp.headers.get('Content-Type')
+
+        if content_type:
+            headers['Content-Type'] = content_type
+
+        # Query for the objects in the segments area to make sure it completed
+        query = {
+            'format': 'json',
+            'prefix': '%s/%s/' % (req.object_name, upload_id),
+            'delimiter': '/'
+        }
+
+        container = req.container_name + MULTIUPLOAD_SUFFIX
+        resp = req.get_response(self.app, 'GET', container, '', query=query)
+        objinfo = json.loads(resp.body)
+        objtable = dict((o['name'],
+                         {'path': '/'.join(['', container, o['name']]),
+                          'etag': o['hash'],
+                          'size_bytes': o['bytes']}) for o in objinfo)
+
+        manifest = []
+        previous_number = 0
+        try:
+            xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
+            if not xml:
+                raise InvalidRequest(msg='You must specify at least one part')
+
+            complete_elem = fromstring(
+                xml, 'CompleteMultipartUpload', self.logger)
+            for part_elem in complete_elem.iterchildren('Part'):
+                part_number = int(part_elem.find('./PartNumber').text)
+
+                if part_number <= previous_number:
+                    raise InvalidPartOrder(upload_id=upload_id)
+                previous_number = part_number
+
+                etag = part_elem.find('./ETag').text
+                if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
+                    # strip double quotes
+                    etag = etag[1:-1]
+
+                info = objtable.get("%s/%s/%s" % (req.object_name, upload_id,
+                                                  part_number))
+                if info is None or info['etag'] != etag:
+                    raise InvalidPart(upload_id=upload_id,
+                                      part_number=part_number)
+
+                info['size_bytes'] = int(info['size_bytes'])
+                manifest.append(info)
+        except (XMLSyntaxError, DocumentInvalid):
+            raise MalformedXML()
+        except ErrorResponse:
+            raise
+        except Exception as e:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            self.logger.error(e)
+            raise exc_type, exc_value, exc_traceback
+
+        # Check the size of each segment except the last and make sure they are
+        # all more than the minimum upload chunk size
+        for info in manifest[:-1]:
+            if info['size_bytes'] < self.conf.min_segment_size:
+                raise EntityTooSmall()
+
+        try:
+            # TODO: add support for versioning
+            if manifest:
+                resp = req.get_response(self.app, 'PUT',
+                                        body=json.dumps(manifest),
+                                        query={'multipart-manifest': 'put'},
+                                        headers=headers)
+            else:
+                # the upload must have consisted of a single zero-length part
+                # just write it directly
+                resp = req.get_response(self.app, 'PUT', body='',
+                                        headers=headers)
+        except BadSwiftRequest as e:
+            msg = str(e)
+            expected_msg = 'too small; each segment must be at least 1 byte'
+            if expected_msg in msg:
+                # FIXME: AWS S3 allows a smaller object than 5 MB if there is
+                # only one part.  Use a COPY request to copy the part object
+                # from the segments container instead.
+                raise EntityTooSmall(msg)
+            else:
+                raise
+
+        # clean up the multipart-upload record
+        obj = '%s/%s' % (req.object_name, upload_id)
+        try:
+            req.get_response(self.app, 'DELETE', container, obj)
+        except NoSuchKey:
+            pass  # We know that this existed long enough for us to HEAD
+
+        result_elem = Element('CompleteMultipartUploadResult')
+
+        # NOTE: boto with sig v4 appends port to HTTP_HOST value at the
+        # request header when the port is non default value and it makes
+        # req.host_url like as http://localhost:8080:8080/path
+        # that obviously invalid. Probably it should be resolved at
+        # swift.common.swob though, tentatively we are parsing and
+        # reconstructing the correct host_url info here.
+        # in detail, https://github.com/boto/boto/pull/3513
+        parsed_url = urlparse(req.host_url)
+        host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
+        if parsed_url.port:
+            host_url += ':%s' % parsed_url.port
+
+        SubElement(result_elem, 'Location').text = host_url + req.path
+        SubElement(result_elem, 'Bucket').text = req.container_name
+        SubElement(result_elem, 'Key').text = req.object_name
+        SubElement(result_elem, 'ETag').text = resp.etag
+
+        resp.body = tostring(result_elem)
+        resp.status = 200
+        resp.content_type = "application/xml"
+
+        return resp

swift/common/middleware/s3api/controllers/obj.py

@@ -0,0 +1,150 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from swift.common.http import HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_NO_CONTENT
+from swift.common.swob import Range, content_range_header_value
+from swift.common.utils import public
+
+from swift.common.middleware.s3api.utils import S3Timestamp
+from swift.common.middleware.s3api.controllers.base import Controller
+from swift.common.middleware.s3api.s3response import S3NotImplemented, \
+    InvalidRange, NoSuchKey, InvalidArgument
+
+
+class ObjectController(Controller):
+    """
+    Handles requests on objects
+    """
+    def _gen_head_range_resp(self, req_range, resp):
+        """
+        Swift doesn't handle Range header for HEAD requests.
+        So, this method generates HEAD range response from HEAD response.
+        S3 return HEAD range response, if the value of range satisfies the
+        conditions which are described in the following document.
+        - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
+        """
+        length = long(resp.headers.get('Content-Length'))
+
+        try:
+            content_range = Range(req_range)
+        except ValueError:
+            return resp
+
+        ranges = content_range.ranges_for_length(length)
+        if ranges == []:
+            raise InvalidRange()
+        elif ranges:
+            if len(ranges) == 1:
+                start, end = ranges[0]
+                resp.headers['Content-Range'] = \
+                    content_range_header_value(start, end, length)
+                resp.headers['Content-Length'] = (end - start)
+                resp.status = HTTP_PARTIAL_CONTENT
+                return resp
+            else:
+                # TODO: It is necessary to confirm whether need to respond to
+                #       multi-part response.(e.g. bytes=0-10,20-30)
+                pass
+
+        return resp
+
+    def GETorHEAD(self, req):
+        resp = req.get_response(self.app)
+
+        if req.method == 'HEAD':
+            resp.app_iter = None
+
+        for key in ('content-type', 'content-language', 'expires',
+                    'cache-control', 'content-disposition',
+                    'content-encoding'):
+            if 'response-' + key in req.params:
+                resp.headers[key] = req.params['response-' + key]
+
+        return resp
+
+    @public
+    def HEAD(self, req):
+        """
+        Handle HEAD Object request
+        """
+        resp = self.GETorHEAD(req)
+
+        if 'range' in req.headers:
+            req_range = req.headers['range']
+            resp = self._gen_head_range_resp(req_range, resp)
+
+        return resp
+
+    @public
+    def GET(self, req):
+        """
+        Handle GET Object request
+        """
+        return self.GETorHEAD(req)
+
+    @public
+    def PUT(self, req):
+        """
+        Handle PUT Object and PUT Object (Copy) request
+        """
+        # set X-Timestamp by s3api to use at copy resp body
+        req_timestamp = S3Timestamp.now()
+        req.headers['X-Timestamp'] = req_timestamp.internal
+        if all(h in req.headers
+               for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')):
+            raise InvalidArgument('x-amz-copy-source-range',
+                                  req.headers['X-Amz-Copy-Source-Range'],
+                                  'Illegal copy header')
+        req.check_copy_source(self.app)
+        resp = req.get_response(self.app)
+
+        if 'X-Amz-Copy-Source' in req.headers:
+            resp.append_copy_resp_body(req.controller_name,
+                                       req_timestamp.s3xmlformat)
+
+            # delete object metadata from response
+            for key in list(resp.headers.keys()):
+                if key.startswith('x-amz-meta-'):
+                    del resp.headers[key]
+
+        resp.status = HTTP_OK
+        return resp
+
+    @public
+    def POST(self, req):
+        raise S3NotImplemented()
+
+    @public
+    def DELETE(self, req):
+        """
+        Handle DELETE Object request
+        """
+        try:
+            query = req.gen_multipart_manifest_delete_query(self.app)
+            req.headers['Content-Type'] = None  # Ignore client content-type
+            resp = req.get_response(self.app, query=query)
+            if query and resp.status_int == HTTP_OK:
+                for chunk in resp.app_iter:
+                    pass  # drain the bulk-deleter response
+                resp.status = HTTP_NO_CONTENT
+                resp.body = ''
+        except NoSuchKey:
+            # expect to raise NoSuchBucket when the bucket doesn't exist
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            req.get_container_info(self.app)
+            raise exc_type, exc_value, exc_traceback
+        return resp

swift/common/middleware/s3api/controllers/s3_acl.py

@@ -0,0 +1,67 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from urllib import quote
+from swift.common.utils import public
+
+from swift.common.middleware.s3api.controllers.base import Controller
+from swift.common.middleware.s3api.s3response import HTTPOk
+from swift.common.middleware.s3api.etree import tostring
+
+
+class S3AclController(Controller):
+    """
+    Handles the following APIs:
+
+    * GET Bucket acl
+    * PUT Bucket acl
+    * GET Object acl
+    * PUT Object acl
+
+    Those APIs are logged as ACL operations in the S3 server log.
+    """
+    @public
+    def GET(self, req):
+        """
+        Handles GET Bucket acl and GET Object acl.
+        """
+        resp = req.get_response(self.app)
+
+        acl = resp.object_acl if req.is_object_request else resp.bucket_acl
+
+        resp = HTTPOk()
+        resp.body = tostring(acl.elem())
+
+        return resp
+
+    @public
+    def PUT(self, req):
+        """
+        Handles PUT Bucket acl and PUT Object acl.
+        """
+        if req.is_object_request:
+            headers = {}
+            src_path = '/%s/%s' % (req.container_name, req.object_name)
+
+            # object-sysmeta' can be updated by 'Copy' method,
+            # but can not be by 'POST' method.
+            # So headers['X-Copy-From'] for copy request is added here.
+            headers['X-Copy-From'] = quote(src_path)
+            headers['Content-Length'] = 0
+            req.get_response(self.app, 'PUT', headers=headers)
+        else:
+            req.get_response(self.app, 'POST')
+
+        return HTTPOk()

swift/common/middleware/s3api/controllers/service.py

@@ -0,0 +1,68 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.utils import json, public
+
+from swift.common.middleware.s3api.controllers.base import Controller
+from swift.common.middleware.s3api.etree import Element, SubElement, tostring
+from swift.common.middleware.s3api.s3response import HTTPOk, AccessDenied, \
+    NoSuchBucket
+from swift.common.middleware.s3api.utils import validate_bucket_name
+
+
+class ServiceController(Controller):
+    """
+    Handles account level requests.
+    """
+    @public
+    def GET(self, req):
+        """
+        Handle GET Service request
+        """
+        resp = req.get_response(self.app, query={'format': 'json'})
+
+        containers = json.loads(resp.body)
+
+        containers = filter(
+            lambda item: validate_bucket_name(
+                item['name'], self.conf.dns_compliant_bucket_names),
+            containers)
+
+        # we don't keep the creation time of a bucket (s3cmd doesn't
+        # work without that) so we use something bogus.
+        elem = Element('ListAllMyBucketsResult')
+
+        owner = SubElement(elem, 'Owner')
+        SubElement(owner, 'ID').text = req.user_id
+        SubElement(owner, 'DisplayName').text = req.user_id
+
+        buckets = SubElement(elem, 'Buckets')
+        for c in containers:
+            if self.conf.s3_acl and self.conf.check_bucket_owner:
+                try:
+                    req.get_response(self.app, 'HEAD', c['name'])
+                except AccessDenied:
+                    continue
+                except NoSuchBucket:
+                    continue
+
+            bucket = SubElement(buckets, 'Bucket')
+            SubElement(bucket, 'Name').text = c['name']
+            SubElement(bucket, 'CreationDate').text = \
+                '2009-02-03T16:45:09.000Z'
+
+        body = tostring(elem)
+
+        return HTTPOk(content_type='application/xml', body=body)

swift/common/middleware/s3api/controllers/versioning.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.utils import public
+
+from swift.common.middleware.s3api.controllers.base import Controller, \
+    bucket_operation
+from swift.common.middleware.s3api.etree import Element, tostring
+from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented
+
+
+class VersioningController(Controller):
+    """
+    Handles the following APIs:
+
+    * GET Bucket versioning
+    * PUT Bucket versioning
+
+    Those APIs are logged as VERSIONING operations in the S3 server log.
+    """
+    @public
+    @bucket_operation
+    def GET(self, req):
+        """
+        Handles GET Bucket versioning.
+        """
+        req.get_response(self.app, method='HEAD')
+
+        # Just report there is no versioning configured here.
+        elem = Element('VersioningConfiguration')
+        body = tostring(elem)
+
+        return HTTPOk(body=body, content_type="text/plain")
+
+    @public
+    @bucket_operation
+    def PUT(self, req):
+        """
+        Handles PUT Bucket versioning.
+        """
+        raise S3NotImplemented()

swift/common/middleware/s3api/etree.py

@@ -0,0 +1,146 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import lxml.etree
+from urllib import quote
+from copy import deepcopy
+from pkg_resources import resource_stream  # pylint: disable-msg=E0611
+import sys
+
+from swift.common.utils import get_logger
+from swift.common.middleware.s3api.exception import S3Exception
+from swift.common.middleware.s3api.utils import camel_to_snake, \
+    utf8encode, utf8decode
+
+XMLNS_S3 = 'http://s3.amazonaws.com/doc/2006-03-01/'
+XMLNS_XSI = 'http://www.w3.org/2001/XMLSchema-instance'
+
+
+class XMLSyntaxError(S3Exception):
+    pass
+
+
+class DocumentInvalid(S3Exception):
+    pass
+
+
+def cleanup_namespaces(elem):
+    def remove_ns(tag, ns):
+        if tag.startswith('{%s}' % ns):
+            tag = tag[len('{%s}' % ns):]
+        return tag
+
+    if not isinstance(elem.tag, basestring):
+        # elem is a comment element.
+        return
+
+    # remove s3 namespace
+    elem.tag = remove_ns(elem.tag, XMLNS_S3)
+
+    # remove default namespace
+    if elem.nsmap and None in elem.nsmap:
+        elem.tag = remove_ns(elem.tag, elem.nsmap[None])
+
+    for e in elem.iterchildren():
+        cleanup_namespaces(e)
+
+
+def fromstring(text, root_tag=None, logger=None):
+    try:
+        elem = lxml.etree.fromstring(text, parser)
+    except lxml.etree.XMLSyntaxError as e:
+        if logger:
+            logger.debug(e)
+        raise XMLSyntaxError(e)
+
+    cleanup_namespaces(elem)
+
+    if root_tag is not None:
+        # validate XML
+        try:
+            path = 'schema/%s.rng' % camel_to_snake(root_tag)
+            with resource_stream(__name__, path) as rng:
+                lxml.etree.RelaxNG(file=rng).assertValid(elem)
+        except IOError as e:
+            # Probably, the schema file doesn't exist.
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            logger = logger or get_logger({}, log_route='s3api')
+            logger.error(e)
+            raise exc_type, exc_value, exc_traceback
+        except lxml.etree.DocumentInvalid as e:
+            if logger:
+                logger.debug(e)
+            raise DocumentInvalid(e)
+
+    return elem
+
+
+def tostring(tree, encoding_type=None, use_s3ns=True):
+    if use_s3ns:
+        nsmap = tree.nsmap.copy()
+        nsmap[None] = XMLNS_S3
+
+        root = Element(tree.tag, attrib=tree.attrib, nsmap=nsmap)
+        root.text = tree.text
+        root.extend(deepcopy(tree.getchildren()))
+        tree = root
+
+    if encoding_type == 'url':
+        tree = deepcopy(tree)
+        for e in tree.iter():
+            # Some elements are not url-encoded even when we specify
+            # encoding_type=url.
+            blacklist = ['LastModified', 'ID', 'DisplayName', 'Initiated']
+            if e.tag not in blacklist:
+                if isinstance(e.text, basestring):
+                    e.text = quote(e.text)
+
+    return lxml.etree.tostring(tree, xml_declaration=True, encoding='UTF-8')
+
+
+class _Element(lxml.etree.ElementBase):
+    """
+    Wrapper Element class of lxml.etree.Element to support
+    a utf-8 encoded non-ascii string as a text.
+
+    Why we need this?:
+    Original lxml.etree.Element supports only unicode for the text.
+    It declines maintainability because we have to call a lot of encode/decode
+    methods to apply account/container/object name (i.e. PATH_INFO) to each
+    Element instance. When using this class, we can remove such a redundant
+    codes from swift.common.middleware.s3api middleware.
+    """
+    def __init__(self, *args, **kwargs):
+        # pylint: disable-msg=E1002
+        super(_Element, self).__init__(*args, **kwargs)
+
+    @property
+    def text(self):
+        """
+        utf-8 wrapper property of lxml.etree.Element.text
+        """
+        return utf8encode(lxml.etree.ElementBase.text.__get__(self))
+
+    @text.setter
+    def text(self, value):
+        lxml.etree.ElementBase.text.__set__(self, utf8decode(value))
+
+
+parser_lookup = lxml.etree.ElementDefaultClassLookup(element=_Element)
+parser = lxml.etree.XMLParser()
+parser.set_element_class_lookup(parser_lookup)
+
+Element = parser.makeelement
+SubElement = lxml.etree.SubElement

swift/common/middleware/s3api/exception.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class S3Exception(Exception):
+    pass
+
+
+class NotS3Request(S3Exception):
+    pass
+
+
+class BadSwiftRequest(S3Exception):
+    pass
+
+
+class ACLError(S3Exception):
+    pass
+
+
+class InvalidSubresource(S3Exception):
+    def __init__(self, resource, cause):
+        self.resource = resource
+        self.cause = cause

swift/common/middleware/s3api/s3api.py

@@ -0,0 +1,280 @@
+# Copyright (c) 2010-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+The s3api middleware will emulate the S3 REST api on top of swift.
+
+To enable this middleware to your configuration, add the s3api middleware
+in front of the auth middleware. See ``proxy-server.conf-sample`` for more
+detail and configurable options.
+
+To set up your client, the access key will be the concatenation of the
+account and user strings that should look like test:tester, and the
+secret access key is the account password.  The host should also point
+to the swift storage hostname.
+
+An example client using the python boto library is as follows::
+
+    from boto.s3.connection import S3Connection
+    connection = S3Connection(
+        aws_access_key_id='test:tester',
+        aws_secret_access_key='testing',
+        port=8080,
+        host='127.0.0.1',
+        is_secure=False,
+        calling_format=boto.s3.connection.OrdinaryCallingFormat())
+
+----------
+Deployment
+----------
+
+Proxy-Server Setting
+^^^^^^^^^^^^^^^^^^^^
+
+Set s3api before your auth in your pipeline in ``proxy-server.conf`` file.
+To enable all compatiblity currently supported, you should make sure that
+bulk, slo, and your auth middleware are also included in your proxy
+pipeline setting.
+
+Minimum example config is::
+
+    [pipeline:main]
+    pipeline = proxy-logging cache s3api tempauth bulk slo proxy-logging
+    proxy-server
+
+When using keystone, the config will be::
+
+    [pipeline:main]
+    pipeline = proxy-logging cache s3api s3token keystoneauth bulk slo
+    proxy-logging proxy-server
+
+.. note::
+    ``keystonemiddleware.authtoken`` can be located before/after s3api but
+    we recommend to put it before s3api because when authtoken is after s3api,
+    both authtoken and s3token will issue the acceptable token to keystone
+    (i.e. authenticate twice).
+
+Object-Server Setting
+^^^^^^^^^^^^^^^^^^^^^
+
+To get better compatibility, you may add S3 supported headers (
+Cache-Control, Content-Language, Expires, and X-Robots-Tag), that are
+not supporeted in Swift by default, into allowed_headers option in
+``object-server.conf`` Please see ``object-server.conf`` for more detail.
+
+-----------
+Constraints
+-----------
+Currently, the s3api is being ported from https://github.com/openstack/swift3
+so any existing issues in swift3 are still remaining. Please make sure
+descriptions in the example ``proxy-server.conf`` and what happens with the
+config, before enabling the options.
+
+-------------
+Supported API
+-------------
+The compatibility will continue to be improved upstream, you can keep and
+eye on compatibility via a check tool build by SwiftStack. See
+https://github.com/swiftstack/s3compat in detail.
+
+"""
+
+from paste.deploy import loadwsgi
+
+from swift.common.wsgi import PipelineWrapper, loadcontext
+
+from swift.common.middleware.s3api.exception import NotS3Request, \
+    InvalidSubresource
+from swift.common.middleware.s3api.s3request import get_request_class
+from swift.common.middleware.s3api.s3response import ErrorResponse, \
+    InternalError, MethodNotAllowed, S3ResponseBase, S3NotImplemented
+from swift.common.utils import get_logger, register_swift_info, \
+    config_true_value, config_positive_int_value
+from swift.common.middleware.s3api.utils import Config
+from swift.common.middleware.s3api.acl_handlers import get_acl_handler
+
+
+class S3ApiMiddleware(object):
+    """S3Api: S3 compatibility middleware"""
+    def __init__(self, app, conf, *args, **kwargs):
+        self.app = app
+        self.conf = Config()
+
+        # Set default values if they are not configured
+        self.conf.allow_no_owner = config_true_value(
+            conf.get('allow_no_owner', False))
+        self.conf.location = conf.get('location', 'US')
+        self.conf.dns_compliant_bucket_names = config_true_value(
+            conf.get('dns_compliant_bucket_names', True))
+        self.conf.max_bucket_listing = config_positive_int_value(
+            conf.get('max_bucket_listing', 1000))
+        self.conf.max_parts_listing = config_positive_int_value(
+            conf.get('max_parts_listing', 1000))
+        self.conf.max_multi_delete_objects = config_positive_int_value(
+            conf.get('max_multi_delete_objects', 1000))
+        self.conf.s3_acl = config_true_value(
+            conf.get('s3_acl', False))
+        self.conf.storage_domain = conf.get('storage_domain', '')
+        self.conf.auth_pipeline_check = config_true_value(
+            conf.get('auth_pipeline_check', True))
+        self.conf.max_upload_part_num = config_positive_int_value(
+            conf.get('max_upload_part_num', 1000))
+        self.conf.check_bucket_owner = config_true_value(
+            conf.get('check_bucket_owner', False))
+        self.conf.force_swift_request_proxy_log = config_true_value(
+            conf.get('force_swift_request_proxy_log', False))
+        self.conf.allow_multipart_uploads = config_true_value(
+            conf.get('allow_multipart_uploads', True))
+        self.conf.min_segment_size = config_positive_int_value(
+            conf.get('min_segment_size', 5242880))
+
+        self.logger = get_logger(
+            conf, log_route=conf.get('log_name', 's3api'))
+        self.slo_enabled = self.conf.allow_multipart_uploads
+        self.check_pipeline(self.conf)
+
+    def __call__(self, env, start_response):
+        try:
+            req_class = get_request_class(env, self.conf.s3_acl)
+            req = req_class(
+                env, self.app, self.slo_enabled, self.conf.storage_domain,
+                self.conf.location, self.conf.force_swift_request_proxy_log,
+                self.conf.dns_compliant_bucket_names,
+                self.conf.allow_multipart_uploads, self.conf.allow_no_owner)
+            resp = self.handle_request(req)
+        except NotS3Request:
+            resp = self.app
+        except InvalidSubresource as e:
+            self.logger.debug(e.cause)
+        except ErrorResponse as err_resp:
+            if isinstance(err_resp, InternalError):
+                self.logger.exception(err_resp)
+            resp = err_resp
+        except Exception as e:
+            self.logger.exception(e)
+            resp = InternalError(reason=e)
+
+        if isinstance(resp, S3ResponseBase) and 'swift.trans_id' in env:
+            resp.headers['x-amz-id-2'] = env['swift.trans_id']
+            resp.headers['x-amz-request-id'] = env['swift.trans_id']
+
+        return resp(env, start_response)
+
+    def handle_request(self, req):
+        self.logger.debug('Calling S3Api Middleware')
+        self.logger.debug(req.__dict__)
+        try:
+            controller = req.controller(self.app, self.conf, self.logger)
+        except S3NotImplemented:
+            # TODO: Probably we should distinct the error to log this warning
+            self.logger.warning('multipart: No SLO middleware in pipeline')
+            raise
+
+        acl_handler = get_acl_handler(req.controller_name)(req, self.logger)
+        req.set_acl_handler(acl_handler)
+
+        if hasattr(controller, req.method):
+            handler = getattr(controller, req.method)
+            if not getattr(handler, 'publicly_accessible', False):
+                raise MethodNotAllowed(req.method,
+                                       req.controller.resource_type())
+            res = handler(req)
+        else:
+            raise MethodNotAllowed(req.method,
+                                   req.controller.resource_type())
+
+        return res
+
+    def check_pipeline(self, conf):
+        """
+        Check that proxy-server.conf has an appropriate pipeline for s3api.
+        """
+        if conf.get('__file__', None) is None:
+            return
+
+        ctx = loadcontext(loadwsgi.APP, conf.__file__)
+        pipeline = str(PipelineWrapper(ctx)).split(' ')
+
+        # Add compatible with 3rd party middleware.
+        self.check_filter_order(pipeline, ['s3api', 'proxy-server'])
+
+        auth_pipeline = pipeline[pipeline.index('s3api') + 1:
+                                 pipeline.index('proxy-server')]
+
+        # Check SLO middleware
+        if self.slo_enabled and 'slo' not in auth_pipeline:
+            self.slo_enabled = False
+            self.logger.warning('s3api middleware requires SLO middleware '
+                                'to support multi-part upload, please add it '
+                                'in pipeline')
+
+        if not conf.auth_pipeline_check:
+            self.logger.debug('Skip pipeline auth check.')
+            return
+
+        if 'tempauth' in auth_pipeline:
+            self.logger.debug('Use tempauth middleware.')
+        elif 'keystoneauth' in auth_pipeline:
+            self.check_filter_order(
+                auth_pipeline,
+                ['s3token', 'keystoneauth'])
+            self.logger.debug('Use keystone middleware.')
+        elif len(auth_pipeline):
+            self.logger.debug('Use third party(unknown) auth middleware.')
+        else:
+            raise ValueError('Invalid pipeline %r: expected auth between '
+                             's3api and proxy-server ' % pipeline)
+
+    def check_filter_order(self, pipeline, required_filters):
+        """
+        Check that required filters are present in order in the pipeline.
+        """
+        indexes = []
+        missing_filters = []
+        for required_filter in required_filters:
+            try:
+                indexes.append(pipeline.index(required_filter))
+            except ValueError as e:
+                self.logger.debug(e)
+                missing_filters.append(required_filter)
+
+        if missing_filters:
+            raise ValueError('Invalid pipeline %r: missing filters %r' % (
+                pipeline, missing_filters))
+
+        if indexes != sorted(indexes):
+            raise ValueError('Invalid pipeline %r: expected filter %s' % (
+                pipeline, ' before '.join(required_filters)))
+
+
+def filter_factory(global_conf, **local_conf):
+    """Standard filter factory to use the middleware with paste.deploy"""
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    register_swift_info(
+        's3api',
+        # TODO: make default values as variables
+        max_bucket_listing=conf.get('max_bucket_listing', 1000),
+        max_parts_listing=conf.get('max_parts_listing', 1000),
+        max_upload_part_num=conf.get('max_upload_part_num', 1000),
+        max_multi_delete_objects=conf.get('max_multi_delete_objects', 1000),
+        allow_multipart_uploads=conf.get('allow_multipart_uploads', True),
+        min_segment_size=conf.get('min_segment_size', 5242880),
+    )
+
+    def s3api_filter(app):
+        return S3ApiMiddleware(app, conf)
+
+    return s3api_filter

swift/common/middleware/s3api/s3response.py

@@ -0,0 +1,684 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from UserDict import DictMixin
+from functools import partial
+
+from swift.common import swob
+from swift.common.utils import config_true_value
+from swift.common.request_helpers import is_sys_meta
+
+from swift.common.middleware.s3api.utils import snake_to_camel, sysmeta_prefix
+from swift.common.middleware.s3api.etree import Element, SubElement, tostring
+
+
+class HeaderKey(str):
+    """
+    A string object that normalizes string as S3 clients expect with title().
+    """
+    def title(self):
+        if self.lower() == 'etag':
+            # AWS Java SDK expects only 'ETag'.
+            return 'ETag'
+        if self.lower().startswith('x-amz-'):
+            # AWS headers returned by S3 are lowercase.
+            return self.lower()
+        return str.title(self)
+
+
+class HeaderKeyDict(swob.HeaderKeyDict):
+    """
+    Similar to the HeaderKeyDict class in Swift, but its key name is normalized
+    as S3 clients expect.
+    """
+    def __getitem__(self, key):
+        return swob.HeaderKeyDict.__getitem__(self, HeaderKey(key))
+
+    def __setitem__(self, key, value):
+        return swob.HeaderKeyDict.__setitem__(self, HeaderKey(key), value)
+
+    def __contains__(self, key):
+        return swob.HeaderKeyDict.__contains__(self, HeaderKey(key))
+
+    def __delitem__(self, key):
+        return swob.HeaderKeyDict.__delitem__(self, HeaderKey(key))
+
+    def get(self, key, default=None):
+        return swob.HeaderKeyDict.get(self, HeaderKey(key), default)
+
+    def pop(self, key, default=None):
+        return swob.HeaderKeyDict.pop(self, HeaderKey(key), default)
+
+
+class S3ResponseBase(object):
+    """
+    Base class for swift3 responses.
+    """
+    pass
+
+
+class S3Response(S3ResponseBase, swob.Response):
+    """
+    Similar to the Response class in Swift, but uses our HeaderKeyDict for
+    headers instead of Swift's HeaderKeyDict.  This also translates Swift
+    specific headers to S3 headers.
+    """
+    def __init__(self, *args, **kwargs):
+        swob.Response.__init__(self, *args, **kwargs)
+
+        if self.etag:
+            # add double quotes to the etag header
+            self.etag = self.etag
+
+        sw_sysmeta_headers = swob.HeaderKeyDict()
+        sw_headers = swob.HeaderKeyDict()
+        headers = HeaderKeyDict()
+        self.is_slo = False
+
+        def is_swift3_sysmeta(sysmeta_key, server_type):
+            swift3_sysmeta_prefix = (
+                'x-%s-sysmeta-swift3' % server_type).lower()
+            return sysmeta_key.lower().startswith(swift3_sysmeta_prefix)
+
+        def is_s3api_sysmeta(sysmeta_key, server_type):
+            s3api_sysmeta_prefix = sysmeta_prefix(_server_type).lower()
+            return sysmeta_key.lower().startswith(s3api_sysmeta_prefix)
+
+        for key, val in self.headers.iteritems():
+            if is_sys_meta('object', key) or is_sys_meta('container', key):
+                _server_type = key.split('-')[1]
+                if is_swift3_sysmeta(key, _server_type):
+                    # To be compatible with older swift3, translate swift3
+                    # sysmeta to s3api sysmeta here
+                    key = sysmeta_prefix(_server_type) + \
+                        key[len('x-%s-sysmeta-swift3-' % _server_type):]
+
+                    if key not in sw_sysmeta_headers:
+                        # To avoid overwrite s3api sysmeta by older swift3
+                        # sysmeta set the key only when the key does not exist
+                        sw_sysmeta_headers[key] = val
+                elif is_s3api_sysmeta(key, _server_type):
+                    sw_sysmeta_headers[key] = val
+            else:
+                sw_headers[key] = val
+
+        # Handle swift headers
+        for key, val in sw_headers.iteritems():
+            _key = key.lower()
+
+            if _key.startswith('x-object-meta-'):
+                # Note that AWS ignores user-defined headers with '=' in the
+                # header name. We translated underscores to '=5F' on the way
+                # in, though.
+                headers['x-amz-meta-' + _key[14:].replace('=5f', '_')] = val
+            elif _key in ('content-length', 'content-type',
+                          'content-range', 'content-encoding',
+                          'content-disposition', 'content-language',
+                          'etag', 'last-modified', 'x-robots-tag',
+                          'cache-control', 'expires'):
+                headers[key] = val
+            elif _key == 'x-static-large-object':
+                # for delete slo
+                self.is_slo = config_true_value(val)
+
+        self.headers = headers
+        # Used for pure swift header handling at the request layer
+        self.sw_headers = sw_headers
+        self.sysmeta_headers = sw_sysmeta_headers
+
+    @classmethod
+    def from_swift_resp(cls, sw_resp):
+        """
+        Create a new S3 response object based on the given Swift response.
+        """
+        if sw_resp.app_iter:
+            body = None
+            app_iter = sw_resp.app_iter
+        else:
+            body = sw_resp.body
+            app_iter = None
+
+        resp = cls(status=sw_resp.status, headers=sw_resp.headers,
+                   request=sw_resp.request, body=body, app_iter=app_iter,
+                   conditional_response=sw_resp.conditional_response)
+        resp.environ.update(sw_resp.environ)
+
+        return resp
+
+    def append_copy_resp_body(self, controller_name, last_modified):
+        elem = Element('Copy%sResult' % controller_name)
+        SubElement(elem, 'LastModified').text = last_modified
+        SubElement(elem, 'ETag').text = '"%s"' % self.etag
+        self.headers['Content-Type'] = 'application/xml'
+        self.body = tostring(elem)
+        self.etag = None
+
+
+HTTPOk = partial(S3Response, status=200)
+HTTPCreated = partial(S3Response, status=201)
+HTTPAccepted = partial(S3Response, status=202)
+HTTPNoContent = partial(S3Response, status=204)
+HTTPPartialContent = partial(S3Response, status=206)
+
+
+class ErrorResponse(S3ResponseBase, swob.HTTPException):
+    """
+    S3 error object.
+
+    Reference information about S3 errors is available at:
+    http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+    """
+    _status = ''
+    _msg = ''
+    _code = ''
+
+    def __init__(self, msg=None, *args, **kwargs):
+        if msg:
+            self._msg = msg
+        if not self._code:
+            self._code = self.__class__.__name__
+
+        self.info = kwargs.copy()
+        for reserved_key in ('headers', 'body'):
+            if self.info.get(reserved_key):
+                del(self.info[reserved_key])
+
+        swob.HTTPException.__init__(self, status=self._status,
+                                    app_iter=self._body_iter(),
+                                    content_type='application/xml', *args,
+                                    **kwargs)
+        self.headers = HeaderKeyDict(self.headers)
+
+    def _body_iter(self):
+        error_elem = Element('Error')
+        SubElement(error_elem, 'Code').text = self._code
+        SubElement(error_elem, 'Message').text = self._msg
+        if 'swift.trans_id' in self.environ:
+            request_id = self.environ['swift.trans_id']
+            SubElement(error_elem, 'RequestId').text = request_id
+
+        self._dict_to_etree(error_elem, self.info)
+
+        yield tostring(error_elem, use_s3ns=False)
+
+    def _dict_to_etree(self, parent, d):
+        for key, value in d.items():
+            tag = re.sub('\W', '', snake_to_camel(key))
+            elem = SubElement(parent, tag)
+
+            if isinstance(value, (dict, DictMixin)):
+                self._dict_to_etree(elem, value)
+            else:
+                try:
+                    elem.text = str(value)
+                except ValueError:
+                    # We set an invalid string for XML.
+                    elem.text = '(invalid string)'
+
+
+class AccessDenied(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'Access Denied.'
+
+
+class AccountProblem(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'There is a problem with your AWS account that prevents the ' \
+           'operation from completing successfully.'
+
+
+class AmbiguousGrantByEmailAddress(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The e-mail address you provided is associated with more than ' \
+           'one account.'
+
+
+class AuthorizationHeaderMalformed(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The authorization header is malformed; the authorization ' \
+           'header requires three components: Credential, SignedHeaders, ' \
+           'and Signature.'
+
+
+class AuthorizationQueryParametersError(ErrorResponse):
+    _status = '400 Bad Request'
+
+
+class BadDigest(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The Content-MD5 you specified did not match what we received.'
+
+
+class BucketAlreadyExists(ErrorResponse):
+    _status = '409 Conflict'
+    _msg = 'The requested bucket name is not available. The bucket ' \
+           'namespace is shared by all users of the system. Please select a ' \
+           'different name and try again.'
+
+    def __init__(self, bucket, msg=None, *args, **kwargs):
+        ErrorResponse.__init__(self, msg, bucket_name=bucket, *args, **kwargs)
+
+
+class BucketAlreadyOwnedByYou(ErrorResponse):
+    _status = '409 Conflict'
+    _msg = 'Your previous request to create the named bucket succeeded and ' \
+           'you already own it.'
+
+    def __init__(self, bucket, msg=None, *args, **kwargs):
+        ErrorResponse.__init__(self, msg, bucket_name=bucket, *args, **kwargs)
+
+
+class BucketNotEmpty(ErrorResponse):
+    _status = '409 Conflict'
+    _msg = 'The bucket you tried to delete is not empty'
+
+
+class CredentialsNotSupported(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'This request does not support credentials.'
+
+
+class CrossLocationLoggingProhibited(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'Cross location logging not allowed. Buckets in one geographic ' \
+           'location cannot log information to a bucket in another location.'
+
+
+class EntityTooSmall(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Your proposed upload is smaller than the minimum allowed object ' \
+           'size.'
+
+
+class EntityTooLarge(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Your proposed upload exceeds the maximum allowed object size.'
+
+
+class ExpiredToken(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The provided token has expired.'
+
+
+class IllegalVersioningConfigurationException(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The Versioning configuration specified in the request is invalid.'
+
+
+class IncompleteBody(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'You did not provide the number of bytes specified by the ' \
+           'Content-Length HTTP header.'
+
+
+class IncorrectNumberOfFilesInPostRequest(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'POST requires exactly one file upload per request.'
+
+
+class InlineDataTooLarge(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Inline data exceeds the maximum allowed size.'
+
+
+class InternalError(ErrorResponse):
+    _status = '500 Internal Server Error'
+    _msg = 'We encountered an internal error. Please try again.'
+
+
+class InvalidAccessKeyId(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'The AWS Access Key Id you provided does not exist in our records.'
+
+
+class InvalidArgument(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Invalid Argument.'
+
+    def __init__(self, name, value, msg=None, *args, **kwargs):
+        ErrorResponse.__init__(self, msg, argument_name=name,
+                               argument_value=value, *args, **kwargs)
+
+
+class InvalidBucketName(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The specified bucket is not valid.'
+
+    def __init__(self, bucket, msg=None, *args, **kwargs):
+        ErrorResponse.__init__(self, msg, bucket_name=bucket, *args, **kwargs)
+
+
+class InvalidBucketState(ErrorResponse):
+    _status = '409 Conflict'
+    _msg = 'The request is not valid with the current state of the bucket.'
+
+
+class InvalidDigest(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The Content-MD5 you specified was an invalid.'
+
+
+class InvalidLocationConstraint(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The specified location constraint is not valid.'
+
+
+class InvalidObjectState(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'The operation is not valid for the current state of the object.'
+
+
+class InvalidPart(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'One or more of the specified parts could not be found. The part ' \
+           'might not have been uploaded, or the specified entity tag might ' \
+           'not have matched the part\'s entity tag.'
+
+
+class InvalidPartOrder(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The list of parts was not in ascending order.Parts list must ' \
+           'specified in order by part number.'
+
+
+class InvalidPayer(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'All access to this object has been disabled.'
+
+
+class InvalidPolicyDocument(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The content of the form does not meet the conditions specified ' \
+           'in the policy document.'
+
+
+class InvalidRange(ErrorResponse):
+    _status = '416 Requested Range Not Satisfiable'
+    _msg = 'The requested range cannot be satisfied.'
+
+
+class InvalidRequest(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Invalid Request.'
+
+
+class InvalidSecurity(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'The provided security credentials are not valid.'
+
+
+class InvalidSOAPRequest(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The SOAP request body is invalid.'
+
+
+class InvalidStorageClass(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The storage class you specified is not valid.'
+
+
+class InvalidTargetBucketForLogging(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The target bucket for logging does not exist, is not owned by ' \
+           'you, or does not have the appropriate grants for the ' \
+           'log-delivery group.'
+
+    def __init__(self, bucket, msg=None, *args, **kwargs):
+        ErrorResponse.__init__(self, msg, target_bucket=bucket, *args,
+                               **kwargs)
+
+
+class InvalidToken(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The provided token is malformed or otherwise invalid.'
+
+
+class InvalidURI(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Couldn\'t parse the specified URI.'
+
+    def __init__(self, uri, msg=None, *args, **kwargs):
+        ErrorResponse.__init__(self, msg, uri=uri, *args, **kwargs)
+
+
+class KeyTooLong(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Your key is too long.'
+
+
+class MalformedACLError(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The XML you provided was not well-formed or did not validate ' \
+           'against our published schema.'
+
+
+class MalformedPOSTRequest(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The body of your POST request is not well-formed ' \
+           'multipart/form-data.'
+
+
+class MalformedXML(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The XML you provided was not well-formed or did not validate ' \
+           'against our published schema.'
+
+
+class MaxMessageLengthExceeded(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Your request was too big.'
+
+
+class MaxPostPreDataLengthExceededError(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Your POST request fields preceding the upload file were too large.'
+
+
+class MetadataTooLarge(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Your metadata headers exceed the maximum allowed metadata size.'
+
+
+class MethodNotAllowed(ErrorResponse):
+    _status = '405 Method Not Allowed'
+    _msg = 'The specified method is not allowed against this resource.'
+
+    def __init__(self, method, resource_type, msg=None, *args, **kwargs):
+        ErrorResponse.__init__(self, msg, method=method,
+                               resource_type=resource_type, *args, **kwargs)
+
+
+class MissingContentLength(ErrorResponse):
+    _status = '411 Length Required'
+    _msg = 'You must provide the Content-Length HTTP header.'
+
+
+class MissingRequestBodyError(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Request body is empty.'
+
+
+class MissingSecurityElement(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The SOAP 1.1 request is missing a security element.'
+
+
+class MissingSecurityHeader(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Your request was missing a required header.'
+
+
+class NoLoggingStatusForKey(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'There is no such thing as a logging status sub-resource for a key.'
+
+
+class NoSuchBucket(ErrorResponse):
+    _status = '404 Not Found'
+    _msg = 'The specified bucket does not exist.'
+
+    def __init__(self, bucket, msg=None, *args, **kwargs):
+        if not bucket:
+            raise InternalError()
+        ErrorResponse.__init__(self, msg, bucket_name=bucket, *args, **kwargs)
+
+
+class NoSuchKey(ErrorResponse):
+    _status = '404 Not Found'
+    _msg = 'The specified key does not exist.'
+
+    def __init__(self, key, msg=None, *args, **kwargs):
+        if not key:
+            raise InternalError()
+        ErrorResponse.__init__(self, msg, key=key, *args, **kwargs)
+
+
+class NoSuchLifecycleConfiguration(ErrorResponse):
+    _status = '404 Not Found'
+    _msg = 'The lifecycle configuration does not exist. .'
+
+
+class NoSuchUpload(ErrorResponse):
+    _status = '404 Not Found'
+    _msg = 'The specified multipart upload does not exist. The upload ID ' \
+           'might be invalid, or the multipart upload might have been ' \
+           'aborted or completed.'
+
+
+class NoSuchVersion(ErrorResponse):
+    _status = '404 Not Found'
+    _msg = 'The specified version does not exist.'
+
+    def __init__(self, key, version_id, msg=None, *args, **kwargs):
+        if not key:
+            raise InternalError()
+        ErrorResponse.__init__(self, msg, key=key, version_id=version_id,
+                               *args, **kwargs)
+
+
+# NotImplemented is a python built-in constant.  Use S3NotImplemented instead.
+class S3NotImplemented(ErrorResponse):
+    _status = '501 Not Implemented'
+    _msg = 'Not implemented.'
+    _code = 'NotImplemented'
+
+
+class NotSignedUp(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'Your account is not signed up for the Amazon S3 service.'
+
+
+class NotSuchBucketPolicy(ErrorResponse):
+    _status = '404 Not Found'
+    _msg = 'The specified bucket does not have a bucket policy.'
+
+
+class OperationAborted(ErrorResponse):
+    _status = '409 Conflict'
+    _msg = 'A conflicting conditional operation is currently in progress ' \
+           'against this resource. Please try again.'
+
+
+class PermanentRedirect(ErrorResponse):
+    _status = '301 Moved Permanently'
+    _msg = 'The bucket you are attempting to access must be addressed using ' \
+           'the specified endpoint. Please send all future requests to this ' \
+           'endpoint.'
+
+
+class PreconditionFailed(ErrorResponse):
+    _status = '412 Precondition Failed'
+    _msg = 'At least one of the preconditions you specified did not hold.'
+
+
+class Redirect(ErrorResponse):
+    _status = '307 Moved Temporarily'
+    _msg = 'Temporary redirect.'
+
+
+class RestoreAlreadyInProgress(ErrorResponse):
+    _status = '409 Conflict'
+    _msg = 'Object restore is already in progress.'
+
+
+class RequestIsNotMultiPartContent(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Bucket POST must be of the enclosure-type multipart/form-data.'
+
+
+class RequestTimeout(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Your socket connection to the server was not read from or ' \
+           'written to within the timeout period.'
+
+
+class RequestTimeTooSkewed(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'The difference between the request time and the current time ' \
+           'is too large.'
+
+
+class RequestTorrentOfBucketError(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'Requesting the torrent file of a bucket is not permitted.'
+
+
+class SignatureDoesNotMatch(ErrorResponse):
+    _status = '403 Forbidden'
+    _msg = 'The request signature we calculated does not match the ' \
+           'signature you provided. Check your key and signing method.'
+
+
+class ServiceUnavailable(ErrorResponse):
+    _status = '503 Service Unavailable'
+    _msg = 'Please reduce your request rate.'
+
+
+class SlowDown(ErrorResponse):
+    _status = '503 Slow Down'
+    _msg = 'Please reduce your request rate.'
+
+
+class TemporaryRedirect(ErrorResponse):
+    _status = '307 Moved Temporarily'
+    _msg = 'You are being redirected to the bucket while DNS updates.'
+
+
+class TokenRefreshRequired(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The provided token must be refreshed.'
+
+
+class TooManyBuckets(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'You have attempted to create more buckets than allowed.'
+
+
+class UnexpectedContent(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'This request does not support content.'
+
+
+class UnresolvableGrantByEmailAddress(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The e-mail address you provided does not match any account on ' \
+           'record.'
+
+
+class UserKeyMustBeSpecified(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = 'The bucket POST must contain the specified field name. If it is ' \
+           'specified, please check the order of the fields.'

swift/common/middleware/s3api/s3token.py

@@ -0,0 +1,324 @@
+# Copyright 2012 OpenStack Foundation
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011,2012 Akira YOSHIYAMA <akirayoshiyama@gmail.com>
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This source code is based ./auth_token.py and ./ec2_token.py.
+# See them for their copyright.
+
+"""
+-------------------
+S3 Token Middleware
+-------------------
+s3token middleware is for authentication with s3api + keystone.
+This middleware:
+
+* Gets a request from the s3api middleware with an S3 Authorization
+  access key.
+* Validates s3 token with Keystone.
+* Transforms the account name to AUTH_%(tenant_name).
+
+"""
+
+import base64
+import json
+
+import requests
+import six
+from six.moves import urllib
+
+from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \
+    HTTPException
+from swift.common.utils import config_true_value, split_path, get_logger
+from swift.common.wsgi import ConfigFileError
+
+
+PROTOCOL_NAME = 'S3 Token Authentication'
+
+# Headers to purge if they came from (or may have come from) the client
+KEYSTONE_AUTH_HEADERS = (
+    'X-Identity-Status', 'X-Service-Identity-Status',
+    'X-Domain-Id', 'X-Service-Domain-Id',
+    'X-Domain-Name', 'X-Service-Domain-Name',
+    'X-Project-Id', 'X-Service-Project-Id',
+    'X-Project-Name', 'X-Service-Project-Name',
+    'X-Project-Domain-Id', 'X-Service-Project-Domain-Id',
+    'X-Project-Domain-Name', 'X-Service-Project-Domain-Name',
+    'X-User-Id', 'X-Service-User-Id',
+    'X-User-Name', 'X-Service-User-Name',
+    'X-User-Domain-Id', 'X-Service-User-Domain-Id',
+    'X-User-Domain-Name', 'X-Service-User-Domain-Name',
+    'X-Roles', 'X-Service-Roles',
+    'X-Is-Admin-Project',
+    'X-Service-Catalog',
+    # Deprecated headers, too...
+    'X-Tenant-Id',
+    'X-Tenant-Name',
+    'X-Tenant',
+    'X-User',
+    'X-Role',
+)
+
+
+def parse_v2_response(token):
+    access_info = token['access']
+    headers = {
+        'X-Identity-Status': 'Confirmed',
+        'X-Roles': ','.join(r['name']
+                            for r in access_info['user']['roles']),
+        'X-User-Id': access_info['user']['id'],
+        'X-User-Name': access_info['user']['name'],
+        'X-Tenant-Id': access_info['token']['tenant']['id'],
+        'X-Tenant-Name': access_info['token']['tenant']['name'],
+        'X-Project-Id': access_info['token']['tenant']['id'],
+        'X-Project-Name': access_info['token']['tenant']['name'],
+    }
+    return (
+        headers,
+        access_info['token'].get('id'),
+        access_info['token']['tenant'])
+
+
+def parse_v3_response(token):
+    token = token['token']
+    headers = {
+        'X-Identity-Status': 'Confirmed',
+        'X-Roles': ','.join(r['name']
+                            for r in token['roles']),
+        'X-User-Id': token['user']['id'],
+        'X-User-Name': token['user']['name'],
+        'X-User-Domain-Id': token['user']['domain']['id'],
+        'X-User-Domain-Name': token['user']['domain']['name'],
+        'X-Tenant-Id': token['project']['id'],
+        'X-Tenant-Name': token['project']['name'],
+        'X-Project-Id': token['project']['id'],
+        'X-Project-Name': token['project']['name'],
+        'X-Project-Domain-Id': token['project']['domain']['id'],
+        'X-Project-Domain-Name': token['project']['domain']['name'],
+    }
+    return headers, None, token['project']
+
+
+class S3Token(object):
+    """Middleware that handles S3 authentication."""
+
+    def __init__(self, app, conf):
+        """Common initialization code."""
+        self._app = app
+        self._logger = get_logger(
+            conf, log_route=conf.get('log_name', 's3token'))
+        self._logger.debug('Starting the %s component', PROTOCOL_NAME)
+        self._timeout = float(conf.get('http_timeout', '10.0'))
+        if not (0 < self._timeout <= 60):
+            raise ValueError('http_timeout must be between 0 and 60 seconds')
+        self._reseller_prefix = conf.get('reseller_prefix', 'AUTH_')
+        self._delay_auth_decision = config_true_value(
+            conf.get('delay_auth_decision'))
+
+        # where to find the auth service (we use this to validate tokens)
+        self._request_uri = conf.get('auth_uri', '').rstrip('/') + '/s3tokens'
+        parsed = urllib.parse.urlsplit(self._request_uri)
+        if not parsed.scheme or not parsed.hostname:
+            raise ConfigFileError(
+                'Invalid auth_uri; must include scheme and host')
+        if parsed.scheme not in ('http', 'https'):
+            raise ConfigFileError(
+                'Invalid auth_uri; scheme must be http or https')
+        if parsed.query or parsed.fragment or '@' in parsed.netloc:
+            raise ConfigFileError('Invalid auth_uri; must not include '
+                                  'username, query, or fragment')
+
+        # SSL
+        insecure = config_true_value(conf.get('insecure'))
+        cert_file = conf.get('certfile')
+        key_file = conf.get('keyfile')
+
+        if insecure:
+            self._verify = False
+        elif cert_file and key_file:
+            self._verify = (cert_file, key_file)
+        elif cert_file:
+            self._verify = cert_file
+        else:
+            self._verify = None
+
+    def _deny_request(self, code):
+        error_cls, message = {
+            'AccessDenied': (HTTPUnauthorized, 'Access denied'),
+            'InvalidURI': (HTTPBadRequest,
+                           'Could not parse the specified URI'),
+        }[code]
+        resp = error_cls(content_type='text/xml')
+        error_msg = ('<?xml version="1.0" encoding="UTF-8"?>\r\n'
+                     '<Error>\r\n  <Code>%s</Code>\r\n  '
+                     '<Message>%s</Message>\r\n</Error>\r\n' %
+                     (code, message))
+        if six.PY3:
+            error_msg = error_msg.encode()
+        resp.body = error_msg
+        return resp
+
+    def _json_request(self, creds_json):
+        headers = {'Content-Type': 'application/json'}
+        try:
+            response = requests.post(self._request_uri,
+                                     headers=headers, data=creds_json,
+                                     verify=self._verify,
+                                     timeout=self._timeout)
+        except requests.exceptions.RequestException as e:
+            self._logger.info('HTTP connection exception: %s', e)
+            raise self._deny_request('InvalidURI')
+
+        if response.status_code < 200 or response.status_code >= 300:
+            self._logger.debug('Keystone reply error: status=%s reason=%s',
+                               response.status_code, response.reason)
+            raise self._deny_request('AccessDenied')
+
+        return response
+
+    def __call__(self, environ, start_response):
+        """Handle incoming request. authenticate and send downstream."""
+        req = Request(environ)
+        self._logger.debug('Calling S3Token middleware.')
+
+        # Always drop auth headers if we're first in the pipeline
+        if 'keystone.token_info' not in req.environ:
+            req.headers.update({h: None for h in KEYSTONE_AUTH_HEADERS})
+
+        try:
+            parts = split_path(req.path, 1, 4, True)
+            version, account, container, obj = parts
+        except ValueError:
+            msg = 'Not a path query: %s, skipping.' % req.path
+            self._logger.debug(msg)
+            return self._app(environ, start_response)
+
+        # Read request signature and access id.
+        s3_auth_details = req.environ.get('s3api.auth_details')
+        if not s3_auth_details:
+            msg = 'No authorization details from s3api. skipping.'
+            self._logger.debug(msg)
+            return self._app(environ, start_response)
+
+        access = s3_auth_details['access_key']
+        if isinstance(access, six.binary_type):
+            access = access.decode('utf-8')
+
+        signature = s3_auth_details['signature']
+        if isinstance(signature, six.binary_type):
+            signature = signature.decode('utf-8')
+
+        string_to_sign = s3_auth_details['string_to_sign']
+        if isinstance(string_to_sign, six.text_type):
+            string_to_sign = string_to_sign.encode('utf-8')
+        token = base64.urlsafe_b64encode(string_to_sign).encode('ascii')
+
+        # NOTE(chmou): This is to handle the special case with nova
+        # when we have the option s3_affix_tenant. We will force it to
+        # connect to another account than the one
+        # authenticated. Before people start getting worried about
+        # security, I should point that we are connecting with
+        # username/token specified by the user but instead of
+        # connecting to its own account we will force it to go to an
+        # another account. In a normal scenario if that user don't
+        # have the reseller right it will just fail but since the
+        # reseller account can connect to every account it is allowed
+        # by the swift_auth middleware.
+        force_tenant = None
+        if ':' in access:
+            access, force_tenant = access.split(':')
+
+        # Authenticate request.
+        creds = {'credentials': {'access': access,
+                                 'token': token,
+                                 'signature': signature}}
+        creds_json = json.dumps(creds)
+        self._logger.debug('Connecting to Keystone sending this JSON: %s',
+                           creds_json)
+        # NOTE(vish): We could save a call to keystone by having
+        #             keystone return token, tenant, user, and roles
+        #             from this call.
+        #
+        # NOTE(chmou): We still have the same problem we would need to
+        #              change token_auth to detect if we already
+        #              identified and not doing a second query and just
+        #              pass it through to swiftauth in this case.
+        try:
+            # NB: requests.Response, not swob.Response
+            resp = self._json_request(creds_json)
+        except HTTPException as e_resp:
+            if self._delay_auth_decision:
+                msg = 'Received error, deferring rejection based on error: %s'
+                self._logger.debug(msg, e_resp.status)
+                return self._app(environ, start_response)
+            else:
+                msg = 'Received error, rejecting request with error: %s'
+                self._logger.debug(msg, e_resp.status)
+                # NB: swob.Response, not requests.Response
+                return e_resp(environ, start_response)
+
+        self._logger.debug('Keystone Reply: Status: %d, Output: %s',
+                           resp.status_code, resp.content)
+
+        try:
+            token = resp.json()
+            if 'access' in token:
+                headers, token_id, tenant = parse_v2_response(token)
+            elif 'token' in token:
+                headers, token_id, tenant = parse_v3_response(token)
+            else:
+                raise ValueError
+
+            # Populate the environment similar to auth_token,
+            # so we don't have to contact Keystone again.
+            #
+            # Note that although the strings are unicode following json
+            # deserialization, Swift's HeaderEnvironProxy handles ensuring
+            # they're stored as native strings
+            req.headers.update(headers)
+            req.environ['keystone.token_info'] = token
+        except (ValueError, KeyError, TypeError):
+            if self._delay_auth_decision:
+                error = ('Error on keystone reply: %d %s - '
+                         'deferring rejection downstream')
+                self._logger.debug(error, resp.status_code, resp.content)
+                return self._app(environ, start_response)
+            else:
+                error = ('Error on keystone reply: %d %s - '
+                         'rejecting request')
+                self._logger.debug(error, resp.status_code, resp.content)
+                return self._deny_request('InvalidURI')(
+                    environ, start_response)
+
+        req.headers['X-Auth-Token'] = token_id
+        tenant_to_connect = force_tenant or tenant['id']
+        if six.PY2 and isinstance(tenant_to_connect, six.text_type):
+            tenant_to_connect = tenant_to_connect.encode('utf-8')
+        self._logger.debug('Connecting with tenant: %s', tenant_to_connect)
+        new_tenant_name = '%s%s' % (self._reseller_prefix, tenant_to_connect)
+        environ['PATH_INFO'] = environ['PATH_INFO'].replace(account,
+                                                            new_tenant_name)
+        return self._app(environ, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+    """Returns a WSGI filter app for use with paste.deploy."""
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def auth_filter(app):
+        return S3Token(app, conf)
+    return auth_filter

swift/common/middleware/s3api/schema/access_control_policy.rng

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <include href="common.rng"/>
+  <start>
+    <element name="AccessControlPolicy">
+      <interleave>
+        <element name="Owner">
+          <ref name="CanonicalUser"/>
+        </element>
+        <element name="AccessControlList">
+          <ref name="AccessControlList"/>
+        </element>
+      </interleave>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/bucket_logging_status.rng

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="BucketLoggingStatus">
+      <optional>
+        <element name="LoggingEnabled">
+          <interleave>
+            <element name="TargetBucket">
+              <data type="string"/>
+            </element>
+            <element name="TargetPrefix">
+              <data type="string"/>
+            </element>
+            <optional>
+              <element name="TargetGrants">
+                <ref name="AccessControlList"/>
+              </element>
+            </optional>
+          </interleave>
+        </element>
+      </optional>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/common.rng

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <define name="CanonicalUser">
+    <interleave>
+      <element name="ID">
+        <data type="string"/>
+      </element>
+      <optional>
+        <element name="DisplayName">
+          <data type="string"/>
+        </element>
+      </optional>
+    </interleave>
+  </define>
+  <define name="StorageClass">
+    <choice>
+      <value>STANDARD</value>
+      <value>REDUCED_REDUNDANCY</value>
+      <value>GLACIER</value>
+      <value>UNKNOWN</value>
+    </choice>
+  </define>
+  <define name="AccessControlList">
+    <zeroOrMore>
+      <element name="Grant">
+        <interleave>
+          <element name="Grantee">
+            <choice>
+              <group>
+                <attribute name="xsi:type">
+                  <value>AmazonCustomerByEmail</value>
+                </attribute>
+                <element name="EmailAddress">
+                  <data type="string"/>
+                </element>
+              </group>
+              <group>
+                <attribute name="xsi:type">
+                  <value>CanonicalUser</value>
+                </attribute>
+                <ref name="CanonicalUser"/>
+              </group>
+              <group>
+                <attribute name="xsi:type">
+                  <value>Group</value>
+                </attribute>
+                <element name="URI">
+                  <data type="string"/>
+                </element>
+              </group>
+            </choice>
+          </element>
+          <element name="Permission">
+            <choice>
+              <value>READ</value>
+              <value>WRITE</value>
+              <value>READ_ACP</value>
+              <value>WRITE_ACP</value>
+              <value>FULL_CONTROL</value>
+            </choice>
+          </element>
+        </interleave>
+      </element>
+    </zeroOrMore>
+  </define>
+</grammar>

swift/common/middleware/s3api/schema/complete_multipart_upload.rng

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="CompleteMultipartUpload">
+      <oneOrMore>
+        <element name="Part">
+          <interleave>
+            <element name="PartNumber">
+              <data type="int"/>
+            </element>
+            <element name="ETag">
+              <data type="string"/>
+            </element>
+          </interleave>
+        </element>
+      </oneOrMore>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/complete_multipart_upload_result.rng

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="CompleteMultipartUploadResult">
+      <element name="Location">
+        <data type="anyURI"/>
+      </element>
+      <element name="Bucket">
+        <data type="string"/>
+      </element>
+      <element name="Key">
+        <data type="string"/>
+      </element>
+      <element name="ETag">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/copy_object_result.rng

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="CopyObjectResult">
+      <element name="LastModified">
+        <data type="dateTime"/>
+      </element>
+      <element name="ETag">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/copy_part_result.rng

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="CopyPartResult">
+      <element name="LastModified">
+        <data type="dateTime"/>
+      </element>
+      <element name="ETag">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/create_bucket_configuration.rng

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element>
+      <anyName/>
+      <element name="LocationConstraint">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/delete.rng

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="Delete">
+      <interleave>
+        <optional>
+          <element name="Quiet">
+            <data type="boolean"/>
+          </element>
+        </optional>
+        <oneOrMore>
+          <element name="Object">
+            <interleave>
+              <element name="Key">
+                <data type="string"/>
+              </element>
+              <optional>
+                <element name="VersionId">
+                  <data type="string"/>
+                </element>
+              </optional>
+            </interleave>
+          </element>
+        </oneOrMore>
+      </interleave>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/delete_result.rng

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="DeleteResult">
+      <zeroOrMore>
+        <choice>
+          <element name="Deleted">
+            <element name="Key">
+              <data type="string"/>
+            </element>
+            <optional>
+              <element name="VersionId">
+                <data type="string"/>
+              </element>
+            </optional>
+            <optional>
+              <element name="DeleteMarker">
+                <data type="boolean"/>
+              </element>
+            </optional>
+            <optional>
+              <element name="DeleteMarkerVersionId">
+                <data type="string"/>
+              </element>
+            </optional>
+          </element>
+          <element name="Error">
+            <element name="Key">
+              <data type="string"/>
+            </element>
+            <optional>
+              <element name="VersionId">
+                <data type="string"/>
+              </element>
+            </optional>
+            <element name="Code">
+              <data type="string"/>
+            </element>
+            <element name="Message">
+              <data type="string"/>
+            </element>
+          </element>
+        </choice>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/error.rng

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="Error">
+      <element name="Code">
+        <data type="string"/>
+      </element>
+      <element name="Message">
+        <data type="string"/>
+      </element>
+      <zeroOrMore>
+        <ref name="DebugInfo"/>
+      </zeroOrMore>
+    </element>
+  </start>
+  <define name="DebugInfo">
+    <element>
+      <anyName/>
+      <zeroOrMore>
+        <choice>
+          <attribute>
+            <anyName/>
+          </attribute>
+          <text/>
+          <ref name="DebugInfo"/>
+        </choice>
+      </zeroOrMore>
+    </element>
+  </define>
+</grammar>

swift/common/middleware/s3api/schema/initiate_multipart_upload_result.rng

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="InitiateMultipartUploadResult">
+      <element name="Bucket">
+        <data type="string"/>
+      </element>
+      <element name="Key">
+        <data type="string"/>
+      </element>
+      <element name="UploadId">
+        <data type="string"/>
+      </element>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/lifecycle_configuration.rng

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="LifecycleConfiguration">
+      <oneOrMore>
+        <element name="Rule">
+          <interleave>
+            <optional>
+              <element name="ID">
+                <data type="string"/>
+              </element>
+            </optional>
+            <element name="Prefix">
+              <data type="string"/>
+            </element>
+            <element name="Status">
+              <choice>
+                <value>Enabled</value>
+                <value>Disabled</value>
+              </choice>
+            </element>
+            <optional>
+              <element name="Transition">
+                <ref name="Transition"/>
+              </element>
+            </optional>
+            <optional>
+              <element name="Expiration">
+                <ref name="Expiration"/>
+              </element>
+            </optional>
+          </interleave>
+        </element>
+      </oneOrMore>
+    </element>
+  </start>
+  <define name="Expiration">
+    <choice>
+      <element name="Days">
+        <data type="int"/>
+      </element>
+      <element name="Date">
+        <data type="dateTime"/>
+      </element>
+    </choice>
+  </define>
+  <define name="Transition">
+    <interleave>
+      <ref name="Expiration"/>
+      <element name="StorageClass">
+        <ref name="StorageClass"/>
+      </element>
+    </interleave>
+  </define>
+</grammar>

swift/common/middleware/s3api/schema/list_all_my_buckets_result.rng

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListAllMyBucketsResult">
+      <element name="Owner">
+        <ref name="CanonicalUser"/>
+      </element>
+      <element name="Buckets">
+        <zeroOrMore>
+          <element name="Bucket">
+            <element name="Name">
+              <data type="string"/>
+            </element>
+            <element name="CreationDate">
+              <data type="dateTime"/>
+            </element>
+          </element>
+        </zeroOrMore>
+      </element>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/list_bucket_result.rng

@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListBucketResult">
+      <element name="Name">
+        <data type="string"/>
+      </element>
+      <element name="Prefix">
+        <data type="string"/>
+      </element>
+      <choice>
+        <group>
+          <element name="Marker">
+            <data type="string"/>
+          </element>
+          <optional>
+            <element name="NextMarker">
+              <data type="string"/>
+            </element>
+          </optional>
+        </group>
+        <group>
+          <optional>
+            <element name="NextContinuationToken">
+              <data type="string"/>
+            </element>
+          </optional>
+          <optional>
+            <element name="ContinuationToken">
+              <data type="string"/>
+            </element>
+          </optional>
+          <optional>
+            <element name="StartAfter">
+              <data type="string"/>
+            </element>
+          </optional>
+          <element name="KeyCount">
+            <data type="int"/>
+          </element>
+        </group>
+      </choice>
+      <element name="MaxKeys">
+        <data type="int"/>
+      </element>
+      <optional>
+        <element name="EncodingType">
+          <data type="string"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="Delimiter">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="IsTruncated">
+        <data type="boolean"/>
+      </element>
+      <zeroOrMore>
+        <element name="Contents">
+          <element name="Key">
+            <data type="string"/>
+          </element>
+          <element name="LastModified">
+            <data type="dateTime"/>
+          </element>
+          <element name="ETag">
+            <data type="string"/>
+          </element>
+          <element name="Size">
+            <data type="long"/>
+          </element>
+          <optional>
+            <element name="Owner">
+              <ref name="CanonicalUser"/>
+            </element>
+          </optional>
+          <element name="StorageClass">
+            <ref name="StorageClass"/>
+          </element>
+        </element>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="CommonPrefixes">
+          <element name="Prefix">
+            <data type="string"/>
+          </element>
+        </element>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/list_multipart_uploads_result.rng

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListMultipartUploadsResult">
+      <element name="Bucket">
+        <data type="string"/>
+      </element>
+      <element name="KeyMarker">
+        <data type="string"/>
+      </element>
+      <element name="UploadIdMarker">
+        <data type="string"/>
+      </element>
+      <element name="NextKeyMarker">
+        <data type="string"/>
+      </element>
+      <element name="NextUploadIdMarker">
+        <data type="string"/>
+      </element>
+      <optional>
+        <element name="Delimiter">
+          <data type="string"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="Prefix">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="MaxUploads">
+        <data type="int"/>
+      </element>
+      <optional>
+        <element name="EncodingType">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="IsTruncated">
+        <data type="boolean"/>
+      </element>
+      <zeroOrMore>
+        <element name="Upload">
+          <element name="Key">
+            <data type="string"/>
+          </element>
+          <element name="UploadId">
+            <data type="string"/>
+          </element>
+          <element name="Initiator">
+            <ref name="CanonicalUser"/>
+          </element>
+          <element name="Owner">
+            <ref name="CanonicalUser"/>
+          </element>
+          <element name="StorageClass">
+            <ref name="StorageClass"/>
+          </element>
+          <element name="Initiated">
+            <data type="dateTime"/>
+          </element>
+        </element>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="CommonPrefixes">
+          <element name="Prefix">
+            <data type="string"/>
+          </element>
+        </element>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/list_parts_result.rng

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListPartsResult">
+      <element name="Bucket">
+        <data type="string"/>
+      </element>
+      <element name="Key">
+        <data type="string"/>
+      </element>
+      <element name="UploadId">
+        <data type="string"/>
+      </element>
+      <element name="Initiator">
+        <ref name="CanonicalUser"/>
+      </element>
+      <element name="Owner">
+        <ref name="CanonicalUser"/>
+      </element>
+      <element name="StorageClass">
+        <ref name="StorageClass"/>
+      </element>
+      <element name="PartNumberMarker">
+        <data type="int"/>
+      </element>
+      <element name="NextPartNumberMarker">
+        <data type="int"/>
+      </element>
+      <element name="MaxParts">
+        <data type="int"/>
+      </element>
+      <optional>
+        <element name="EncodingType">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="IsTruncated">
+        <data type="boolean"/>
+      </element>
+      <zeroOrMore>
+        <element name="Part">
+          <element name="PartNumber">
+            <data type="int"/>
+          </element>
+          <element name="LastModified">
+            <data type="dateTime"/>
+          </element>
+          <element name="ETag">
+            <data type="string"/>
+          </element>
+          <element name="Size">
+            <data type="long"/>
+          </element>
+        </element>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/list_versions_result.rng

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="common.rng"/>
+  <start>
+    <element name="ListVersionsResult">
+      <element name="Name">
+        <data type="string"/>
+      </element>
+      <element name="Prefix">
+        <data type="string"/>
+      </element>
+      <element name="KeyMarker">
+        <data type="string"/>
+      </element>
+      <element name="VersionIdMarker">
+        <data type="string"/>
+      </element>
+      <optional>
+        <element name="NextKeyMarker">
+          <data type="string"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="NextVersionIdMarker">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="MaxKeys">
+        <data type="int"/>
+      </element>
+      <optional>
+        <element name="EncodingType">
+          <data type="string"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="Delimiter">
+          <data type="string"/>
+        </element>
+      </optional>
+      <element name="IsTruncated">
+        <data type="boolean"/>
+      </element>
+      <zeroOrMore>
+        <choice>
+          <element name="Version">
+            <element name="Key">
+              <data type="string"/>
+            </element>
+            <element name="VersionId">
+              <data type="string"/>
+            </element>
+            <element name="IsLatest">
+              <data type="boolean"/>
+            </element>
+            <element name="LastModified">
+              <data type="dateTime"/>
+            </element>
+            <element name="ETag">
+              <data type="string"/>
+            </element>
+            <element name="Size">
+              <data type="long"/>
+            </element>
+            <optional>
+              <element name="Owner">
+                <ref name="CanonicalUser"/>
+              </element>
+            </optional>
+            <element name="StorageClass">
+              <ref name="StorageClass"/>
+            </element>
+          </element>
+          <element name="DeleteMarker">
+            <element name="Key">
+              <data type="string"/>
+            </element>
+            <element name="VersionId">
+              <data type="string"/>
+            </element>
+            <element name="IsLatest">
+              <data type="boolean"/>
+            </element>
+            <element name="LastModified">
+              <data type="dateTime"/>
+            </element>
+            <optional>
+              <element name="Owner">
+                <ref name="CanonicalUser"/>
+              </element>
+            </optional>
+          </element>
+        </choice>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="CommonPrefixes">
+          <element name="Prefix">
+            <data type="string"/>
+          </element>
+        </element>
+      </zeroOrMore>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/location_constraint.rng

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <start>
+    <element name="LocationConstraint">
+      <data type="string"/>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/schema/versioning_configuration.rng

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <start>
+    <element name="VersioningConfiguration">
+      <interleave>
+        <optional>
+          <element name="Status">
+            <choice>
+              <value>Enabled</value>
+              <value>Suspended</value>
+            </choice>
+          </element>
+        </optional>
+        <optional>
+          <element name="MfaDelete">
+            <choice>
+              <value>Enabled</value>
+              <value>Disabled</value>
+            </choice>
+          </element>
+        </optional>
+      </interleave>
+    </element>
+  </start>
+</grammar>

swift/common/middleware/s3api/subresource.py

@@ -0,0 +1,563 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+---------------------------
+s3api's ACLs implementation
+---------------------------
+s3api uses a different implementation approach to achieve S3 ACLs.
+
+First, we should understand what we have to design to achieve real S3 ACLs.
+Current s3api(real S3)'s ACLs Model is as follows::
+
+    AccessControlPolicy:
+        Owner:
+        AccessControlList:
+            Grant[n]:
+                (Grantee, Permission)
+
+Each bucket or object has its own acl consisting of Owner and
+AcessControlList. AccessControlList can contain some Grants.
+By default, AccessControlList has only one Grant to allow FULL
+CONTROLL to owner. Each Grant includes single pair with Grantee,
+Permission. Grantee is the user (or user group) allowed the given permission.
+
+This module defines the groups and the relation tree.
+
+If you wanna get more information about S3's ACLs model in detail,
+please see official documentation here,
+
+http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html
+
+"""
+from functools import partial
+
+from swift.common.utils import json
+
+from swift.common.middleware.s3api.s3response import InvalidArgument, \
+    MalformedACLError, S3NotImplemented, InvalidRequest, AccessDenied
+from swift.common.middleware.s3api.etree import Element, SubElement, tostring
+from swift.common.middleware.s3api.utils import sysmeta_header
+from swift.common.middleware.s3api.exception import InvalidSubresource
+
+XMLNS_XSI = 'http://www.w3.org/2001/XMLSchema-instance'
+PERMISSIONS = ['FULL_CONTROL', 'READ', 'WRITE', 'READ_ACP', 'WRITE_ACP']
+LOG_DELIVERY_USER = '.log_delivery'
+
+
+def encode_acl(resource, acl):
+    """
+    Encode an ACL instance to Swift metadata.
+
+    Given a resource type and an ACL instance, this method returns HTTP
+    headers, which can be used for Swift metadata.
+    """
+    header_value = {"Owner": acl.owner.id}
+    grants = []
+    for grant in acl.grants:
+        grant = {"Permission": grant.permission,
+                 "Grantee": str(grant.grantee)}
+        grants.append(grant)
+    header_value.update({"Grant": grants})
+    headers = {}
+    key = sysmeta_header(resource, 'acl')
+    headers[key] = json.dumps(header_value, separators=(',', ':'))
+
+    return headers
+
+
+def decode_acl(resource, headers, allow_no_owner):
+    """
+    Decode Swift metadata to an ACL instance.
+
+    Given a resource type and HTTP headers, this method returns an ACL
+    instance.
+    """
+    value = ''
+
+    key = sysmeta_header(resource, 'acl')
+    if key in headers:
+        value = headers[key]
+
+    if value == '':
+        # Fix me: In the case of value is empty or not dict instance,
+        # I want an instance of Owner as None.
+        # However, in the above process would occur error in reference
+        # to an instance variable of Owner.
+        return ACL(Owner(None, None), [], True, allow_no_owner)
+
+    try:
+        encode_value = json.loads(value)
+        if not isinstance(encode_value, dict):
+            return ACL(Owner(None, None), [], True, allow_no_owner)
+
+        id = None
+        name = None
+        grants = []
+        if 'Owner' in encode_value:
+            id = encode_value['Owner']
+            name = encode_value['Owner']
+        if 'Grant' in encode_value:
+            for grant in encode_value['Grant']:
+                grantee = None
+                # pylint: disable-msg=E1101
+                for group in Group.__subclasses__():
+                    if group.__name__ == grant['Grantee']:
+                        grantee = group()
+                if not grantee:
+                    grantee = User(grant['Grantee'])
+                permission = grant['Permission']
+                grants.append(Grant(grantee, permission))
+        return ACL(Owner(id, name), grants, True, allow_no_owner)
+    except Exception as e:
+        raise InvalidSubresource((resource, 'acl', value), e)
+
+
+class Grantee(object):
+    """
+    Base class for grantee.
+
+    Methods:
+
+    * init: create a Grantee instance
+    * elem: create an ElementTree from itself
+
+    Static Methods:
+
+    * from_header: convert a grantee string in the HTTP header
+                   to an Grantee instance.
+    * from_elem: convert a ElementTree to an Grantee instance.
+
+    """
+    # Needs confirmation whether we really need these methods or not.
+    # * encode (method): create a JSON which includes whole own elements
+    # * encode_from_elem (static method): convert from an ElementTree to a JSON
+    # * elem_from_json (static method): convert from a JSON to an ElementTree
+    # * from_json (static method): convert a Json string to an Grantee
+    #                              instance.
+
+    def __contains__(self, key):
+        """
+        The key argument is a S3 user id.  This method checks that the user id
+        belongs to this class.
+        """
+        raise S3NotImplemented()
+
+    def elem(self):
+        """
+        Get an etree element of this instance.
+        """
+        raise S3NotImplemented()
+
+    @staticmethod
+    def from_elem(elem):
+        type = elem.get('{%s}type' % XMLNS_XSI)
+        if type == 'CanonicalUser':
+            value = elem.find('./ID').text
+            return User(value)
+        elif type == 'Group':
+            value = elem.find('./URI').text
+            subclass = get_group_subclass_from_uri(value)
+            return subclass()
+        elif type == 'AmazonCustomerByEmail':
+            raise S3NotImplemented()
+        else:
+            raise MalformedACLError()
+
+    @staticmethod
+    def from_header(grantee):
+        """
+        Convert a grantee string in the HTTP header to an Grantee instance.
+        """
+        type, value = grantee.split('=', 1)
+        value = value.strip('"\'')
+        if type == 'id':
+            return User(value)
+        elif type == 'emailAddress':
+            raise S3NotImplemented()
+        elif type == 'uri':
+            # return a subclass instance of Group class
+            subclass = get_group_subclass_from_uri(value)
+            return subclass()
+        else:
+            raise InvalidArgument(type, value,
+                                  'Argument format not recognized')
+
+
+class User(Grantee):
+    """
+    Canonical user class for S3 accounts.
+    """
+    type = 'CanonicalUser'
+
+    def __init__(self, name):
+        self.id = name
+        self.display_name = name
+
+    def __contains__(self, key):
+        return key == self.id
+
+    def elem(self):
+        elem = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
+        elem.set('{%s}type' % XMLNS_XSI, self.type)
+        SubElement(elem, 'ID').text = self.id
+        SubElement(elem, 'DisplayName').text = self.display_name
+        return elem
+
+    def __str__(self):
+        return self.display_name
+
+
+class Owner(object):
+    """
+    Owner class for S3 accounts
+    """
+    def __init__(self, id, name):
+        self.id = id
+        self.name = name
+
+
+def get_group_subclass_from_uri(uri):
+    """
+    Convert a URI to one of the predefined groups.
+    """
+    for group in Group.__subclasses__():  # pylint: disable-msg=E1101
+        if group.uri == uri:
+            return group
+    raise InvalidArgument('uri', uri, 'Invalid group uri')
+
+
+class Group(Grantee):
+    """
+    Base class for Amazon S3 Predefined Groups
+    """
+    type = 'Group'
+    uri = ''
+
+    def __init__(self):
+        # Initialize method to clarify this has nothing to do
+        pass
+
+    def elem(self):
+        elem = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
+        elem.set('{%s}type' % XMLNS_XSI, self.type)
+        SubElement(elem, 'URI').text = self.uri
+
+        return elem
+
+    def __str__(self):
+        return self.__class__.__name__
+
+
+def canned_acl_grantees(bucket_owner, object_owner=None):
+    """
+    A set of predefined grants supported by AWS S3.
+    """
+    owner = object_owner or bucket_owner
+
+    return {
+        'private': [
+            ('FULL_CONTROL', User(owner.name)),
+        ],
+        'public-read': [
+            ('READ', AllUsers()),
+            ('FULL_CONTROL', User(owner.name)),
+        ],
+        'public-read-write': [
+            ('READ', AllUsers()),
+            ('WRITE', AllUsers()),
+            ('FULL_CONTROL', User(owner.name)),
+        ],
+        'authenticated-read': [
+            ('READ', AuthenticatedUsers()),
+            ('FULL_CONTROL', User(owner.name)),
+        ],
+        'bucket-owner-read': [
+            ('READ', User(bucket_owner.name)),
+            ('FULL_CONTROL', User(owner.name)),
+        ],
+        'bucket-owner-full-control': [
+            ('FULL_CONTROL', User(owner.name)),
+            ('FULL_CONTROL', User(bucket_owner.name)),
+        ],
+        'log-delivery-write': [
+            ('WRITE', LogDelivery()),
+            ('READ_ACP', LogDelivery()),
+            ('FULL_CONTROL', User(owner.name)),
+        ],
+    }
+
+
+class AuthenticatedUsers(Group):
+    """
+    This group represents all AWS accounts.  Access permission to this group
+    allows any AWS account to access the resource.  However, all requests must
+    be signed (authenticated).
+    """
+    uri = 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'
+
+    def __contains__(self, key):
+        # s3api handles only signed requests.
+        return True
+
+
+class AllUsers(Group):
+    """
+    Access permission to this group allows anyone to access the resource.  The
+    requests can be signed (authenticated) or unsigned (anonymous).  Unsigned
+    requests omit the Authentication header in the request.
+
+    Note: s3api regards unsigned requests as Swift API accesses, and bypasses
+    them to Swift.  As a result, AllUsers behaves completely same as
+    AuthenticatedUsers.
+    """
+    uri = 'http://acs.amazonaws.com/groups/global/AllUsers'
+
+    def __contains__(self, key):
+        return True
+
+
+class LogDelivery(Group):
+    """
+    WRITE and READ_ACP permissions on a bucket enables this group to write
+    server access logs to the bucket.
+    """
+    uri = 'http://acs.amazonaws.com/groups/s3/LogDelivery'
+
+    def __contains__(self, key):
+        if ':' in key:
+            tenant, user = key.split(':', 1)
+        else:
+            user = key
+        return user == LOG_DELIVERY_USER
+
+
+class Grant(object):
+    """
+    Grant Class which includes both Grantee and Permission
+    """
+
+    def __init__(self, grantee, permission):
+        """
+        :param grantee: a grantee class or its subclass
+        :param permission: string
+        """
+        if permission.upper() not in PERMISSIONS:
+            raise S3NotImplemented()
+        if not isinstance(grantee, Grantee):
+            raise ValueError()
+        self.grantee = grantee
+        self.permission = permission
+
+    @classmethod
+    def from_elem(cls, elem):
+        """
+        Convert an ElementTree to an ACL instance
+        """
+        grantee = Grantee.from_elem(elem.find('./Grantee'))
+        permission = elem.find('./Permission').text
+        return cls(grantee, permission)
+
+    def elem(self):
+        """
+        Create an etree element.
+        """
+        elem = Element('Grant')
+        elem.append(self.grantee.elem())
+        SubElement(elem, 'Permission').text = self.permission
+
+        return elem
+
+    def allow(self, grantee, permission):
+        return permission == self.permission and grantee in self.grantee
+
+
+class ACL(object):
+    """
+    S3 ACL class.
+
+    Refs (S3 API - acl-overview:
+          http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html):
+
+    The sample ACL includes an Owner element identifying the owner via the
+    AWS account's canonical user ID. The Grant element identifies the grantee
+    (either an AWS account or a predefined group), and the permission granted.
+    This default ACL has one Grant element for the owner. You grant permissions
+    by adding Grant elements, each grant identifying the grantee and the
+    permission.
+    """
+    metadata_name = 'acl'
+    root_tag = 'AccessControlPolicy'
+    max_xml_length = 200 * 1024
+
+    def __init__(self, owner, grants=None, s3_acl=False, allow_no_owner=False):
+        """
+        :param owner: Owner instance for ACL instance
+        :param grants: a list of Grant instances
+        :param s3_acl: boolean indicates whether this class is used under
+            s3_acl is True or False (from s3api middleware configuration)
+        :param allow_no_owner: boolean indicates this ACL instance can be
+            handled when no owner information found
+        """
+        self.owner = owner
+        self.grants = grants or []
+        self.s3_acl = s3_acl
+        self.allow_no_owner = allow_no_owner
+
+    def __repr__(self):
+        return tostring(self.elem())
+
+    @classmethod
+    def from_elem(cls, elem, s3_acl=False, allow_no_owner=False):
+        """
+        Convert an ElementTree to an ACL instance
+        """
+        id = elem.find('./Owner/ID').text
+        try:
+            name = elem.find('./Owner/DisplayName').text
+        except AttributeError:
+            name = id
+
+        grants = [Grant.from_elem(e)
+                  for e in elem.findall('./AccessControlList/Grant')]
+        return cls(Owner(id, name), grants, s3_acl, allow_no_owner)
+
+    def elem(self):
+        """
+        Decode the value to an ACL instance.
+        """
+        elem = Element(self.root_tag)
+
+        owner = SubElement(elem, 'Owner')
+        SubElement(owner, 'ID').text = self.owner.id
+        SubElement(owner, 'DisplayName').text = self.owner.name
+
+        SubElement(elem, 'AccessControlList').extend(
+            g.elem() for g in self.grants
+        )
+
+        return elem
+
+    def check_owner(self, user_id):
+        """
+        Check that the user is an owner.
+        """
+        if not self.s3_acl:
+            # Ignore S3api ACL.
+            return
+
+        if not self.owner.id:
+            if self.allow_no_owner:
+                # No owner means public.
+                return
+            raise AccessDenied()
+
+        if user_id != self.owner.id:
+            raise AccessDenied()
+
+    def check_permission(self, user_id, permission):
+        """
+        Check that the user has a permission.
+        """
+        if not self.s3_acl:
+            # Ignore S3api ACL.
+            return
+
+        try:
+            # owners have full control permission
+            self.check_owner(user_id)
+            return
+        except AccessDenied:
+            pass
+
+        if permission in PERMISSIONS:
+            for g in self.grants:
+                if g.allow(user_id, 'FULL_CONTROL') or \
+                        g.allow(user_id, permission):
+                    return
+
+        raise AccessDenied()
+
+    @classmethod
+    def from_headers(cls, headers, bucket_owner, object_owner=None,
+                     as_private=True):
+        """
+        Convert HTTP headers to an ACL instance.
+        """
+        grants = []
+        try:
+            for key, value in headers.items():
+                if key.lower().startswith('x-amz-grant-'):
+                    permission = key[len('x-amz-grant-'):]
+                    permission = permission.upper().replace('-', '_')
+                    if permission not in PERMISSIONS:
+                        continue
+                    for grantee in value.split(','):
+                        grants.append(
+                            Grant(Grantee.from_header(grantee), permission))
+
+            if 'x-amz-acl' in headers:
+                try:
+                    acl = headers['x-amz-acl']
+                    if len(grants) > 0:
+                        err_msg = 'Specifying both Canned ACLs and Header ' \
+                            'Grants is not allowed'
+                        raise InvalidRequest(err_msg)
+                    grantees = canned_acl_grantees(
+                        bucket_owner, object_owner)[acl]
+                    for permission, grantee in grantees:
+                        grants.append(Grant(grantee, permission))
+                except KeyError:
+                    # expects canned_acl_grantees()[] raises KeyError
+                    raise InvalidArgument('x-amz-acl', headers['x-amz-acl'])
+        except (KeyError, ValueError):
+            # TODO: think about we really catch this except sequence
+            raise InvalidRequest()
+
+        if len(grants) == 0:
+            # No ACL headers
+            if as_private:
+                return ACLPrivate(bucket_owner, object_owner)
+            else:
+                return None
+
+        return cls(object_owner or bucket_owner, grants)
+
+
+class CannedACL(object):
+    """
+    A dict-like object that returns canned ACL.
+    """
+    def __getitem__(self, key):
+        def acl(key, bucket_owner, object_owner=None,
+                s3_acl=False, allow_no_owner=False):
+            grants = []
+            grantees = canned_acl_grantees(bucket_owner, object_owner)[key]
+            for permission, grantee in grantees:
+                grants.append(Grant(grantee, permission))
+            return ACL(object_owner or bucket_owner,
+                       grants, s3_acl, allow_no_owner)
+
+        return partial(acl, key)
+
+
+canned_acl = CannedACL()
+
+ACLPrivate = canned_acl['private']
+ACLPublicRead = canned_acl['public-read']
+ACLPublicReadWrite = canned_acl['public-read-write']
+ACLAuthenticatedRead = canned_acl['authenticated-read']
+ACLBucketOwnerRead = canned_acl['bucket-owner-read']
+ACLBucketOwnerFullControl = canned_acl['bucket-owner-full-control']
+ACLLogDeliveryWrite = canned_acl['log-delivery-write']

swift/common/middleware/s3api/utils.py

@@ -0,0 +1,190 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import calendar
+import email.utils
+import re
+import time
+import uuid
+
+# Need for check_path_header
+from swift.common import utils
+
+MULTIUPLOAD_SUFFIX = '+segments'
+
+
+def sysmeta_prefix(resource):
+    """
+    Returns the system metadata prefix for given resource type.
+    """
+    if resource.lower() == 'object':
+        return 'x-object-sysmeta-s3api-'
+    else:
+        return 'x-container-sysmeta-s3api-'
+
+
+def sysmeta_header(resource, name):
+    """
+    Returns the system metadata header for given resource type and name.
+    """
+    return sysmeta_prefix(resource) + name
+
+
+def camel_to_snake(camel):
+    return re.sub('(.)([A-Z])', r'\1_\2', camel).lower()
+
+
+def snake_to_camel(snake):
+    return snake.title().replace('_', '')
+
+
+def unique_id():
+    return base64.urlsafe_b64encode(str(uuid.uuid4()))
+
+
+def utf8encode(s):
+    if isinstance(s, unicode):
+        s = s.encode('utf8')
+    return s
+
+
+def utf8decode(s):
+    if isinstance(s, str):
+        s = s.decode('utf8')
+    return s
+
+
+def validate_bucket_name(name, dns_compliant_bucket_names):
+        """
+        Validates the name of the bucket against S3 criteria,
+        http://docs.amazonwebservices.com/AmazonS3/latest/BucketRestrictions.html
+        True is valid, False is invalid.
+        """
+        valid_chars = '-.a-z0-9'
+        if not dns_compliant_bucket_names:
+            valid_chars += 'A-Z_'
+        max_len = 63 if dns_compliant_bucket_names else 255
+
+        if len(name) < 3 or len(name) > max_len or not name[0].isalnum():
+            # Bucket names should be between 3 and 63 (or 255) characters long
+            # Bucket names must start with a letter or a number
+            return False
+        elif dns_compliant_bucket_names and (
+                '.-' in name or '-.' in name or '..' in name or
+                not name[-1].isalnum()):
+            # Bucket names cannot contain dashes next to periods
+            # Bucket names cannot contain two adjacent periods
+            # Bucket names must end with a letter or a number
+            return False
+        elif name.endswith('.'):
+            # Bucket names must not end with dot
+            return False
+        elif re.match("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)"
+                      "{3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$",
+                      name):
+            # Bucket names cannot be formatted as an IP Address
+            return False
+        elif not re.match("^[%s]*$" % valid_chars, name):
+            # Bucket names can contain lowercase letters, numbers, and hyphens.
+            return False
+        else:
+            return True
+
+
+class S3Timestamp(utils.Timestamp):
+    @property
+    def s3xmlformat(self):
+        return self.isoformat[:-7] + '.000Z'
+
+    @property
+    def amz_date_format(self):
+        """
+        this format should be like 'YYYYMMDDThhmmssZ'
+        """
+        return self.isoformat.replace(
+            '-', '').replace(':', '')[:-7] + 'Z'
+
+    @classmethod
+    def now(cls):
+        return cls(time.time())
+
+
+def mktime(timestamp_str, time_format='%Y-%m-%dT%H:%M:%S'):
+    """
+    mktime creates a float instance in epoch time really like as time.mktime
+
+    the difference from time.mktime is allowing to 2 formats string for the
+    argument for the S3 testing usage.
+    TODO: support
+
+    :param timestamp_str: a string of timestamp formatted as
+                          (a) RFC2822 (e.g. date header)
+                          (b) %Y-%m-%dT%H:%M:%S (e.g. copy result)
+    :param time_format: a string of format to parse in (b) process
+    :return : a float instance in epoch time
+    """
+    # time_tuple is the *remote* local time
+    time_tuple = email.utils.parsedate_tz(timestamp_str)
+    if time_tuple is None:
+        time_tuple = time.strptime(timestamp_str, time_format)
+        # add timezone info as utc (no time difference)
+        time_tuple += (0, )
+
+    # We prefer calendar.gmtime and a manual adjustment over
+    # email.utils.mktime_tz because older versions of Python (<2.7.4) may
+    # double-adjust for timezone in some situations (such when swift changes
+    # os.environ['TZ'] without calling time.tzset()).
+    epoch_time = calendar.timegm(time_tuple) - time_tuple[9]
+
+    return epoch_time
+
+
+class Config(dict):
+    def __init__(self, base=None):
+        if base is not None:
+            self.update(base)
+
+    def __getattr__(self, name):
+        if name not in self:
+            raise AttributeError("No attribute '%s'" % name)
+
+        return self[name]
+
+    def __setattr__(self, name, value):
+        self[name] = value
+
+    def __delattr__(self, name):
+        del self[name]
+
+    def update(self, other):
+        if hasattr(other, 'keys'):
+            for key in other.keys():
+                self[key] = other[key]
+        else:
+            for key, value in other:
+                self[key] = value
+
+    def __setitem__(self, key, value):
+        if isinstance(self.get(key), bool):
+            dict.__setitem__(self, key, utils.config_true_value(value))
+        elif isinstance(self.get(key), int):
+            try:
+                dict.__setitem__(self, key, int(value))
+            except ValueError:
+                if value:  # No need to raise the error if value is ''
+                    raise
+        else:
+            dict.__setitem__(self, key, value)

swift/common/middleware/tempauth.py

@@ -273,7 +273,7 @@ class TempAuth(object):
             return self.app(env, start_response)
         if env.get('PATH_INFO', '').startswith(self.auth_prefix):
             return self.handle(env, start_response)
-        s3 = env.get('swift3.auth_details')
+        s3 = env.get('s3api.auth_details')
         token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))
         service_token = env.get('HTTP_X_SERVICE_TOKEN')
         if s3 or (token and token.startswith(self.reseller_prefix)):
@@ -435,7 +435,7 @@ class TempAuth(object):
             else:
                 groups = groups.encode('utf8')
 
-        s3_auth_details = env.get('swift3.auth_details')
+        s3_auth_details = env.get('s3api.auth_details')
         if s3_auth_details:
             if 'check_signature' not in s3_auth_details:
                 self.logger.warning(

test-requirements.txt

@@ -12,6 +12,12 @@ os-testr>=0.8.0 # Apache-2.0
 mock>=2.0 # BSD
 python-swiftclient
 python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0
+reno>=1.8.0  # Apache-2.0
+python-openstackclient
+boto
+requests-mock>=1.2.0 # Apache-2.0
+fixtures>=3.0.0 # Apache-2.0/BSD
+keystonemiddleware>=4.17.0 # Apache-2.0
 
 # Security checks
 bandit>=1.1.0 # Apache-2.0

test/functional/__init__.py

@@ -412,6 +412,46 @@ def _load_domain_remap_staticweb(proxy_conf_file, swift_conf_file, **kwargs):
     return test_conf_file, swift_conf_file
 
 
+def _load_s3api(proxy_conf_file, swift_conf_file, **kwargs):
+    """
+    Load s3api configuration and override proxy-server.conf contents.
+
+    :param proxy_conf_file: Source proxy conf filename
+    :param swift_conf_file: Source swift conf filename
+    :returns: Tuple of paths to the proxy conf file and swift conf file to use
+    :raises InProcessException: raised if proxy conf contents are invalid
+    """
+    _debug('Setting configuration for s3api')
+
+    # The global conf dict cannot be used to modify the pipeline.
+    # The pipeline loader requires the pipeline to be set in the local_conf.
+    # If pipeline is set in the global conf dict (which in turn populates the
+    # DEFAULTS options) then it prevents pipeline being loaded into the local
+    # conf during wsgi load_app.
+    # Therefore we must modify the [pipeline:main] section.
+
+    conf = ConfigParser()
+    conf.read(proxy_conf_file)
+    try:
+        section = 'pipeline:main'
+        pipeline = conf.get(section, 'pipeline')
+        pipeline = pipeline.replace(
+            "tempauth",
+            "s3api tempauth")
+        conf.set(section, 'pipeline', pipeline)
+        conf.set('filter:s3api', 's3_acl', 'true')
+    except NoSectionError as err:
+        msg = 'Error problem with proxy conf file %s: %s' % \
+              (proxy_conf_file, err)
+        raise InProcessException(msg)
+
+    test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
+    with open(test_conf_file, 'w') as fp:
+        conf.write(fp)
+
+    return test_conf_file, swift_conf_file
+
+
 # Mapping from possible values of the variable
 # SWIFT_TEST_IN_PROCESS_CONF_LOADER
 # to the method to call for loading the associated configuration
@@ -421,6 +461,7 @@ conf_loaders = {
     'encryption': _load_encryption,
     'ec': _load_ec_as_default_policy,
     'domain_remap_staticweb': _load_domain_remap_staticweb,
+    's3api': _load_s3api,
 }
 
 
@@ -520,6 +561,12 @@ def in_process_setup(the_object_server=object_server):
         'account_autocreate': 'true',
         'allow_versions': 'True',
         'allow_versioned_writes': 'True',
+        # TODO: move this into s3api config loader because they are
+        #       required by only s3api
+        'allowed_headers':
+            "Content-Disposition, Content-Encoding, X-Delete-At, "
+            "X-Object-Manifest, X-Static-Large-Object, Cache-Control, "
+            "Content-Language, Expires, X-Robots-Tag",
         # Below are values used by the functional test framework, as well as
         # by the various in-process swift servers
         'auth_host': '127.0.0.1',
@@ -531,6 +578,8 @@ def in_process_setup(the_object_server=object_server):
         'account': 'test',
         'username': 'tester',
         'password': 'testing',
+        's3_access_key': 'test:tester',
+        's3_secret_key': 'testing',
         # User on a second account (needs admin access to the account)
         'account2': 'test2',
         'username2': 'tester2',
@@ -538,6 +587,8 @@ def in_process_setup(the_object_server=object_server):
         # User on same account as first, but without admin access
         'username3': 'tester3',
         'password3': 'testing3',
+        's3_access_key2': 'test:tester3',
+        's3_secret_key2': 'testing3',
         # Service user and prefix (emulates glance, cinder, etc. user)
         'account5': 'test5',
         'username5': 'tester5',

test/functional/s3api/__init__.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2011-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest2
+import traceback
+import test.functional as tf
+from test.functional.s3api.s3_test_client import Connection
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class S3ApiBase(unittest2.TestCase):
+    def __init__(self, method_name):
+        super(S3ApiBase, self).__init__(method_name)
+        self.method_name = method_name
+
+    def setUp(self):
+        if 's3api' not in tf.cluster_info:
+            raise tf.SkipTest('s3api middleware is not enabled')
+        try:
+            self.conn = Connection()
+            self.conn.reset()
+        except Exception:
+            message = '%s got an error during initialize process.\n\n%s' % \
+                      (self.method_name, traceback.format_exc())
+            # TODO: Find a way to make this go to FAIL instead of Error
+            self.fail(message)
+
+    def assertCommonResponseHeaders(self, headers, etag=None):
+        """
+        asserting common response headers with args
+        :param headers: a dict of response headers
+        :param etag: a string of md5(content).hexdigest() if not given,
+                     this won't assert anything about etag. (e.g. DELETE obj)
+        """
+        self.assertTrue(headers['x-amz-id-2'] is not None)
+        self.assertTrue(headers['x-amz-request-id'] is not None)
+        self.assertTrue(headers['date'] is not None)
+        # TODO; requires consideration
+        # self.assertTrue(headers['server'] is not None)
+        if etag is not None:
+            self.assertTrue('etag' in headers)  # sanity
+            self.assertEqual(etag, headers['etag'].strip('"'))

test/functional/s3api/s3_test_client.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import test.functional as tf
+from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \
+    BotoClientError, S3ResponseError
+
+RETRY_COUNT = 3
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class Connection(object):
+    """
+    Connection class used for S3 functional testing.
+    """
+    def __init__(self, aws_access_key='test:tester',
+                 aws_secret_key='testing',
+                 user_id='test:tester'):
+        """
+        Initialize method.
+
+        :param aws_access_key: a string of aws access key
+        :param aws_secret_key: a string of aws secret key
+        :param user_id: a string consists of TENANT and USER name used for
+                        asserting Owner ID (not required S3Connection)
+
+        In default, Connection class will be initialized as tester user
+        behaves as:
+        user_test_tester = testing .admin
+
+        """
+        self.aws_access_key = aws_access_key
+        self.aws_secret_key = aws_secret_key
+        self.user_id = user_id
+        # NOTE: auth_host and auth_port can be different from storage location
+        self.host = tf.config['auth_host']
+        self.port = int(tf.config['auth_port'])
+        self.conn = \
+            S3Connection(aws_access_key, aws_secret_key, is_secure=False,
+                         host=self.host, port=self.port,
+                         calling_format=OrdinaryCallingFormat())
+        self.conn.auth_region_name = 'US'
+
+    def reset(self):
+        """
+        Reset all swift environment to keep clean. As a result by calling this
+        method, we can assume the backend swift keeps no containers and no
+        objects on this connection's account.
+        """
+        exceptions = []
+        for i in range(RETRY_COUNT):
+            try:
+                buckets = self.conn.get_all_buckets()
+                if not buckets:
+                    break
+
+                for bucket in buckets:
+                    try:
+                        for upload in bucket.list_multipart_uploads():
+                            upload.cancel_upload()
+
+                        for obj in bucket.list():
+                            bucket.delete_key(obj.name)
+
+                        self.conn.delete_bucket(bucket.name)
+                    except S3ResponseError as e:
+                        # 404 means NoSuchBucket, NoSuchKey, or NoSuchUpload
+                        if e.status != 404:
+                            raise
+            except (BotoClientError, S3ResponseError) as e:
+                exceptions.append(e)
+        if exceptions:
+            # raise the first exception
+            raise exceptions.pop(0)
+
+    def make_request(self, method, bucket='', obj='', headers=None, body='',
+                     query=None):
+        """
+        Wrapper method of S3Connection.make_request.
+
+        :param method: a string of HTTP request method
+        :param bucket: a string of bucket name
+        :param obj: a string of object name
+        :param headers: a dictionary of headers
+        :param body: a string of data binary sent to S3 as a request body
+        :param query: a string of HTTP query argument
+
+        :returns: a tuple of (int(status_code), headers dict, response body)
+        """
+        response = \
+            self.conn.make_request(method, bucket=bucket, key=obj,
+                                   headers=headers, data=body,
+                                   query_args=query, sender=None,
+                                   override_num_retries=RETRY_COUNT,
+                                   retry_handler=None)
+        return response.status, dict(response.getheaders()), response.read()
+
+    def generate_url_and_headers(self, method, bucket='', obj='',
+                                 expires_in=3600):
+        url = self.conn.generate_url(expires_in, method, bucket, obj)
+        if os.environ.get('S3_USE_SIGV4') == "True":
+            # V4 signatures are known-broken in boto, but we can work around it
+            if url.startswith('https://'):
+                url = 'http://' + url[8:]
+            return url, {'Host': '%(host)s:%(port)d:%(port)d' % {
+                'host': self.host, 'port': self.port}}
+        return url, {}
+
+
+# TODO: make sure where this function is used
+def get_admin_connection():
+    """
+    Return tester connection behaves as:
+    user_test_admin = admin .admin
+    """
+    aws_access_key = tf.config['s3_access_key']
+    aws_secret_key = tf.config['s3_secret_key']
+    user_id = tf.config['s3_access_key']
+    return Connection(aws_access_key, aws_secret_key, user_id)

test/functional/s3api/test_acl.py

@@ -0,0 +1,156 @@
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest2
+import os
+import test.functional as tf
+from swift.common.middleware.s3api.etree import fromstring
+from test.functional.s3api import S3ApiBase
+from test.functional.s3api.s3_test_client import Connection
+from test.functional.s3api.utils import get_error_code
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class TestS3Acl(S3ApiBase):
+    def setUp(self):
+        super(TestS3Acl, self).setUp()
+        self.bucket = 'bucket'
+        self.obj = 'object'
+        if 's3_access_key2' not in tf.config or \
+                's3_secret_key2' not in tf.config:
+            raise tf.SkipTest(
+                'TestS3Acl requires s3_access_key2 and s3_secret_key2 setting')
+        self.conn.make_request('PUT', self.bucket)
+        access_key2 = tf.config['s3_access_key2']
+        secret_key2 = tf.config['s3_secret_key2']
+        self.conn2 = Connection(access_key2, secret_key2, access_key2)
+
+    def test_acl(self):
+        self.conn.make_request('PUT', self.bucket, self.obj)
+        query = 'acl'
+
+        # PUT Bucket ACL
+        headers = {'x-amz-acl': 'public-read'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, headers=headers,
+                                   query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertEqual(headers['content-length'], '0')
+
+        # GET Bucket ACL
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        # TODO: Fix the response that last-modified must be in the response.
+        # self.assertTrue(headers['last-modified'] is not None)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        self.assertTrue(headers['content-type'] is not None)
+        elem = fromstring(body, 'AccessControlPolicy')
+        owner = elem.find('Owner')
+        self.assertEqual(owner.find('ID').text, self.conn.user_id)
+        self.assertEqual(owner.find('DisplayName').text, self.conn.user_id)
+        acl = elem.find('AccessControlList')
+        self.assertTrue(acl.find('Grant') is not None)
+
+        # GET Object ACL
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, self.obj, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        # TODO: Fix the response that last-modified must be in the response.
+        # self.assertTrue(headers['last-modified'] is not None)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        self.assertTrue(headers['content-type'] is not None)
+        elem = fromstring(body, 'AccessControlPolicy')
+        owner = elem.find('Owner')
+        self.assertEqual(owner.find('ID').text, self.conn.user_id)
+        self.assertEqual(owner.find('DisplayName').text, self.conn.user_id)
+        acl = elem.find('AccessControlList')
+        self.assertTrue(acl.find('Grant') is not None)
+
+    def test_put_bucket_acl_error(self):
+        req_headers = {'x-amz-acl': 'public-read'}
+        aws_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            aws_error_conn.make_request('PUT', self.bucket,
+                                        headers=req_headers, query='acl')
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('PUT', 'nothing',
+                                   headers=req_headers, query='acl')
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        status, headers, body = \
+            self.conn2.make_request('PUT', self.bucket,
+                                    headers=req_headers, query='acl')
+        self.assertEqual(get_error_code(body), 'AccessDenied')
+
+    def test_get_bucket_acl_error(self):
+        aws_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            aws_error_conn.make_request('GET', self.bucket, query='acl')
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('GET', 'nothing', query='acl')
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        status, headers, body = \
+            self.conn2.make_request('GET', self.bucket, query='acl')
+        self.assertEqual(get_error_code(body), 'AccessDenied')
+
+    def test_get_object_acl_error(self):
+        self.conn.make_request('PUT', self.bucket, self.obj)
+
+        aws_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            aws_error_conn.make_request('GET', self.bucket, self.obj,
+                                        query='acl')
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, 'nothing', query='acl')
+        self.assertEqual(get_error_code(body), 'NoSuchKey')
+
+        status, headers, body = \
+            self.conn2.make_request('GET', self.bucket, self.obj, query='acl')
+        self.assertEqual(get_error_code(body), 'AccessDenied')
+
+
+class TestS3AclSigV4(TestS3Acl):
+    @classmethod
+    def setUpClass(cls):
+        os.environ['S3_USE_SIGV4'] = "True"
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['S3_USE_SIGV4']
+
+    def setUp(self):
+        super(TestS3AclSigV4, self).setUp()
+
+
+if __name__ == '__main__':
+    unittest2.main()

test/functional/s3api/test_bucket.py

@@ -0,0 +1,487 @@
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest2
+import os
+
+import test.functional as tf
+from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
+    SubElement
+from test.functional.s3api import S3ApiBase
+from test.functional.s3api.s3_test_client import Connection
+from test.functional.s3api.utils import get_error_code
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class TestS3ApiBucket(S3ApiBase):
+    def setUp(self):
+        super(TestS3ApiBucket, self).setUp()
+
+    def _gen_location_xml(self, location):
+        elem = Element('CreateBucketConfiguration')
+        SubElement(elem, 'LocationConstraint').text = location
+        return tostring(elem)
+
+    def test_bucket(self):
+        bucket = 'bucket'
+        max_bucket_listing = tf.cluster_info['s3api'].get(
+            'max_bucket_listing', 1000)
+
+        # PUT Bucket
+        status, headers, body = self.conn.make_request('PUT', bucket)
+        self.assertEqual(status, 200)
+
+        self.assertCommonResponseHeaders(headers)
+        self.assertIn(headers['location'], (
+            '/' + bucket,  # swob won't touch it...
+            # but webob (which we get because of auth_token) *does*
+            'http://%s%s/%s' % (
+                self.conn.host,
+                '' if self.conn.port == 80 else ':%d' % self.conn.port,
+                bucket),
+            # This is all based on the Host header the client provided,
+            # and boto will double-up ports for sig v4. See
+            #   - https://github.com/boto/boto/issues/2623
+            #   - https://github.com/boto/boto/issues/3716
+            # with proposed fixes at
+            #   - https://github.com/boto/boto/pull/3513
+            #   - https://github.com/boto/boto/pull/3676
+            'http://%s%s:%d/%s' % (
+                self.conn.host,
+                '' if self.conn.port == 80 else ':%d' % self.conn.port,
+                self.conn.port,
+                bucket),
+        ))
+        self.assertEqual(headers['content-length'], '0')
+
+        # GET Bucket(Without Object)
+        status, headers, body = self.conn.make_request('GET', bucket)
+        self.assertEqual(status, 200)
+
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue(headers['content-type'] is not None)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        # TODO; requires consideration
+        # self.assertEqual(headers['transfer-encoding'], 'chunked')
+
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('Name').text, bucket)
+        self.assertIsNone(elem.find('Prefix').text)
+        self.assertIsNone(elem.find('Marker').text)
+        self.assertEqual(
+            elem.find('MaxKeys').text, str(max_bucket_listing))
+        self.assertEqual(elem.find('IsTruncated').text, 'false')
+        objects = elem.findall('./Contents')
+        self.assertEqual(list(objects), [])
+
+        # GET Bucket(With Object)
+        req_objects = ('object', 'object2')
+        for obj in req_objects:
+            self.conn.make_request('PUT', bucket, obj)
+        status, headers, body = self.conn.make_request('GET', bucket)
+        self.assertEqual(status, 200)
+
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('Name').text, bucket)
+        self.assertIsNone(elem.find('Prefix').text)
+        self.assertIsNone(elem.find('Marker').text)
+        self.assertEqual(elem.find('MaxKeys').text,
+                         str(max_bucket_listing))
+        self.assertEqual(elem.find('IsTruncated').text, 'false')
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), 2)
+        for o in resp_objects:
+            self.assertTrue(o.find('Key').text in req_objects)
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertTrue(o.find('StorageClass').text is not None)
+            self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
+            self.assertTrue(o.find('Owner/DisplayName').text,
+                            self.conn.user_id)
+
+        # HEAD Bucket
+        status, headers, body = self.conn.make_request('HEAD', bucket)
+        self.assertEqual(status, 200)
+
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue(headers['content-type'] is not None)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        # TODO; requires consideration
+        # self.assertEqual(headers['transfer-encoding'], 'chunked')
+
+        # DELETE Bucket
+        for obj in req_objects:
+            self.conn.make_request('DELETE', bucket, obj)
+        status, headers, body = self.conn.make_request('DELETE', bucket)
+        self.assertEqual(status, 204)
+
+        self.assertCommonResponseHeaders(headers)
+
+    def test_put_bucket_error(self):
+        status, headers, body = \
+            self.conn.make_request('PUT', 'bucket+invalid')
+        self.assertEqual(get_error_code(body), 'InvalidBucketName')
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = auth_error_conn.make_request('PUT', 'bucket')
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        self.conn.make_request('PUT', 'bucket')
+        status, headers, body = self.conn.make_request('PUT', 'bucket')
+        self.assertEqual(get_error_code(body), 'BucketAlreadyExists')
+
+    def test_put_bucket_with_LocationConstraint(self):
+        bucket = 'bucket'
+        xml = self._gen_location_xml('US')
+        status, headers, body = \
+            self.conn.make_request('PUT', bucket, body=xml)
+        self.assertEqual(status, 200)
+
+    def test_get_bucket_error(self):
+        self.conn.make_request('PUT', 'bucket')
+
+        status, headers, body = \
+            self.conn.make_request('GET', 'bucket+invalid')
+        self.assertEqual(get_error_code(body), 'InvalidBucketName')
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = auth_error_conn.make_request('GET', 'bucket')
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = self.conn.make_request('GET', 'nothing')
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+    def _prepare_test_get_bucket(self, bucket, objects):
+        self.conn.make_request('PUT', bucket)
+        for obj in objects:
+            self.conn.make_request('PUT', bucket, obj)
+
+    def test_get_bucket_with_delimiter(self):
+        bucket = 'bucket'
+        put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
+                       'dir/subdir/object')
+        self._prepare_test_get_bucket(bucket, put_objects)
+
+        delimiter = '/'
+        query = 'delimiter=%s' % delimiter
+        expect_objects = ('object', 'object2')
+        expect_prefixes = ('dir/', 'subdir/', 'subdir2/')
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('Delimiter').text, delimiter)
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), len(expect_objects))
+        for i, o in enumerate(resp_objects):
+            self.assertEqual(o.find('Key').text, expect_objects[i])
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertEqual(o.find('StorageClass').text, 'STANDARD')
+            self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
+            self.assertTrue(o.find('Owner/DisplayName').text,
+                            self.conn.user_id)
+        resp_prefixes = elem.findall('CommonPrefixes')
+        self.assertEqual(len(resp_prefixes), len(expect_prefixes))
+        for i, p in enumerate(resp_prefixes):
+            self.assertEqual(p.find('./Prefix').text, expect_prefixes[i])
+
+    def test_get_bucket_with_encoding_type(self):
+        bucket = 'bucket'
+        put_objects = ('object', 'object2')
+        self._prepare_test_get_bucket(bucket, put_objects)
+
+        encoding_type = 'url'
+        query = 'encoding-type=%s' % encoding_type
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('EncodingType').text, encoding_type)
+
+    def test_get_bucket_with_marker(self):
+        bucket = 'bucket'
+        put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
+                       'dir/subdir/object')
+        self._prepare_test_get_bucket(bucket, put_objects)
+
+        marker = 'object'
+        query = 'marker=%s' % marker
+        expect_objects = ('object2', 'subdir/object', 'subdir2/object')
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('Marker').text, marker)
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), len(expect_objects))
+        for i, o in enumerate(resp_objects):
+            self.assertEqual(o.find('Key').text, expect_objects[i])
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertEqual(o.find('StorageClass').text, 'STANDARD')
+            self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
+            self.assertTrue(o.find('Owner/DisplayName').text,
+                            self.conn.user_id)
+
+    def test_get_bucket_with_max_keys(self):
+        bucket = 'bucket'
+        put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
+                       'dir/subdir/object')
+        self._prepare_test_get_bucket(bucket, put_objects)
+
+        max_keys = '2'
+        query = 'max-keys=%s' % max_keys
+        expect_objects = ('dir/subdir/object', 'object')
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('MaxKeys').text, max_keys)
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), len(expect_objects))
+        for i, o in enumerate(resp_objects):
+            self.assertEqual(o.find('Key').text, expect_objects[i])
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertEqual(o.find('StorageClass').text, 'STANDARD')
+            self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
+            self.assertTrue(o.find('Owner/DisplayName').text,
+                            self.conn.user_id)
+
+    def test_get_bucket_with_prefix(self):
+        bucket = 'bucket'
+        req_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
+                       'dir/subdir/object')
+        self._prepare_test_get_bucket(bucket, req_objects)
+
+        prefix = 'object'
+        query = 'prefix=%s' % prefix
+        expect_objects = ('object', 'object2')
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('Prefix').text, prefix)
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), len(expect_objects))
+        for i, o in enumerate(resp_objects):
+            self.assertEqual(o.find('Key').text, expect_objects[i])
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertEqual(o.find('StorageClass').text, 'STANDARD')
+            self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
+            self.assertTrue(o.find('Owner/DisplayName').text,
+                            self.conn.user_id)
+
+    def test_get_bucket_v2_with_start_after(self):
+        bucket = 'bucket'
+        put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
+                       'dir/subdir/object')
+        self._prepare_test_get_bucket(bucket, put_objects)
+
+        marker = 'object'
+        query = 'list-type=2&start-after=%s' % marker
+        expect_objects = ('object2', 'subdir/object', 'subdir2/object')
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('StartAfter').text, marker)
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), len(expect_objects))
+        for i, o in enumerate(resp_objects):
+            self.assertEqual(o.find('Key').text, expect_objects[i])
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertEqual(o.find('StorageClass').text, 'STANDARD')
+            self.assertIsNone(o.find('Owner/ID'))
+            self.assertIsNone(o.find('Owner/DisplayName'))
+
+    def test_get_bucket_v2_with_fetch_owner(self):
+        bucket = 'bucket'
+        put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
+                       'dir/subdir/object')
+        self._prepare_test_get_bucket(bucket, put_objects)
+
+        query = 'list-type=2&fetch-owner=true'
+        expect_objects = ('dir/subdir/object', 'object', 'object2',
+                          'subdir/object', 'subdir2/object')
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('KeyCount').text, '5')
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), len(expect_objects))
+        for i, o in enumerate(resp_objects):
+            self.assertEqual(o.find('Key').text, expect_objects[i])
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertEqual(o.find('StorageClass').text, 'STANDARD')
+            self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
+            self.assertTrue(o.find('Owner/DisplayName').text,
+                            self.conn.user_id)
+
+    def test_get_bucket_v2_with_continuation_token(self):
+        bucket = 'bucket'
+        put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
+                       'dir/subdir/object')
+        self._prepare_test_get_bucket(bucket, put_objects)
+
+        query = 'list-type=2&max-keys=3'
+        expect_objects = ('dir/subdir/object', 'object', 'object2')
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('MaxKeys').text, '3')
+        self.assertEqual(elem.find('KeyCount').text, '3')
+        self.assertEqual(elem.find('IsTruncated').text, 'true')
+        next_cont_token_elem = elem.find('NextContinuationToken')
+        self.assertIsNotNone(next_cont_token_elem)
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), len(expect_objects))
+        for i, o in enumerate(resp_objects):
+            self.assertEqual(o.find('Key').text, expect_objects[i])
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertEqual(o.find('StorageClass').text, 'STANDARD')
+            self.assertIsNone(o.find('Owner/ID'))
+            self.assertIsNone(o.find('Owner/DisplayName'))
+
+        query = 'list-type=2&max-keys=3&continuation-token=%s' % \
+            next_cont_token_elem.text
+        expect_objects = ('subdir/object', 'subdir2/object')
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('MaxKeys').text, '3')
+        self.assertEqual(elem.find('KeyCount').text, '2')
+        self.assertEqual(elem.find('IsTruncated').text, 'false')
+        self.assertIsNone(elem.find('NextContinuationToken'))
+        cont_token_elem = elem.find('ContinuationToken')
+        self.assertEqual(cont_token_elem.text, next_cont_token_elem.text)
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), len(expect_objects))
+        for i, o in enumerate(resp_objects):
+            self.assertEqual(o.find('Key').text, expect_objects[i])
+            self.assertTrue(o.find('LastModified').text is not None)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertTrue(o.find('ETag').text is not None)
+            self.assertTrue(o.find('Size').text is not None)
+            self.assertEqual(o.find('StorageClass').text, 'STANDARD')
+            self.assertIsNone(o.find('Owner/ID'))
+            self.assertIsNone(o.find('Owner/DisplayName'))
+
+    def test_head_bucket_error(self):
+        self.conn.make_request('PUT', 'bucket')
+
+        status, headers, body = \
+            self.conn.make_request('HEAD', 'bucket+invalid')
+        self.assertEqual(status, 400)
+        self.assertEqual(body, '')  # sanity
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('HEAD', 'bucket')
+        self.assertEqual(status, 403)
+        self.assertEqual(body, '')  # sanity
+
+        status, headers, body = self.conn.make_request('HEAD', 'nothing')
+        self.assertEqual(status, 404)
+        self.assertEqual(body, '')  # sanity
+
+    def test_delete_bucket_error(self):
+        status, headers, body = \
+            self.conn.make_request('DELETE', 'bucket+invalid')
+        self.assertEqual(get_error_code(body), 'InvalidBucketName')
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('DELETE', 'bucket')
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = self.conn.make_request('DELETE', 'bucket')
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+    def test_bucket_invalid_method_error(self):
+        # non existed verb in the controller
+        status, headers, body = \
+            self.conn.make_request('GETPUT', 'bucket')
+        self.assertEqual(get_error_code(body), 'MethodNotAllowed')
+        # the method exists in the controller but deny as MethodNotAllowed
+        status, headers, body = \
+            self.conn.make_request('_delete_segments_bucket', 'bucket')
+        self.assertEqual(get_error_code(body), 'MethodNotAllowed')
+
+
+class TestS3ApiBucketSigV4(TestS3ApiBucket):
+    @classmethod
+    def setUpClass(cls):
+        os.environ['S3_USE_SIGV4'] = "True"
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['S3_USE_SIGV4']
+
+    def setUp(self):
+        super(TestS3ApiBucket, self).setUp()
+
+
+if __name__ == '__main__':
+    unittest2.main()

test/functional/s3api/test_multi_delete.py

@@ -0,0 +1,248 @@
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest2
+import os
+import test.functional as tf
+from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
+    SubElement
+from swift.common.middleware.s3api.controllers.multi_delete import \
+    MAX_MULTI_DELETE_BODY_SIZE
+
+from test.functional.s3api import S3ApiBase
+from test.functional.s3api.s3_test_client import Connection
+from test.functional.s3api.utils import get_error_code, calculate_md5
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class TestS3ApiMultiDelete(S3ApiBase):
+    def setUp(self):
+        super(TestS3ApiMultiDelete, self).setUp()
+
+    def _prepare_test_delete_multi_objects(self, bucket, objects):
+        self.conn.make_request('PUT', bucket)
+        for obj in objects:
+            self.conn.make_request('PUT', bucket, obj)
+
+    def _gen_multi_delete_xml(self, objects, quiet=None):
+        elem = Element('Delete')
+        if quiet:
+            SubElement(elem, 'Quiet').text = quiet
+        for key in objects:
+            obj = SubElement(elem, 'Object')
+            SubElement(obj, 'Key').text = key
+
+        return tostring(elem, use_s3ns=False)
+
+    def _gen_invalid_multi_delete_xml(self, hasObjectTag=False):
+        elem = Element('Delete')
+        if hasObjectTag:
+            obj = SubElement(elem, 'Object')
+            SubElement(obj, 'Key').text = ''
+
+        return tostring(elem, use_s3ns=False)
+
+    def test_delete_multi_objects(self):
+        bucket = 'bucket'
+        put_objects = ['obj%s' % var for var in xrange(4)]
+        self._prepare_test_delete_multi_objects(bucket, put_objects)
+        query = 'delete'
+
+        # Delete an object via MultiDelete API
+        req_objects = ['obj0']
+        xml = self._gen_multi_delete_xml(req_objects)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue(headers['content-type'] is not None)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        elem = fromstring(body)
+        resp_objects = elem.findall('Deleted')
+        self.assertEqual(len(resp_objects), len(req_objects))
+        for o in resp_objects:
+            self.assertTrue(o.find('Key').text in req_objects)
+
+        # Delete 2 objects via MultiDelete API
+        req_objects = ['obj1', 'obj2']
+        xml = self._gen_multi_delete_xml(req_objects)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'DeleteResult')
+        resp_objects = elem.findall('Deleted')
+        self.assertEqual(len(resp_objects), len(req_objects))
+        for o in resp_objects:
+            self.assertTrue(o.find('Key').text in req_objects)
+
+        # Delete 2 objects via MultiDelete API but one (obj4) doesn't exist.
+        req_objects = ['obj3', 'obj4']
+        xml = self._gen_multi_delete_xml(req_objects)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'DeleteResult')
+        resp_objects = elem.findall('Deleted')
+        # S3 assumes a NoSuchKey object as deleted.
+        self.assertEqual(len(resp_objects), len(req_objects))
+        for o in resp_objects:
+            self.assertTrue(o.find('Key').text in req_objects)
+
+        # Delete 2 objects via MultiDelete API but no objects exist
+        req_objects = ['obj4', 'obj5']
+        xml = self._gen_multi_delete_xml(req_objects)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'DeleteResult')
+        resp_objects = elem.findall('Deleted')
+        self.assertEqual(len(resp_objects), len(req_objects))
+        for o in resp_objects:
+            self.assertTrue(o.find('Key').text in req_objects)
+
+    def test_delete_multi_objects_error(self):
+        bucket = 'bucket'
+        put_objects = ['obj']
+        self._prepare_test_delete_multi_objects(bucket, put_objects)
+        xml = self._gen_multi_delete_xml(put_objects)
+        content_md5 = calculate_md5(xml)
+        query = 'delete'
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('POST', bucket, body=xml,
+                                         headers={
+                                             'Content-MD5': content_md5
+                                         },
+                                         query=query)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('POST', 'nothing', body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        # without Object tag
+        xml = self._gen_invalid_multi_delete_xml()
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'MalformedXML')
+
+        # without value of Key tag
+        xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified')
+
+        # specified number of objects are over max_multi_delete_objects
+        # (Default 1000), but xml size is smaller than 61365 bytes.
+        req_objects = ['obj%s' for var in xrange(1001)]
+        xml = self._gen_multi_delete_xml(req_objects)
+        self.assertTrue(len(xml.encode('utf-8')) <= MAX_MULTI_DELETE_BODY_SIZE)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'MalformedXML')
+
+        # specified xml size is over 61365 bytes, but number of objects are
+        # smaller than max_multi_delete_objects.
+        obj = 'a' * 1024
+        req_objects = [obj + str(var) for var in xrange(999)]
+        xml = self._gen_multi_delete_xml(req_objects)
+        self.assertTrue(len(xml.encode('utf-8')) > MAX_MULTI_DELETE_BODY_SIZE)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'MalformedXML')
+
+    def test_delete_multi_objects_with_quiet(self):
+        bucket = 'bucket'
+        put_objects = ['obj']
+        query = 'delete'
+
+        # with Quiet true
+        quiet = 'true'
+        self._prepare_test_delete_multi_objects(bucket, put_objects)
+        xml = self._gen_multi_delete_xml(put_objects, quiet)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'DeleteResult')
+        resp_objects = elem.findall('Deleted')
+        self.assertEqual(len(resp_objects), 0)
+
+        # with Quiet false
+        quiet = 'false'
+        self._prepare_test_delete_multi_objects(bucket, put_objects)
+        xml = self._gen_multi_delete_xml(put_objects, quiet)
+        content_md5 = calculate_md5(xml)
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, body=xml,
+                                   headers={'Content-MD5': content_md5},
+                                   query=query)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'DeleteResult')
+        resp_objects = elem.findall('Deleted')
+        self.assertEqual(len(resp_objects), 1)
+
+
+class TestS3ApiMultiDeleteSigV4(TestS3ApiMultiDelete):
+    @classmethod
+    def setUpClass(cls):
+        os.environ['S3_USE_SIGV4'] = "True"
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['S3_USE_SIGV4']
+
+    def setUp(self):
+        super(TestS3ApiMultiDeleteSigV4, self).setUp()
+
+
+if __name__ == '__main__':
+    unittest2.main()

test/functional/s3api/test_multi_upload.py

@@ -0,0 +1,849 @@
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import unittest2
+import os
+import boto
+
+# For an issue with venv and distutils, disable pylint message here
+# pylint: disable-msg=E0611,F0401
+from distutils.version import StrictVersion
+
+from hashlib import md5
+from itertools import izip, izip_longest
+
+import test.functional as tf
+from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
+    SubElement
+from swift.common.middleware.s3api.utils import mktime
+
+from test.functional.s3api import S3ApiBase
+from test.functional.s3api.s3_test_client import Connection
+from test.functional.s3api.utils import get_error_code, get_error_msg
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class TestS3ApiMultiUpload(S3ApiBase):
+    def setUp(self):
+        super(TestS3ApiMultiUpload, self).setUp()
+        if not tf.cluster_info['s3api'].get('allow_multipart_uploads', False):
+            raise tf.SkipTest('multipart upload is not enebled')
+
+        self.min_segment_size = int(tf.cluster_info['s3api'].get(
+            'min_segment_size', 5242880))
+
+    def _gen_comp_xml(self, etags):
+        elem = Element('CompleteMultipartUpload')
+        for i, etag in enumerate(etags):
+            elem_part = SubElement(elem, 'Part')
+            SubElement(elem_part, 'PartNumber').text = str(i + 1)
+            SubElement(elem_part, 'ETag').text = etag
+        return tostring(elem)
+
+    def _initiate_multi_uploads_result_generator(self, bucket, keys,
+                                                 headers=None, trials=1):
+        if headers is None:
+            headers = [None] * len(keys)
+        self.conn.make_request('PUT', bucket)
+        query = 'uploads'
+        for key, key_headers in izip_longest(keys, headers):
+            for i in xrange(trials):
+                status, resp_headers, body = \
+                    self.conn.make_request('POST', bucket, key,
+                                           headers=key_headers, query=query)
+                yield status, resp_headers, body
+
+    def _upload_part(self, bucket, key, upload_id, content=None, part_num=1):
+        query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id)
+        content = content if content else 'a' * self.min_segment_size
+        status, headers, body = \
+            self.conn.make_request('PUT', bucket, key, body=content,
+                                   query=query)
+        return status, headers, body
+
+    def _upload_part_copy(self, src_bucket, src_obj, dst_bucket, dst_key,
+                          upload_id, part_num=1, src_range=None):
+
+        src_path = '%s/%s' % (src_bucket, src_obj)
+        query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id)
+        req_headers = {'X-Amz-Copy-Source': src_path}
+        if src_range:
+            req_headers['X-Amz-Copy-Source-Range'] = src_range
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_key,
+                                   headers=req_headers,
+                                   query=query)
+        elem = fromstring(body, 'CopyPartResult')
+        etag = elem.find('ETag').text.strip('"')
+        return status, headers, body, etag
+
+    def _complete_multi_upload(self, bucket, key, upload_id, xml):
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, body=xml,
+                                   query=query)
+        return status, headers, body
+
+    def test_object_multi_upload(self):
+        bucket = 'bucket'
+        keys = ['obj1', 'obj2', 'obj3']
+        headers = [None,
+                   {'Content-MD5': base64.b64encode('a' * 16).strip()},
+                   {'Etag': 'nonsense'}]
+        uploads = []
+
+        results_generator = self._initiate_multi_uploads_result_generator(
+            bucket, keys, headers=headers)
+
+        # Initiate Multipart Upload
+        for expected_key, (status, headers, body) in \
+                izip(keys, results_generator):
+            self.assertEqual(status, 200)
+            self.assertCommonResponseHeaders(headers)
+            self.assertTrue('content-type' in headers)
+            self.assertEqual(headers['content-type'], 'application/xml')
+            self.assertTrue('content-length' in headers)
+            self.assertEqual(headers['content-length'], str(len(body)))
+            elem = fromstring(body, 'InitiateMultipartUploadResult')
+            self.assertEqual(elem.find('Bucket').text, bucket)
+            key = elem.find('Key').text
+            self.assertEqual(expected_key, key)
+            upload_id = elem.find('UploadId').text
+            self.assertTrue(upload_id is not None)
+            self.assertTrue((key, upload_id) not in uploads)
+            uploads.append((key, upload_id))
+
+        self.assertEqual(len(uploads), len(keys))  # sanity
+
+        # List Multipart Uploads
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'application/xml')
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        elem = fromstring(body, 'ListMultipartUploadsResult')
+        self.assertEqual(elem.find('Bucket').text, bucket)
+        self.assertIsNone(elem.find('KeyMarker').text)
+        self.assertEqual(elem.find('NextKeyMarker').text, uploads[-1][0])
+        self.assertIsNone(elem.find('UploadIdMarker').text)
+        self.assertEqual(elem.find('NextUploadIdMarker').text, uploads[-1][1])
+        self.assertEqual(elem.find('MaxUploads').text, '1000')
+        self.assertTrue(elem.find('EncodingType') is None)
+        self.assertEqual(elem.find('IsTruncated').text, 'false')
+        self.assertEqual(len(elem.findall('Upload')), 3)
+        for (expected_key, expected_upload_id), u in \
+                izip(uploads, elem.findall('Upload')):
+            key = u.find('Key').text
+            upload_id = u.find('UploadId').text
+            self.assertEqual(expected_key, key)
+            self.assertEqual(expected_upload_id, upload_id)
+            self.assertEqual(u.find('Initiator/ID').text,
+                             self.conn.user_id)
+            self.assertEqual(u.find('Initiator/DisplayName').text,
+                             self.conn.user_id)
+            self.assertEqual(u.find('Owner/ID').text, self.conn.user_id)
+            self.assertEqual(u.find('Owner/DisplayName').text,
+                             self.conn.user_id)
+            self.assertEqual(u.find('StorageClass').text, 'STANDARD')
+            self.assertTrue(u.find('Initiated').text is not None)
+
+        # Upload Part
+        key, upload_id = uploads[0]
+        content = 'a' * self.min_segment_size
+        etag = md5(content).hexdigest()
+        status, headers, body = \
+            self._upload_part(bucket, key, upload_id, content)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers, etag)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], '0')
+        expected_parts_list = [(headers['etag'], mktime(headers['date']))]
+
+        # Upload Part Copy
+        key, upload_id = uploads[1]
+        src_bucket = 'bucket2'
+        src_obj = 'obj3'
+        src_content = 'b' * self.min_segment_size
+        etag = md5(src_content).hexdigest()
+
+        # prepare src obj
+        self.conn.make_request('PUT', src_bucket)
+        self.conn.make_request('PUT', src_bucket, src_obj, body=src_content)
+        _, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj)
+        self.assertCommonResponseHeaders(headers)
+
+        status, headers, body, resp_etag = \
+            self._upload_part_copy(src_bucket, src_obj, bucket,
+                                   key, upload_id)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'application/xml')
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        self.assertTrue('etag' not in headers)
+        elem = fromstring(body, 'CopyPartResult')
+
+        last_modified = elem.find('LastModified').text
+        self.assertTrue(last_modified is not None)
+
+        self.assertEqual(resp_etag, etag)
+
+        # Check last-modified timestamp
+        key, upload_id = uploads[1]
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, key, query=query)
+
+        self.assertEqual(200, status)
+        elem = fromstring(body, 'ListPartsResult')
+
+        # FIXME: COPY result drops milli/microseconds but GET doesn't
+        last_modified_gets = [p.find('LastModified').text
+                              for p in elem.iterfind('Part')]
+        self.assertEqual(
+            last_modified_gets[0].rsplit('.', 1)[0],
+            last_modified.rsplit('.', 1)[0],
+            '%r != %r' % (last_modified_gets[0], last_modified))
+        # There should be *exactly* two parts in the result
+        self.assertEqual(1, len(last_modified_gets))
+
+        # List Parts
+        key, upload_id = uploads[0]
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, key, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'application/xml')
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        elem = fromstring(body, 'ListPartsResult')
+        self.assertEqual(elem.find('Bucket').text, bucket)
+        self.assertEqual(elem.find('Key').text, key)
+        self.assertEqual(elem.find('UploadId').text, upload_id)
+        self.assertEqual(elem.find('Initiator/ID').text, self.conn.user_id)
+        self.assertEqual(elem.find('Initiator/DisplayName').text,
+                         self.conn.user_id)
+        self.assertEqual(elem.find('Owner/ID').text, self.conn.user_id)
+        self.assertEqual(elem.find('Owner/DisplayName').text,
+                         self.conn.user_id)
+        self.assertEqual(elem.find('StorageClass').text, 'STANDARD')
+        self.assertEqual(elem.find('PartNumberMarker').text, '0')
+        self.assertEqual(elem.find('NextPartNumberMarker').text, '1')
+        self.assertEqual(elem.find('MaxParts').text, '1000')
+        self.assertEqual(elem.find('IsTruncated').text, 'false')
+        self.assertEqual(len(elem.findall('Part')), 1)
+
+        # etags will be used to generate xml for Complete Multipart Upload
+        etags = []
+        for (expected_etag, expected_date), p in \
+                izip(expected_parts_list, elem.findall('Part')):
+            last_modified = p.find('LastModified').text
+            self.assertTrue(last_modified is not None)
+            # TODO: sanity check
+            #       (kota_) How do we check the sanity?
+            #       the last-modified header drops milli-seconds info
+            #       by the constraint of the format.
+            #       For now, we can do either the format check or round check
+            # last_modified_from_xml = mktime(last_modified)
+            # self.assertEqual(expected_date,
+            #                   last_modified_from_xml)
+            self.assertEqual(expected_etag, p.find('ETag').text)
+            self.assertEqual(self.min_segment_size, int(p.find('Size').text))
+            etags.append(p.find('ETag').text)
+
+        # Abort Multipart Uploads
+        # note that uploads[1] has part data while uploads[2] does not
+        for key, upload_id in uploads[1:]:
+            query = 'uploadId=%s' % upload_id
+            status, headers, body = \
+                self.conn.make_request('DELETE', bucket, key, query=query)
+            self.assertEqual(status, 204)
+            self.assertCommonResponseHeaders(headers)
+            self.assertTrue('content-type' in headers)
+            self.assertEqual(headers['content-type'],
+                             'text/html; charset=UTF-8')
+            self.assertTrue('content-length' in headers)
+            self.assertEqual(headers['content-length'], '0')
+
+        # Complete Multipart Upload
+        key, upload_id = uploads[0]
+        xml = self._gen_comp_xml(etags)
+        status, headers, body = \
+            self._complete_multi_upload(bucket, key, upload_id, xml)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'application/xml')
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        elem = fromstring(body, 'CompleteMultipartUploadResult')
+        # TODO: use tf.config value
+        self.assertEqual(
+            'http://%s:%s/bucket/obj1' % (self.conn.host, self.conn.port),
+            elem.find('Location').text)
+        self.assertEqual(elem.find('Bucket').text, bucket)
+        self.assertEqual(elem.find('Key').text, key)
+        # TODO: confirm completed etag value
+        self.assertTrue(elem.find('ETag').text is not None)
+
+    def test_initiate_multi_upload_error(self):
+        bucket = 'bucket'
+        key = 'obj'
+        self.conn.make_request('PUT', bucket)
+        query = 'uploads'
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('POST', bucket, key, query=query)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, resp_headers, body = \
+            self.conn.make_request('POST', 'nothing', key, query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+    def test_list_multi_uploads_error(self):
+        bucket = 'bucket'
+        self.conn.make_request('PUT', bucket)
+        query = 'uploads'
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('GET', bucket, query=query)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('GET', 'nothing', query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+    def test_upload_part_error(self):
+        bucket = 'bucket'
+        self.conn.make_request('PUT', bucket)
+        query = 'uploads'
+        key = 'obj'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+
+        query = 'partNumber=%s&uploadId=%s' % (1, upload_id)
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('PUT', bucket, key, query=query)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('PUT', 'nothing', key, query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        query = 'partNumber=%s&uploadId=%s' % (1, 'nothing')
+        status, headers, body = \
+            self.conn.make_request('PUT', bucket, key, query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchUpload')
+
+        query = 'partNumber=%s&uploadId=%s' % (0, upload_id)
+        status, headers, body = \
+            self.conn.make_request('PUT', bucket, key, query=query)
+        self.assertEqual(get_error_code(body), 'InvalidArgument')
+        err_msg = 'Part number must be an integer between 1 and'
+        self.assertTrue(err_msg in get_error_msg(body))
+
+    def test_upload_part_copy_error(self):
+        src_bucket = 'src'
+        src_obj = 'src'
+        self.conn.make_request('PUT', src_bucket)
+        self.conn.make_request('PUT', src_bucket, src_obj)
+        src_path = '%s/%s' % (src_bucket, src_obj)
+
+        bucket = 'bucket'
+        self.conn.make_request('PUT', bucket)
+        key = 'obj'
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+
+        query = 'partNumber=%s&uploadId=%s' % (1, upload_id)
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('PUT', bucket, key,
+                                         headers={
+                                             'X-Amz-Copy-Source': src_path
+                                         },
+                                         query=query)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('PUT', 'nothing', key,
+                                   headers={'X-Amz-Copy-Source': src_path},
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        query = 'partNumber=%s&uploadId=%s' % (1, 'nothing')
+        status, headers, body = \
+            self.conn.make_request('PUT', bucket, key,
+                                   headers={'X-Amz-Copy-Source': src_path},
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchUpload')
+
+        src_path = '%s/%s' % (src_bucket, 'nothing')
+        query = 'partNumber=%s&uploadId=%s' % (1, upload_id)
+        status, headers, body = \
+            self.conn.make_request('PUT', bucket, key,
+                                   headers={'X-Amz-Copy-Source': src_path},
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchKey')
+
+    def test_list_parts_error(self):
+        bucket = 'bucket'
+        self.conn.make_request('PUT', bucket)
+        key = 'obj'
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+
+        query = 'uploadId=%s' % upload_id
+        auth_error_conn = Connection(aws_secret_key='invalid')
+
+        status, headers, body = \
+            auth_error_conn.make_request('GET', bucket, key, query=query)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('GET', 'nothing', key, query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        query = 'uploadId=%s' % 'nothing'
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, key, query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchUpload')
+
+    def test_abort_multi_upload_error(self):
+        bucket = 'bucket'
+        self.conn.make_request('PUT', bucket)
+        key = 'obj'
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+        self._upload_part(bucket, key, upload_id)
+
+        query = 'uploadId=%s' % upload_id
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('DELETE', bucket, key, query=query)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        status, headers, body = \
+            self.conn.make_request('DELETE', 'nothing', key, query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        status, headers, body = \
+            self.conn.make_request('DELETE', bucket, 'nothing', query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchUpload')
+
+        query = 'uploadId=%s' % 'nothing'
+        status, headers, body = \
+            self.conn.make_request('DELETE', bucket, key, query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchUpload')
+
+    def test_complete_multi_upload_error(self):
+        bucket = 'bucket'
+        keys = ['obj', 'obj2']
+        self.conn.make_request('PUT', bucket)
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, keys[0], query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+
+        etags = []
+        for i in xrange(1, 3):
+            query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
+            status, headers, body = \
+                self.conn.make_request('PUT', bucket, keys[0], query=query)
+            etags.append(headers['etag'])
+        xml = self._gen_comp_xml(etags)
+
+        # part 1 too small
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, keys[0], body=xml,
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'EntityTooSmall')
+
+        # invalid credentials
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('POST', bucket, keys[0], body=xml,
+                                         query=query)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+
+        # wrong/missing bucket
+        status, headers, body = \
+            self.conn.make_request('POST', 'nothing', keys[0], query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        # wrong upload ID
+        query = 'uploadId=%s' % 'nothing'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, keys[0], body=xml,
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'NoSuchUpload')
+
+        # without Part tag in xml
+        query = 'uploadId=%s' % upload_id
+        xml = self._gen_comp_xml([])
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, keys[0], body=xml,
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'MalformedXML')
+
+        # with invalid etag in xml
+        invalid_etag = 'invalid'
+        xml = self._gen_comp_xml([invalid_etag])
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, keys[0], body=xml,
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'InvalidPart')
+
+        # without part in Swift
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, keys[1], query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+        query = 'uploadId=%s' % upload_id
+        xml = self._gen_comp_xml([etags[0]])
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, keys[1], body=xml,
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'InvalidPart')
+
+    def test_complete_upload_min_segment_size(self):
+        bucket = 'bucket'
+        key = 'obj'
+        self.conn.make_request('PUT', bucket)
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+
+        # multi parts with no body
+        etags = []
+        for i in xrange(1, 3):
+            query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
+            status, headers, body = \
+                self.conn.make_request('PUT', bucket, key, query=query)
+            etags.append(headers['etag'])
+            xml = self._gen_comp_xml(etags)
+
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, body=xml,
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'EntityTooSmall')
+
+        # multi parts with all parts less than min segment size
+        etags = []
+        for i in xrange(1, 3):
+            query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
+            status, headers, body = \
+                self.conn.make_request('PUT', bucket, key, query=query,
+                                       body='AA')
+            etags.append(headers['etag'])
+            xml = self._gen_comp_xml(etags)
+
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, body=xml,
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'EntityTooSmall')
+
+        # one part and less than min segment size
+        etags = []
+        query = 'partNumber=1&uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('PUT', bucket, key, query=query,
+                                   body='AA')
+        etags.append(headers['etag'])
+        xml = self._gen_comp_xml(etags)
+
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, body=xml,
+                                   query=query)
+        self.assertEqual(status, 200)
+
+        # multi parts with all parts except the first part less than min
+        # segment size
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+
+        etags = []
+        body_size = [self.min_segment_size, self.min_segment_size - 1, 2]
+        for i in xrange(1, 3):
+            query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
+            status, headers, body = \
+                self.conn.make_request('PUT', bucket, key, query=query,
+                                       body='A' * body_size[i])
+            etags.append(headers['etag'])
+            xml = self._gen_comp_xml(etags)
+
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, body=xml,
+                                   query=query)
+        self.assertEqual(get_error_code(body), 'EntityTooSmall')
+
+        # multi parts with all parts except last part more than min segment
+        # size
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+
+        etags = []
+        body_size = [self.min_segment_size, self.min_segment_size, 2]
+        for i in xrange(1, 3):
+            query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
+            status, headers, body = \
+                self.conn.make_request('PUT', bucket, key, query=query,
+                                       body='A' * body_size[i])
+            etags.append(headers['etag'])
+            xml = self._gen_comp_xml(etags)
+
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, body=xml,
+                                   query=query)
+        self.assertEqual(status, 200)
+
+    def test_complete_upload_with_fewer_etags(self):
+        bucket = 'bucket'
+        key = 'obj'
+        self.conn.make_request('PUT', bucket)
+        query = 'uploads'
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, query=query)
+        elem = fromstring(body, 'InitiateMultipartUploadResult')
+        upload_id = elem.find('UploadId').text
+
+        etags = []
+        for i in xrange(1, 4):
+            query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
+            status, headers, body = \
+                self.conn.make_request('PUT', bucket, key,
+                                       body='A' * 1024 * 1024 * 5, query=query)
+            etags.append(headers['etag'])
+        query = 'uploadId=%s' % upload_id
+        xml = self._gen_comp_xml(etags[:-1])
+        status, headers, body = \
+            self.conn.make_request('POST', bucket, key, body=xml,
+                                   query=query)
+        self.assertEqual(status, 200)
+
+    def test_object_multi_upload_part_copy_range(self):
+        bucket = 'bucket'
+        keys = ['obj1']
+        uploads = []
+
+        results_generator = self._initiate_multi_uploads_result_generator(
+            bucket, keys)
+
+        # Initiate Multipart Upload
+        for expected_key, (status, headers, body) in \
+                izip(keys, results_generator):
+            self.assertEqual(status, 200)
+            self.assertCommonResponseHeaders(headers)
+            self.assertTrue('content-type' in headers)
+            self.assertEqual(headers['content-type'], 'application/xml')
+            self.assertTrue('content-length' in headers)
+            self.assertEqual(headers['content-length'], str(len(body)))
+            elem = fromstring(body, 'InitiateMultipartUploadResult')
+            self.assertEqual(elem.find('Bucket').text, bucket)
+            key = elem.find('Key').text
+            self.assertEqual(expected_key, key)
+            upload_id = elem.find('UploadId').text
+            self.assertTrue(upload_id is not None)
+            self.assertTrue((key, upload_id) not in uploads)
+            uploads.append((key, upload_id))
+
+        self.assertEqual(len(uploads), len(keys))  # sanity
+
+        # Upload Part Copy Range
+        key, upload_id = uploads[0]
+        src_bucket = 'bucket2'
+        src_obj = 'obj4'
+        src_content = 'y' * (self.min_segment_size / 2) + 'z' * \
+            self.min_segment_size
+        src_range = 'bytes=0-%d' % (self.min_segment_size - 1)
+        etag = md5(src_content[:self.min_segment_size]).hexdigest()
+
+        # prepare src obj
+        self.conn.make_request('PUT', src_bucket)
+        self.conn.make_request('PUT', src_bucket, src_obj, body=src_content)
+        _, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj)
+        self.assertCommonResponseHeaders(headers)
+
+        status, headers, body, resp_etag = \
+            self._upload_part_copy(src_bucket, src_obj, bucket,
+                                   key, upload_id, 1, src_range)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'application/xml')
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], str(len(body)))
+        self.assertTrue('etag' not in headers)
+        elem = fromstring(body, 'CopyPartResult')
+
+        last_modified = elem.find('LastModified').text
+        self.assertTrue(last_modified is not None)
+
+        self.assertEqual(resp_etag, etag)
+
+        # Check last-modified timestamp
+        key, upload_id = uploads[0]
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, key, query=query)
+
+        elem = fromstring(body, 'ListPartsResult')
+
+        # FIXME: COPY result drops milli/microseconds but GET doesn't
+        last_modified_gets = [p.find('LastModified').text
+                              for p in elem.iterfind('Part')]
+        self.assertEqual(
+            last_modified_gets[0].rsplit('.', 1)[0],
+            last_modified.rsplit('.', 1)[0],
+            '%r != %r' % (last_modified_gets[0], last_modified))
+
+        # There should be *exactly* one parts in the result
+        self.assertEqual(1, len(last_modified_gets))
+
+        # Abort Multipart Upload
+        key, upload_id = uploads[0]
+        query = 'uploadId=%s' % upload_id
+        status, headers, body = \
+            self.conn.make_request('DELETE', bucket, key, query=query)
+
+        # sanity checks
+        self.assertEqual(status, 204)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], '0')
+
+
+class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
+    @classmethod
+    def setUpClass(cls):
+        os.environ['S3_USE_SIGV4'] = "True"
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['S3_USE_SIGV4']
+
+    def setUp(self):
+        super(TestS3ApiMultiUploadSigV4, self).setUp()
+
+    def test_object_multi_upload_part_copy_range(self):
+        if StrictVersion(boto.__version__) < StrictVersion('3.0'):
+            self.skipTest('This stuff got the issue of boto<=2.x')
+
+    def test_delete_bucket_multi_upload_object_exisiting(self):
+        bucket = 'bucket'
+        keys = ['obj1']
+        uploads = []
+
+        results_generator = self._initiate_multi_uploads_result_generator(
+            bucket, keys)
+
+        # Initiate Multipart Upload
+        for expected_key, (status, _, body) in \
+                izip(keys, results_generator):
+            self.assertEqual(status, 200)  # sanity
+            elem = fromstring(body, 'InitiateMultipartUploadResult')
+            key = elem.find('Key').text
+            self.assertEqual(expected_key, key)  # sanity
+            upload_id = elem.find('UploadId').text
+            self.assertTrue(upload_id is not None)  # sanity
+            self.assertTrue((key, upload_id) not in uploads)
+            uploads.append((key, upload_id))
+
+        self.assertEqual(len(uploads), len(keys))  # sanity
+
+        # Upload Part
+        key, upload_id = uploads[0]
+        content = 'a' * self.min_segment_size
+        status, headers, body = \
+            self._upload_part(bucket, key, upload_id, content)
+        self.assertEqual(status, 200)
+
+        # Complete Multipart Upload
+        key, upload_id = uploads[0]
+        etags = [md5(content).hexdigest()]
+        xml = self._gen_comp_xml(etags)
+        status, headers, body = \
+            self._complete_multi_upload(bucket, key, upload_id, xml)
+        self.assertEqual(status, 200)  # sanity
+
+        # GET multipart object
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, key)
+        self.assertEqual(status, 200)  # sanity
+        self.assertEqual(content, body)  # sanity
+
+        # DELETE bucket while the object existing
+        status, headers, body = \
+            self.conn.make_request('DELETE', bucket)
+        self.assertEqual(status, 409)  # sanity
+
+        # The object must still be there.
+        status, headers, body = \
+            self.conn.make_request('GET', bucket, key)
+        self.assertEqual(status, 200)  # sanity
+        self.assertEqual(content, body)  # sanity
+
+
+if __name__ == '__main__':
+    unittest2.main()

test/functional/s3api/test_object.py

@@ -0,0 +1,873 @@
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest2
+import os
+import boto
+
+# For an issue with venv and distutils, disable pylint message here
+# pylint: disable-msg=E0611,F0401
+from distutils.version import StrictVersion
+
+import email.parser
+from email.utils import formatdate, parsedate
+from time import mktime
+from hashlib import md5
+from urllib import quote
+
+import test.functional as tf
+
+from swift.common.middleware.s3api.etree import fromstring
+
+from test.functional.s3api import S3ApiBase
+from test.functional.s3api.s3_test_client import Connection
+from test.functional.s3api.utils import get_error_code, calculate_md5
+
+DAY = 86400.0  # 60 * 60 * 24 (sec)
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class TestS3ApiObject(S3ApiBase):
+    def setUp(self):
+        super(TestS3ApiObject, self).setUp()
+        self.bucket = 'bucket'
+        self.conn.make_request('PUT', self.bucket)
+
+    def _assertObjectEtag(self, bucket, obj, etag):
+        status, headers, _ = self.conn.make_request('HEAD', bucket, obj)
+        self.assertEqual(status, 200)  # sanity
+        self.assertCommonResponseHeaders(headers, etag)
+
+    def test_object(self):
+        obj = 'object name with %-sign'
+        content = 'abc123'
+        etag = md5(content).hexdigest()
+
+        # PUT Object
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, body=content)
+        self.assertEqual(status, 200)
+
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-length' in headers)  # sanity
+        self.assertEqual(headers['content-length'], '0')
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+        # PUT Object Copy
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_obj'
+        self.conn.make_request('PUT', dst_bucket)
+        headers = {'x-amz-copy-source': '/%s/%s' % (self.bucket, obj)}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj,
+                                   headers=headers)
+        self.assertEqual(status, 200)
+
+        # PUT Object Copy with URL-encoded Source
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_obj'
+        self.conn.make_request('PUT', dst_bucket)
+        headers = {'x-amz-copy-source': quote('/%s/%s' % (self.bucket, obj))}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj,
+                                   headers=headers)
+        self.assertEqual(status, 200)
+
+        self.assertCommonResponseHeaders(headers)
+        self.assertEqual(headers['content-length'], str(len(body)))
+
+        elem = fromstring(body, 'CopyObjectResult')
+        self.assertTrue(elem.find('LastModified').text is not None)
+        last_modified_xml = elem.find('LastModified').text
+        self.assertTrue(elem.find('ETag').text is not None)
+        self.assertEqual(etag, elem.find('ETag').text.strip('"'))
+        self._assertObjectEtag(dst_bucket, dst_obj, etag)
+
+        # Check timestamp on Copy:
+        status, headers, body = \
+            self.conn.make_request('GET', dst_bucket)
+        self.assertEqual(status, 200)
+        elem = fromstring(body, 'ListBucketResult')
+
+        # FIXME: COPY result drops milli/microseconds but GET doesn't
+        self.assertEqual(
+            elem.find('Contents').find("LastModified").text.rsplit('.', 1)[0],
+            last_modified_xml.rsplit('.', 1)[0])
+
+        # GET Object
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj)
+        self.assertEqual(status, 200)
+
+        self.assertCommonResponseHeaders(headers, etag)
+        self.assertTrue(headers['last-modified'] is not None)
+        self.assertTrue(headers['content-type'] is not None)
+        self.assertEqual(headers['content-length'], str(len(content)))
+
+        # HEAD Object
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj)
+        self.assertEqual(status, 200)
+
+        self.assertCommonResponseHeaders(headers, etag)
+        self.assertTrue(headers['last-modified'] is not None)
+        self.assertTrue('content-type' in headers)
+        self.assertEqual(headers['content-length'], str(len(content)))
+
+        # DELETE Object
+        status, headers, body = \
+            self.conn.make_request('DELETE', self.bucket, obj)
+        self.assertEqual(status, 204)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_put_object_error(self):
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('PUT', self.bucket, 'object')
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        status, headers, body = \
+            self.conn.make_request('PUT', 'bucket2', 'object')
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+    def test_put_object_copy_error(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+        dst_bucket = 'dst-bucket'
+        self.conn.make_request('PUT', dst_bucket)
+        dst_obj = 'dst_object'
+
+        headers = {'x-amz-copy-source': '/%s/%s' % (self.bucket, obj)}
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('PUT', dst_bucket, dst_obj, headers)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        # /src/nothing -> /dst/dst
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, 'nothing')}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers)
+        self.assertEqual(get_error_code(body), 'NoSuchKey')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        # /nothing/src -> /dst/dst
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % ('nothing', obj)}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers)
+        # TODO: source bucket is not check.
+        # self.assertEqual(get_error_code(body), 'NoSuchBucket')
+
+        # /src/src -> /nothing/dst
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj)}
+        status, headers, body = \
+            self.conn.make_request('PUT', 'nothing', dst_obj, headers)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+    def test_get_object_error(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('GET', self.bucket, obj)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, 'invalid')
+        self.assertEqual(get_error_code(body), 'NoSuchKey')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        status, headers, body = self.conn.make_request('GET', 'invalid', obj)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+    def test_head_object_error(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('HEAD', self.bucket, obj)
+        self.assertEqual(status, 403)
+        self.assertEqual(body, '')  # sanity
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, 'invalid')
+        self.assertEqual(status, 404)
+        self.assertEqual(body, '')  # sanity
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        status, headers, body = \
+            self.conn.make_request('HEAD', 'invalid', obj)
+        self.assertEqual(status, 404)
+        self.assertEqual(body, '')  # sanity
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+    def test_delete_object_error(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = \
+            auth_error_conn.make_request('DELETE', self.bucket, obj)
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        status, headers, body = \
+            self.conn.make_request('DELETE', self.bucket, 'invalid')
+        self.assertEqual(get_error_code(body), 'NoSuchKey')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+        status, headers, body = \
+            self.conn.make_request('DELETE', 'invalid', obj)
+        self.assertEqual(get_error_code(body), 'NoSuchBucket')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+    def test_put_object_content_encoding(self):
+        obj = 'object'
+        etag = md5().hexdigest()
+        headers = {'Content-Encoding': 'gzip'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers)
+        self.assertEqual(status, 200)
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj)
+        self.assertTrue('content-encoding' in headers)  # sanity
+        self.assertEqual(headers['content-encoding'], 'gzip')
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_put_object_content_md5(self):
+        obj = 'object'
+        content = 'abcdefghij'
+        etag = md5(content).hexdigest()
+        headers = {'Content-MD5': calculate_md5(content)}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_put_object_content_type(self):
+        obj = 'object'
+        content = 'abcdefghij'
+        etag = md5(content).hexdigest()
+        headers = {'Content-Type': 'text/plain'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+        self.assertEqual(status, 200)
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj)
+        self.assertEqual(headers['content-type'], 'text/plain')
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_put_object_conditional_requests(self):
+        obj = 'object'
+        content = 'abcdefghij'
+        headers = {'If-None-Match': '*'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+        self.assertEqual(status, 501)
+
+        headers = {'If-Match': '*'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+        self.assertEqual(status, 501)
+
+        headers = {'If-Modified-Since': 'Sat, 27 Jun 2015 00:00:00 GMT'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+        self.assertEqual(status, 501)
+
+        headers = {'If-Unmodified-Since': 'Sat, 27 Jun 2015 00:00:00 GMT'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+        self.assertEqual(status, 501)
+
+        # None of the above should actually have created an object
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj, {}, '')
+        self.assertEqual(status, 404)
+
+    def test_put_object_expect(self):
+        obj = 'object'
+        content = 'abcdefghij'
+        etag = md5(content).hexdigest()
+        headers = {'Expect': '100-continue'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def _test_put_object_headers(self, req_headers, expected_headers=None):
+        if expected_headers is None:
+            expected_headers = req_headers
+        obj = 'object'
+        content = 'abcdefghij'
+        etag = md5(content).hexdigest()
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj,
+                                   req_headers, content)
+        self.assertEqual(status, 200)
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj)
+        for header, value in expected_headers.items():
+            self.assertIn(header.lower(), headers)
+            self.assertEqual(headers[header.lower()], value)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_put_object_metadata(self):
+        self._test_put_object_headers({
+            'X-Amz-Meta-Bar': 'foo',
+            'X-Amz-Meta-Bar2': 'foo2'})
+
+    def test_put_object_weird_metadata(self):
+        req_headers = dict(
+            ('x-amz-meta-' + c, c)
+            for c in '!"#$%&\'()*+-./<=>?@[\\]^`{|}~')
+        exp_headers = dict(
+            ('x-amz-meta-' + c, c)
+            for c in '!#$%&\'(*+-.^`|~')
+        self._test_put_object_headers(req_headers, exp_headers)
+
+    def test_put_object_underscore_in_metadata(self):
+        # Break this out separately for ease of testing pre-0.19.0 eventlet
+        self._test_put_object_headers({
+            'X-Amz-Meta-Foo-Bar': 'baz',
+            'X-Amz-Meta-Foo_Bar': 'also baz'})
+
+    def test_put_object_content_headers(self):
+        self._test_put_object_headers({
+            'Content-Type': 'foo/bar',
+            'Content-Encoding': 'baz',
+            'Content-Disposition': 'attachment',
+            'Content-Language': 'en'})
+
+    def test_put_object_cache_control(self):
+        self._test_put_object_headers({
+            'Cache-Control': 'private, some-extension'})
+
+    def test_put_object_expires(self):
+        self._test_put_object_headers({
+            # We don't validate that the Expires header is a valid date
+            'Expires': 'a valid HTTP-date timestamp'})
+
+    def test_put_object_robots_tag(self):
+        self._test_put_object_headers({
+            'X-Robots-Tag': 'googlebot: noarchive'})
+
+    def test_put_object_storage_class(self):
+        obj = 'object'
+        content = 'abcdefghij'
+        etag = md5(content).hexdigest()
+        headers = {'X-Amz-Storage-Class': 'STANDARD'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_put_object_copy_source_params(self):
+        obj = 'object'
+        src_headers = {'X-Amz-Meta-Test': 'src'}
+        src_body = 'some content'
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_object'
+        self.conn.make_request('PUT', self.bucket, obj, src_headers, src_body)
+        self.conn.make_request('PUT', dst_bucket)
+
+        headers = {'X-Amz-Copy-Source': '/%s/%s?nonsense' % (
+            self.bucket, obj)}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers)
+        self.assertEqual(status, 400)
+        self.assertEqual(get_error_code(body), 'InvalidArgument')
+
+        headers = {'X-Amz-Copy-Source': '/%s/%s?versionId=null&nonsense' % (
+            self.bucket, obj)}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers)
+        self.assertEqual(status, 400)
+        self.assertEqual(get_error_code(body), 'InvalidArgument')
+
+        headers = {'X-Amz-Copy-Source': '/%s/%s?versionId=null' % (
+            self.bucket, obj)}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        status, headers, body = \
+            self.conn.make_request('GET', dst_bucket, dst_obj)
+        self.assertEqual(status, 200)
+        self.assertEqual(headers['x-amz-meta-test'], 'src')
+        self.assertEqual(body, src_body)
+
+    def test_put_object_copy_source(self):
+        obj = 'object'
+        content = 'abcdefghij'
+        etag = md5(content).hexdigest()
+        self.conn.make_request('PUT', self.bucket, obj, body=content)
+
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_object'
+        self.conn.make_request('PUT', dst_bucket)
+
+        # /src/src -> /dst/dst
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj)}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(dst_bucket, dst_obj, etag)
+
+        # /src/src -> /src/dst
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj)}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, dst_obj, headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, dst_obj, etag)
+
+        # /src/src -> /src/src
+        # need changes to copy itself (e.g. metadata)
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj),
+                   'X-Amz-Meta-Foo': 'bar',
+                   'X-Amz-Metadata-Directive': 'REPLACE'}
+        status, headers, body = \
+            self.conn.make_request('PUT', self.bucket, obj, headers)
+        self.assertEqual(status, 200)
+        self._assertObjectEtag(self.bucket, obj, etag)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_put_object_copy_metadata_directive(self):
+        obj = 'object'
+        src_headers = {'X-Amz-Meta-Test': 'src'}
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_object'
+        self.conn.make_request('PUT', self.bucket, obj, headers=src_headers)
+        self.conn.make_request('PUT', dst_bucket)
+
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj),
+                   'X-Amz-Metadata-Directive': 'REPLACE',
+                   'X-Amz-Meta-Test': 'dst'}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        status, headers, body = \
+            self.conn.make_request('HEAD', dst_bucket, dst_obj)
+        self.assertEqual(headers['x-amz-meta-test'], 'dst')
+
+    def test_put_object_copy_source_if_modified_since(self):
+        obj = 'object'
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_object'
+        etag = md5().hexdigest()
+        self.conn.make_request('PUT', self.bucket, obj)
+        self.conn.make_request('PUT', dst_bucket)
+
+        _, headers, _ = self.conn.make_request('HEAD', self.bucket, obj)
+        src_datetime = mktime(parsedate(headers['last-modified']))
+        src_datetime = src_datetime - DAY
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj),
+                   'X-Amz-Copy-Source-If-Modified-Since':
+                   formatdate(src_datetime)}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_put_object_copy_source_if_unmodified_since(self):
+        obj = 'object'
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_object'
+        etag = md5().hexdigest()
+        self.conn.make_request('PUT', self.bucket, obj)
+        self.conn.make_request('PUT', dst_bucket)
+
+        _, headers, _ = self.conn.make_request('HEAD', self.bucket, obj)
+        src_datetime = mktime(parsedate(headers['last-modified']))
+        src_datetime = src_datetime + DAY
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj),
+                   'X-Amz-Copy-Source-If-Unmodified-Since':
+                   formatdate(src_datetime)}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_put_object_copy_source_if_match(self):
+        obj = 'object'
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_object'
+        etag = md5().hexdigest()
+        self.conn.make_request('PUT', self.bucket, obj)
+        self.conn.make_request('PUT', dst_bucket)
+
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj)
+
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj),
+                   'X-Amz-Copy-Source-If-Match': etag}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_put_object_copy_source_if_none_match(self):
+        obj = 'object'
+        dst_bucket = 'dst-bucket'
+        dst_obj = 'dst_object'
+        etag = md5().hexdigest()
+        self.conn.make_request('PUT', self.bucket, obj)
+        self.conn.make_request('PUT', dst_bucket)
+
+        headers = {'X-Amz-Copy-Source': '/%s/%s' % (self.bucket, obj),
+                   'X-Amz-Copy-Source-If-None-Match': 'none-match'}
+        status, headers, body = \
+            self.conn.make_request('PUT', dst_bucket, dst_obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self._assertObjectEtag(self.bucket, obj, etag)
+
+    def test_get_object_response_content_type(self):
+        obj = 'obj'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        query = 'response-content-type=text/plain'
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertEqual(headers['content-type'], 'text/plain')
+
+    def test_get_object_response_content_language(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        query = 'response-content-language=en'
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertEqual(headers['content-language'], 'en')
+
+    def test_get_object_response_cache_control(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        query = 'response-cache-control=private'
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertEqual(headers['cache-control'], 'private')
+
+    def test_get_object_response_content_disposition(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        query = 'response-content-disposition=inline'
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertEqual(headers['content-disposition'], 'inline')
+
+    def test_get_object_response_content_encoding(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        query = 'response-content-encoding=gzip'
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, query=query)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+        self.assertEqual(headers['content-encoding'], 'gzip')
+
+    def test_get_object_range(self):
+        obj = 'object'
+        content = 'abcdefghij'
+        headers = {'x-amz-meta-test': 'swift'}
+        self.conn.make_request(
+            'PUT', self.bucket, obj, headers=headers, body=content)
+
+        headers = {'Range': 'bytes=1-5'}
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 206)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], '5')
+        self.assertTrue('x-amz-meta-test' in headers)
+        self.assertEqual('swift', headers['x-amz-meta-test'])
+        self.assertEqual(body, 'bcdef')
+
+        headers = {'Range': 'bytes=5-'}
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 206)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], '5')
+        self.assertTrue('x-amz-meta-test' in headers)
+        self.assertEqual('swift', headers['x-amz-meta-test'])
+        self.assertEqual(body, 'fghij')
+
+        headers = {'Range': 'bytes=-5'}
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 206)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-length' in headers)
+        self.assertEqual(headers['content-length'], '5')
+        self.assertTrue('x-amz-meta-test' in headers)
+        self.assertEqual('swift', headers['x-amz-meta-test'])
+        self.assertEqual(body, 'fghij')
+
+        ranges = ['1-2', '4-5']
+
+        headers = {'Range': 'bytes=%s' % ','.join(ranges)}
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 206)
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue('content-length' in headers)
+
+        self.assertTrue('content-type' in headers)  # sanity
+        content_type, boundary = headers['content-type'].split(';')
+
+        self.assertEqual('multipart/byteranges', content_type)
+        self.assertTrue(boundary.startswith('boundary='))  # sanity
+        boundary_str = boundary[len('boundary='):]
+
+        # TODO: Using swift.common.utils.multipart_byteranges_to_document_iters
+        #       could be easy enough.
+        parser = email.parser.FeedParser()
+        parser.feed(
+            "Content-Type: multipart/byterange; boundary=%s\r\n\r\n" %
+            boundary_str)
+        parser.feed(body)
+        message = parser.close()
+
+        self.assertTrue(message.is_multipart())  # sanity check
+        mime_parts = message.get_payload()
+        self.assertEqual(len(mime_parts), len(ranges))  # sanity
+
+        for index, range_value in enumerate(ranges):
+            start, end = map(int, range_value.split('-'))
+            # go to next section and check sanity
+            self.assertTrue(mime_parts[index])
+
+            part = mime_parts[index]
+            self.assertEqual(
+                'application/octet-stream', part.get_content_type())
+            expected_range = 'bytes %s/%s' % (range_value, len(content))
+            self.assertEqual(
+                expected_range, part.get('Content-Range'))
+            # rest
+            payload = part.get_payload().strip()
+            self.assertEqual(content[start:end + 1], payload)
+
+    def test_get_object_if_modified_since(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        _, headers, _ = self.conn.make_request('HEAD', self.bucket, obj)
+        src_datetime = mktime(parsedate(headers['last-modified']))
+        src_datetime = src_datetime - DAY
+        headers = {'If-Modified-Since': formatdate(src_datetime)}
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_get_object_if_unmodified_since(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        _, headers, _ = self.conn.make_request('HEAD', self.bucket, obj)
+        src_datetime = mktime(parsedate(headers['last-modified']))
+        src_datetime = src_datetime + DAY
+        headers = \
+            {'If-Unmodified-Since': formatdate(src_datetime)}
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_get_object_if_match(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj)
+        etag = headers['etag']
+
+        headers = {'If-Match': etag}
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_get_object_if_none_match(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        headers = {'If-None-Match': 'none-match'}
+        status, headers, body = \
+            self.conn.make_request('GET', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_head_object_range(self):
+        obj = 'object'
+        content = 'abcdefghij'
+        self.conn.make_request('PUT', self.bucket, obj, body=content)
+
+        headers = {'Range': 'bytes=1-5'}
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj, headers=headers)
+        self.assertEqual(headers['content-length'], '5')
+        self.assertCommonResponseHeaders(headers)
+
+        headers = {'Range': 'bytes=5-'}
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj, headers=headers)
+        self.assertEqual(headers['content-length'], '5')
+        self.assertCommonResponseHeaders(headers)
+
+        headers = {'Range': 'bytes=-5'}
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj, headers=headers)
+        self.assertEqual(headers['content-length'], '5')
+        self.assertCommonResponseHeaders(headers)
+
+    def test_head_object_if_modified_since(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        _, headers, _ = self.conn.make_request('HEAD', self.bucket, obj)
+        dt = mktime(parsedate(headers['last-modified']))
+        dt = dt - DAY
+
+        headers = {'If-Modified-Since': formatdate(dt)}
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_head_object_if_unmodified_since(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        _, headers, _ = self.conn.make_request('HEAD', self.bucket, obj)
+        dt = mktime(parsedate(headers['last-modified']))
+        dt = dt + DAY
+
+        headers = {'If-Unmodified-Since': formatdate(dt)}
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_head_object_if_match(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj)
+        etag = headers['etag']
+
+        headers = {'If-Match': etag}
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+
+    def test_head_object_if_none_match(self):
+        obj = 'object'
+        self.conn.make_request('PUT', self.bucket, obj)
+
+        headers = {'If-None-Match': 'none-match'}
+        status, headers, body = \
+            self.conn.make_request('HEAD', self.bucket, obj, headers=headers)
+        self.assertEqual(status, 200)
+        self.assertCommonResponseHeaders(headers)
+
+
+class TestS3ApiObjectSigV4(TestS3ApiObject):
+    @classmethod
+    def setUpClass(cls):
+        os.environ['S3_USE_SIGV4'] = "True"
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['S3_USE_SIGV4']
+
+    def setUp(self):
+        super(TestS3ApiObjectSigV4, self).setUp()
+
+    @unittest2.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
+                      'This stuff got the signing issue of boto<=2.x')
+    def test_put_object_metadata(self):
+        super(TestS3ApiObjectSigV4, self).test_put_object_metadata()
+
+    @unittest2.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
+                      'This stuff got the signing issue of boto<=2.x')
+    def test_put_object_copy_source_if_modified_since(self):
+        super(TestS3ApiObjectSigV4, self).\
+            test_put_object_copy_source_if_modified_since()
+
+    @unittest2.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
+                      'This stuff got the signing issue of boto<=2.x')
+    def test_put_object_copy_source_if_unmodified_since(self):
+        super(TestS3ApiObjectSigV4, self).\
+            test_put_object_copy_source_if_unmodified_since()
+
+    @unittest2.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
+                      'This stuff got the signing issue of boto<=2.x')
+    def test_put_object_copy_source_if_match(self):
+        super(TestS3ApiObjectSigV4,
+              self).test_put_object_copy_source_if_match()
+
+    @unittest2.skipIf(StrictVersion(boto.__version__) < StrictVersion('3.0'),
+                      'This stuff got the signing issue of boto<=2.x')
+    def test_put_object_copy_source_if_none_match(self):
+        super(TestS3ApiObjectSigV4,
+              self).test_put_object_copy_source_if_none_match()
+
+
+if __name__ == '__main__':
+    unittest2.main()

test/functional/s3api/test_presigned.py

@@ -0,0 +1,237 @@
+# Copyright (c) 2016 SwiftStack, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import requests
+
+from swift.common.middleware.s3api.etree import fromstring
+
+import test.functional as tf
+
+from test.functional.s3api import S3ApiBase
+from test.functional.s3api.utils import get_error_code, get_error_msg
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class TestS3ApiPresignedUrls(S3ApiBase):
+    def test_bucket(self):
+        bucket = 'test-bucket'
+        req_objects = ('object', 'object2')
+        max_bucket_listing = tf.cluster_info['s3api'].get(
+            'max_bucket_listing', 1000)
+
+        # GET Bucket (Without Object)
+        status, _junk, _junk = self.conn.make_request('PUT', bucket)
+        self.assertEqual(status, 200)
+
+        url, headers = self.conn.generate_url_and_headers('GET', bucket)
+        resp = requests.get(url, headers=headers)
+        self.assertEqual(resp.status_code, 200,
+                         'Got %d %s' % (resp.status_code, resp.content))
+        self.assertCommonResponseHeaders(resp.headers)
+        self.assertIsNotNone(resp.headers['content-type'])
+        self.assertEqual(resp.headers['content-length'],
+                         str(len(resp.content)))
+
+        elem = fromstring(resp.content, 'ListBucketResult')
+        self.assertEqual(elem.find('Name').text, bucket)
+        self.assertIsNone(elem.find('Prefix').text)
+        self.assertIsNone(elem.find('Marker').text)
+        self.assertEqual(elem.find('MaxKeys').text,
+                         str(max_bucket_listing))
+        self.assertEqual(elem.find('IsTruncated').text, 'false')
+        objects = elem.findall('./Contents')
+        self.assertEqual(list(objects), [])
+
+        # GET Bucket (With Object)
+        for obj in req_objects:
+            status, _junk, _junk = self.conn.make_request('PUT', bucket, obj)
+            self.assertEqual(
+                status, 200,
+                'Got %d response while creating %s' % (status, obj))
+
+        resp = requests.get(url, headers=headers)
+        self.assertEqual(resp.status_code, 200,
+                         'Got %d %s' % (resp.status_code, resp.content))
+        self.assertCommonResponseHeaders(resp.headers)
+        self.assertIsNotNone(resp.headers['content-type'])
+        self.assertEqual(resp.headers['content-length'],
+                         str(len(resp.content)))
+
+        elem = fromstring(resp.content, 'ListBucketResult')
+        self.assertEqual(elem.find('Name').text, bucket)
+        self.assertIsNone(elem.find('Prefix').text)
+        self.assertIsNone(elem.find('Marker').text)
+        self.assertEqual(elem.find('MaxKeys').text,
+                         str(max_bucket_listing))
+        self.assertEqual(elem.find('IsTruncated').text, 'false')
+        resp_objects = elem.findall('./Contents')
+        self.assertEqual(len(list(resp_objects)), 2)
+        for o in resp_objects:
+            self.assertIn(o.find('Key').text, req_objects)
+            self.assertIsNotNone(o.find('LastModified').text)
+            self.assertRegexpMatches(
+                o.find('LastModified').text,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+            self.assertIsNotNone(o.find('ETag').text)
+            self.assertEqual(o.find('Size').text, '0')
+            self.assertIsNotNone(o.find('StorageClass').text is not None)
+            self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
+            self.assertEqual(o.find('Owner/DisplayName').text,
+                             self.conn.user_id)
+        # DELETE Bucket
+        for obj in req_objects:
+            self.conn.make_request('DELETE', bucket, obj)
+        url, headers = self.conn.generate_url_and_headers('DELETE', bucket)
+        resp = requests.delete(url, headers=headers)
+        self.assertEqual(resp.status_code, 204,
+                         'Got %d %s' % (resp.status_code, resp.content))
+
+    def test_expiration_limits(self):
+        if os.environ.get('S3_USE_SIGV4'):
+            self._test_expiration_limits_v4()
+        else:
+            self._test_expiration_limits_v2()
+
+    def _test_expiration_limits_v2(self):
+        bucket = 'test-bucket'
+
+        # Expiration date is too far in the future
+        url, headers = self.conn.generate_url_and_headers(
+            'GET', bucket, expires_in=2 ** 32)
+        resp = requests.get(url, headers=headers)
+        self.assertEqual(resp.status_code, 403,
+                         'Got %d %s' % (resp.status_code, resp.content))
+        self.assertEqual(get_error_code(resp.content),
+                         'AccessDenied')
+        self.assertIn('Invalid date (should be seconds since epoch)',
+                      get_error_msg(resp.content))
+
+    def _test_expiration_limits_v4(self):
+        bucket = 'test-bucket'
+
+        # Expiration is negative
+        url, headers = self.conn.generate_url_and_headers(
+            'GET', bucket, expires_in=-1)
+        resp = requests.get(url, headers=headers)
+        self.assertEqual(resp.status_code, 400,
+                         'Got %d %s' % (resp.status_code, resp.content))
+        self.assertEqual(get_error_code(resp.content),
+                         'AuthorizationQueryParametersError')
+        self.assertIn('X-Amz-Expires must be non-negative',
+                      get_error_msg(resp.content))
+
+        # Expiration date is too far in the future
+        for exp in (7 * 24 * 60 * 60 + 1,
+                    2 ** 63 - 1):
+            url, headers = self.conn.generate_url_and_headers(
+                'GET', bucket, expires_in=exp)
+            resp = requests.get(url, headers=headers)
+            self.assertEqual(resp.status_code, 400,
+                             'Got %d %s' % (resp.status_code, resp.content))
+            self.assertEqual(get_error_code(resp.content),
+                             'AuthorizationQueryParametersError')
+            self.assertIn('X-Amz-Expires must be less than 604800 seconds',
+                          get_error_msg(resp.content))
+
+        # Expiration date is *way* too far in the future, or isn't a number
+        for exp in (2 ** 63, 'foo'):
+            url, headers = self.conn.generate_url_and_headers(
+                'GET', bucket, expires_in=2 ** 63)
+            resp = requests.get(url, headers=headers)
+            self.assertEqual(resp.status_code, 400,
+                             'Got %d %s' % (resp.status_code, resp.content))
+            self.assertEqual(get_error_code(resp.content),
+                             'AuthorizationQueryParametersError')
+            self.assertEqual('X-Amz-Expires should be a number',
+                             get_error_msg(resp.content))
+
+    def test_object(self):
+        bucket = 'test-bucket'
+        obj = 'object'
+
+        status, _junk, _junk = self.conn.make_request('PUT', bucket)
+        self.assertEqual(status, 200)
+
+        # HEAD/missing object
+        head_url, headers = self.conn.generate_url_and_headers(
+            'HEAD', bucket, obj)
+        resp = requests.head(head_url, headers=headers)
+        self.assertEqual(resp.status_code, 404,
+                         'Got %d %s' % (resp.status_code, resp.content))
+
+        # Wrong verb
+        resp = requests.get(head_url)
+        self.assertEqual(resp.status_code, 403,
+                         'Got %d %s' % (resp.status_code, resp.content))
+        self.assertEqual(get_error_code(resp.content),
+                         'SignatureDoesNotMatch')
+
+        # PUT empty object
+        put_url, headers = self.conn.generate_url_and_headers(
+            'PUT', bucket, obj)
+        resp = requests.put(put_url, data='', headers=headers)
+        self.assertEqual(resp.status_code, 200,
+                         'Got %d %s' % (resp.status_code, resp.content))
+        # GET empty object
+        get_url, headers = self.conn.generate_url_and_headers(
+            'GET', bucket, obj)
+        resp = requests.get(get_url, headers=headers)
+        self.assertEqual(resp.status_code, 200,
+                         'Got %d %s' % (resp.status_code, resp.content))
+        self.assertEqual(resp.content, '')
+
+        # PUT over object
+        resp = requests.put(put_url, data='foobar', headers=headers)
+        self.assertEqual(resp.status_code, 200,
+                         'Got %d %s' % (resp.status_code, resp.content))
+
+        # GET non-empty object
+        resp = requests.get(get_url, headers=headers)
+        self.assertEqual(resp.status_code, 200,
+                         'Got %d %s' % (resp.status_code, resp.content))
+        self.assertEqual(resp.content, 'foobar')
+
+        # DELETE Object
+        delete_url, headers = self.conn.generate_url_and_headers(
+            'DELETE', bucket, obj)
+        resp = requests.delete(delete_url, headers=headers)
+        self.assertEqual(resp.status_code, 204,
+                         'Got %d %s' % (resp.status_code, resp.content))
+
+        # Final cleanup
+        status, _junk, _junk = self.conn.make_request('DELETE', bucket)
+        self.assertEqual(status, 204)
+
+
+class TestS3ApiPresignedUrlsSigV4(TestS3ApiPresignedUrls):
+    @classmethod
+    def setUpClass(cls):
+        os.environ['S3_USE_SIGV4'] = "True"
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['S3_USE_SIGV4']
+
+    def setUp(self):
+        super(TestS3ApiPresignedUrlsSigV4, self).setUp()

test/functional/s3api/test_service.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest2
+import os
+
+import test.functional as tf
+
+from swift.common.middleware.s3api.etree import fromstring
+
+from test.functional.s3api import S3ApiBase
+from test.functional.s3api.s3_test_client import Connection
+from test.functional.s3api.utils import get_error_code
+
+
+def setUpModule():
+    tf.setup_package()
+
+
+def tearDownModule():
+    tf.teardown_package()
+
+
+class TestS3ApiService(S3ApiBase):
+    def setUp(self):
+        super(TestS3ApiService, self).setUp()
+
+    def test_service(self):
+        # GET Service(without bucket)
+        status, headers, body = self.conn.make_request('GET')
+        self.assertEqual(status, 200)
+
+        self.assertCommonResponseHeaders(headers)
+        self.assertTrue(headers['content-type'] is not None)
+        # TODO; requires consideration
+        # self.assertEqual(headers['transfer-encoding'], 'chunked')
+
+        elem = fromstring(body, 'ListAllMyBucketsResult')
+        buckets = elem.findall('./Buckets/Bucket')
+        self.assertEqual(list(buckets), [])
+        owner = elem.find('Owner')
+        self.assertEqual(self.conn.user_id, owner.find('ID').text)
+        self.assertEqual(self.conn.user_id, owner.find('DisplayName').text)
+
+        # GET Service(with Bucket)
+        req_buckets = ('bucket', 'bucket2')
+        for bucket in req_buckets:
+            self.conn.make_request('PUT', bucket)
+        status, headers, body = self.conn.make_request('GET')
+        self.assertEqual(status, 200)
+
+        elem = fromstring(body, 'ListAllMyBucketsResult')
+        resp_buckets = elem.findall('./Buckets/Bucket')
+        self.assertEqual(len(list(resp_buckets)), 2)
+        for b in resp_buckets:
+            self.assertTrue(b.find('Name').text in req_buckets)
+            self.assertTrue(b.find('CreationDate') is not None)
+
+    def test_service_error_signature_not_match(self):
+        auth_error_conn = Connection(aws_secret_key='invalid')
+        status, headers, body = auth_error_conn.make_request('GET')
+        self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
+        self.assertEqual(headers['content-type'], 'application/xml')
+
+    def test_service_error_no_date_header(self):
+        # Without x-amz-date/Date header, that makes 403 forbidden
+        status, headers, body = self.conn.make_request(
+            'GET', headers={'Date': '', 'x-amz-date': ''})
+        self.assertEqual(status, 403)
+        self.assertEqual(get_error_code(body), 'AccessDenied')
+        self.assertIn('AWS authentication requires a valid Date '
+                      'or x-amz-date header', body)
+
+
+class TestS3ApiServiceSigV4(TestS3ApiService):
+    @classmethod
+    def setUpClass(cls):
+        os.environ['S3_USE_SIGV4'] = "True"
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['S3_USE_SIGV4']
+
+    def setUp(self):
+        super(TestS3ApiServiceSigV4, self).setUp()
+
+if __name__ == '__main__':
+    unittest2.main()

test/functional/s3api/utils.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from hashlib import md5
+from swift.common.middleware.s3api.etree import fromstring
+
+
+def get_error_code(body):
+    elem = fromstring(body, 'Error')
+    return elem.find('Code').text
+
+
+def get_error_msg(body):
+    elem = fromstring(body, 'Error')
+    return elem.find('Message').text
+
+
+def calculate_md5(body):
+    return md5(body).digest().encode('base64').strip()

test/sample.conf

@@ -17,6 +17,8 @@ auth_prefix = /auth/
 account = test
 username = tester
 password = testing
+s3_access_key = test:tester
+s3_secret_key = testing
 
 # User on a second account (needs admin access to the account)
 account2 = test2
@@ -26,6 +28,9 @@ password2 = testing2
 # User on same account as first, but without admin access
 username3 = tester3
 password3 = testing3
+# s3api requires the same account with the primary one and different users
+s3_access_key2 = test:tester3
+s3_secret_key2 = testing3
 
 # Fourth user is required for keystone v3 specific tests.
 # Account must be in a non-default domain.

test/unit/common/middleware/s3api/__init__.py

@@ -0,0 +1,163 @@
+# Copyright (c) 2011-2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from datetime import datetime
+import email
+import time
+
+from swift.common import swob
+
+from swift.common.middleware.s3api.s3api import S3ApiMiddleware
+from helpers import FakeSwift
+from swift.common.middleware.s3api.etree import fromstring
+from swift.common.middleware.s3api.utils import Config
+
+
+class FakeApp(object):
+    def __init__(self):
+        self.swift = FakeSwift()
+
+    def _update_s3_path_info(self, env):
+        """
+        For S3 requests, Swift auth middleware replaces a user name in
+        env['PATH_INFO'] with a valid tenant id.
+        E.g. '/v1/test:tester/bucket/object' will become
+        '/v1/AUTH_test/bucket/object'. This method emulates the behavior.
+        """
+        _, authorization = env['HTTP_AUTHORIZATION'].split(' ')
+        tenant_user, sign = authorization.rsplit(':', 1)
+        tenant, user = tenant_user.rsplit(':', 1)
+
+        path = env['PATH_INFO']
+        env['PATH_INFO'] = path.replace(tenant_user, 'AUTH_' + tenant)
+
+    def __call__(self, env, start_response):
+        if 'HTTP_AUTHORIZATION' in env:
+            self._update_s3_path_info(env)
+
+        return self.swift(env, start_response)
+
+
+class S3ApiTestCase(unittest.TestCase):
+    def __init__(self, name):
+        unittest.TestCase.__init__(self, name)
+
+    def setUp(self):
+        # setup default config
+        self.conf = Config({
+            'allow_no_owner': False,
+            'location': 'US',
+            'dns_compliant_bucket_names': True,
+            'max_bucket_listing': 1000,
+            'max_parts_listing': 1000,
+            'max_multi_delete_objects': 1000,
+            's3_acl': False,
+            'storage_domain': 'localhost',
+            'auth_pipeline_check': True,
+            'max_upload_part_num': 1000,
+            'check_bucket_owner': False,
+            'force_swift_request_proxy_log': False,
+            'allow_multipart_uploads': True,
+            'min_segment_size': 5242880,
+        })
+        # those 2 settings has existed the original test setup
+        self.conf.log_level = 'debug'
+
+        self.app = FakeApp()
+        self.swift = self.app.swift
+        self.s3api = S3ApiMiddleware(self.app, self.conf)
+
+        self.swift.register('HEAD', '/v1/AUTH_test',
+                            swob.HTTPOk, {}, None)
+        self.swift.register('HEAD', '/v1/AUTH_test/bucket',
+                            swob.HTTPNoContent, {}, None)
+        self.swift.register('PUT', '/v1/AUTH_test/bucket',
+                            swob.HTTPCreated, {}, None)
+        self.swift.register('POST', '/v1/AUTH_test/bucket',
+                            swob.HTTPNoContent, {}, None)
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket',
+                            swob.HTTPNoContent, {}, None)
+
+        self.swift.register('GET', '/v1/AUTH_test/bucket/object',
+                            swob.HTTPOk, {}, "")
+        self.swift.register('PUT', '/v1/AUTH_test/bucket/object',
+                            swob.HTTPCreated, {}, None)
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket/object',
+                            swob.HTTPNoContent, {}, None)
+
+    def _get_error_code(self, body):
+        elem = fromstring(body, 'Error')
+        return elem.find('./Code').text
+
+    def _get_error_message(self, body):
+        elem = fromstring(body, 'Error')
+        return elem.find('./Message').text
+
+    def _test_method_error(self, method, path, response_class, headers={}):
+        if not path.startswith('/'):
+            path = '/' + path  # add a missing slash before the path
+
+        uri = '/v1/AUTH_test'
+        if path != '/':
+            uri += path
+
+        self.swift.register(method, uri, response_class, headers, None)
+        headers.update({'Authorization': 'AWS test:tester:hmac',
+                        'Date': self.get_date_header()})
+        req = swob.Request.blank(path, environ={'REQUEST_METHOD': method},
+                                 headers=headers)
+        status, headers, body = self.call_s3api(req)
+        return self._get_error_code(body)
+
+    def get_date_header(self):
+        # email.utils.formatdate returns utc timestamp in default
+        return email.utils.formatdate(time.time())
+
+    def get_v4_amz_date_header(self):
+        return datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
+
+    def call_app(self, req, app=None, expect_exception=False):
+        if app is None:
+            app = self.app
+
+        req.headers.setdefault("User-Agent", "Mozzarella Foxfire")
+
+        status = [None]
+        headers = [None]
+
+        def start_response(s, h, ei=None):
+            status[0] = s
+            headers[0] = swob.HeaderKeyDict(h)
+
+        body_iter = app(req.environ, start_response)
+        body = ''
+        caught_exc = None
+        try:
+            for chunk in body_iter:
+                body += chunk
+        except Exception as exc:
+            if expect_exception:
+                caught_exc = exc
+            else:
+                raise
+
+        if expect_exception:
+            return status[0], headers[0], body, caught_exc
+        else:
+            return status[0], headers[0], body
+
+    def call_s3api(self, req, **kwargs):
+        return self.call_app(req, app=self.s3api, **kwargs)

test/unit/common/middleware/s3api/exceptions.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class NotMethodException(Exception):
+    pass

test/unit/common/middleware/s3api/helpers.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This stuff can't live in test/unit/__init__.py due to its swob dependency.
+
+from copy import deepcopy
+from hashlib import md5
+from swift.common import swob
+from swift.common.utils import split_path
+from swift.common.request_helpers import is_sys_meta
+
+
+class FakeSwift(object):
+    """
+    A good-enough fake Swift proxy server to use in testing middleware.
+    """
+
+    def __init__(self, s3_acl=False):
+        self._calls = []
+        self.req_method_paths = []
+        self.swift_sources = []
+        self.uploaded = {}
+        # mapping of (method, path) --> (response class, headers, body)
+        self._responses = {}
+        self.s3_acl = s3_acl
+
+    def _fake_auth_middleware(self, env):
+        if 'swift.authorize_override' in env:
+            return
+
+        if 'HTTP_AUTHORIZATION' not in env:
+            return
+
+        _, authorization = env['HTTP_AUTHORIZATION'].split(' ')
+        tenant_user, sign = authorization.rsplit(':', 1)
+        tenant, user = tenant_user.rsplit(':', 1)
+
+        path = env['PATH_INFO']
+        env['PATH_INFO'] = path.replace(tenant_user, 'AUTH_' + tenant)
+
+        env['REMOTE_USER'] = 'authorized'
+
+        if env['REQUEST_METHOD'] == 'TEST':
+            # AccessDenied by default at s3acl authenticate
+            env['swift.authorize'] = \
+                lambda req: swob.HTTPForbidden(request=req)
+        else:
+            env['swift.authorize'] = lambda req: None
+
+    def __call__(self, env, start_response):
+        if self.s3_acl:
+            self._fake_auth_middleware(env)
+
+        req = swob.Request(env)
+        method = env['REQUEST_METHOD']
+        path = env['PATH_INFO']
+        _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
+                                       rest_with_last=True)
+        if env.get('QUERY_STRING'):
+            path += '?' + env['QUERY_STRING']
+
+        if 'swift.authorize' in env:
+            resp = env['swift.authorize'](req)
+            if resp:
+                return resp(env, start_response)
+
+        headers = req.headers
+        self._calls.append((method, path, headers))
+        self.swift_sources.append(env.get('swift.source'))
+
+        try:
+            resp_class, raw_headers, body = self._responses[(method, path)]
+            headers = swob.HeaderKeyDict(raw_headers)
+        except KeyError:
+            # FIXME: suppress print state error for python3 compatibility.
+            # pylint: disable-msg=E1601
+            if (env.get('QUERY_STRING')
+                    and (method, env['PATH_INFO']) in self._responses):
+                resp_class, raw_headers, body = self._responses[
+                    (method, env['PATH_INFO'])]
+                headers = swob.HeaderKeyDict(raw_headers)
+            elif method == 'HEAD' and ('GET', path) in self._responses:
+                resp_class, raw_headers, _ = self._responses[('GET', path)]
+                body = None
+                headers = swob.HeaderKeyDict(raw_headers)
+            elif method == 'GET' and obj and path in self.uploaded:
+                resp_class = swob.HTTPOk
+                headers, body = self.uploaded[path]
+            else:
+                print("Didn't find %r in allowed responses" %
+                      ((method, path),))
+                raise
+
+        # simulate object PUT
+        if method == 'PUT' and obj:
+            input = env['wsgi.input'].read()
+            etag = md5(input).hexdigest()
+            headers.setdefault('Etag', etag)
+            headers.setdefault('Content-Length', len(input))
+
+            # keep it for subsequent GET requests later
+            self.uploaded[path] = (deepcopy(headers), input)
+            if "CONTENT_TYPE" in env:
+                self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
+
+        # range requests ought to work, but copies are special
+        support_range_and_conditional = not (
+            method == 'PUT' and
+            'X-Copy-From' in req.headers and
+            'Range' in req.headers)
+        resp = resp_class(req=req, headers=headers, body=body,
+                          conditional_response=support_range_and_conditional)
+        return resp(env, start_response)
+
+    @property
+    def calls(self):
+        return [(method, path) for method, path, headers in self._calls]
+
+    @property
+    def calls_with_headers(self):
+        return self._calls
+
+    @property
+    def call_count(self):
+        return len(self._calls)
+
+    def register(self, method, path, response_class, headers, body):
+        # assuming the path format like /v1/account/container/object
+        resource_map = ['account', 'container', 'object']
+        acos = filter(None, split_path(path, 0, 4, True)[1:])
+        index = len(acos) - 1
+        resource = resource_map[index]
+        if (method, path) in self._responses:
+            old_headers = self._responses[(method, path)][1]
+            headers = headers.copy()
+            for key, value in old_headers.iteritems():
+                if is_sys_meta(resource, key) and key not in headers:
+                    # keep old sysmeta for s3acl
+                    headers.update({key: value})
+
+        self._responses[(method, path)] = (response_class, headers, body)
+
+    def register_unconditionally(self, method, path, response_class, headers,
+                                 body):
+        # register() keeps old sysmeta around, but
+        # register_unconditionally() keeps nothing.
+        self._responses[(method, path)] = (response_class, headers, body)
+
+    def clear_calls(self):
+        del self._calls[:]
+
+
+class UnreadableInput(object):
+    # Some clients will send neither a Content-Length nor a Transfer-Encoding
+    # header, which will cause (some versions of?) eventlet to bomb out on
+    # reads. This class helps us simulate that behavior.
+    def __init__(self, test_case):
+        self.calls = 0
+        self.test_case = test_case
+
+    def read(self, *a, **kw):
+        self.calls += 1
+        # Calling wsgi.input.read with neither a Content-Length nor
+        # a Transfer-Encoding header will raise TypeError (See
+        # https://bugs.launchpad.net/swift3/+bug/1593870 in detail)
+        # This unreadable class emulates the behavior
+        raise TypeError
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.test_case.assertEqual(0, self.calls)

test/unit/common/middleware/s3api/test_acl.py

@@ -0,0 +1,230 @@
+# Copyright (c) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import mock
+
+from cStringIO import StringIO
+from hashlib import md5
+
+from swift.common.swob import Request, HTTPAccepted
+from swift.common.middleware.s3api.etree import fromstring, tostring, \
+    Element, SubElement, XMLNS_XSI
+from swift.common.middleware.s3api.s3response import InvalidArgument
+from swift.common.middleware.s3api.acl_utils import handle_acl_header
+
+from test.unit.common.middleware.s3api import S3ApiTestCase
+from test.unit.common.middleware.s3api.helpers import UnreadableInput
+from test.unit.common.middleware.s3api.test_s3_acl import s3acl
+
+
+class TestS3ApiAcl(S3ApiTestCase):
+
+    def setUp(self):
+        super(TestS3ApiAcl, self).setUp()
+        # All ACL API should be called against to existing bucket.
+        self.swift.register('PUT', '/v1/AUTH_test/bucket',
+                            HTTPAccepted, {}, None)
+
+    def _check_acl(self, owner, body):
+        elem = fromstring(body, 'AccessControlPolicy')
+        permission = elem.find('./AccessControlList/Grant/Permission').text
+        self.assertEqual(permission, 'FULL_CONTROL')
+        name = elem.find('./AccessControlList/Grant/Grantee/ID').text
+        self.assertEqual(name, owner)
+
+    def test_bucket_acl_GET(self):
+        req = Request.blank('/bucket?acl',
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self._check_acl('test:tester', body)
+
+    def test_bucket_acl_PUT(self):
+        elem = Element('AccessControlPolicy')
+        owner = SubElement(elem, 'Owner')
+        SubElement(owner, 'ID').text = 'id'
+        acl = SubElement(elem, 'AccessControlList')
+        grant = SubElement(acl, 'Grant')
+        grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
+        grantee.set('{%s}type' % XMLNS_XSI, 'Group')
+        SubElement(grantee, 'URI').text = \
+            'http://acs.amazonaws.com/groups/global/AllUsers'
+        SubElement(grant, 'Permission').text = 'READ'
+
+        xml = tostring(elem)
+        req = Request.blank('/bucket?acl',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()},
+                            body=xml)
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+        req = Request.blank('/bucket?acl',
+                            environ={'REQUEST_METHOD': 'PUT',
+                                     'wsgi.input': StringIO(xml)},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'Transfer-Encoding': 'chunked'})
+        self.assertIsNone(req.content_length)
+        self.assertIsNone(req.message_length())
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+    def test_bucket_canned_acl_PUT(self):
+        req = Request.blank('/bucket?acl',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'X-AMZ-ACL': 'public-read'})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_canned_acl_PUT_with_s3acl(self):
+        req = Request.blank('/bucket?acl',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'X-AMZ-ACL': 'public-read'})
+        with mock.patch('swift.common.middleware.s3api.s3request.'
+                        'handle_acl_header') as mock_handler:
+            status, headers, body = self.call_s3api(req)
+            self.assertEqual(status.split()[0], '200')
+            self.assertEqual(mock_handler.call_count, 0)
+
+    def test_bucket_fails_with_both_acl_header_and_xml_PUT(self):
+        elem = Element('AccessControlPolicy')
+        owner = SubElement(elem, 'Owner')
+        SubElement(owner, 'ID').text = 'id'
+        acl = SubElement(elem, 'AccessControlList')
+        grant = SubElement(acl, 'Grant')
+        grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
+        grantee.set('{%s}type' % XMLNS_XSI, 'Group')
+        SubElement(grantee, 'URI').text = \
+            'http://acs.amazonaws.com/groups/global/AllUsers'
+        SubElement(grant, 'Permission').text = 'READ'
+
+        xml = tostring(elem)
+        req = Request.blank('/bucket?acl',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'X-AMZ-ACL': 'public-read'},
+                            body=xml)
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(self._get_error_code(body),
+                         'UnexpectedContent')
+
+    def _test_put_no_body(self, use_content_length=False,
+                          use_transfer_encoding=False, string_to_md5=''):
+        content_md5 = md5(string_to_md5).digest().encode('base64').strip()
+        with UnreadableInput(self) as fake_input:
+            req = Request.blank(
+                '/bucket?acl',
+                environ={
+                    'REQUEST_METHOD': 'PUT',
+                    'wsgi.input': fake_input},
+                headers={
+                    'Authorization': 'AWS test:tester:hmac',
+                    'Date': self.get_date_header(),
+                    'Content-MD5': content_md5},
+                body='')
+            if not use_content_length:
+                req.environ.pop('CONTENT_LENGTH')
+            if use_transfer_encoding:
+                req.environ['HTTP_TRANSFER_ENCODING'] = 'chunked'
+            status, headers, body = self.call_s3api(req)
+        self.assertEqual(status, '400 Bad Request')
+        self.assertEqual(self._get_error_code(body), 'MissingSecurityHeader')
+        self.assertEqual(self._get_error_message(body),
+                         'Your request was missing a required header.')
+        self.assertIn('<MissingHeaderName>x-amz-acl</MissingHeaderName>', body)
+
+    @s3acl
+    def test_bucket_fails_with_neither_acl_header_nor_xml_PUT(self):
+        self._test_put_no_body()
+        self._test_put_no_body(string_to_md5='test')
+        self._test_put_no_body(use_content_length=True)
+        self._test_put_no_body(use_content_length=True, string_to_md5='test')
+        self._test_put_no_body(use_transfer_encoding=True)
+        self._test_put_no_body(use_transfer_encoding=True, string_to_md5='zz')
+
+    def test_object_acl_GET(self):
+        req = Request.blank('/bucket/object?acl',
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self._check_acl('test:tester', body)
+
+    def test_invalid_xml(self):
+        req = Request.blank('/bucket?acl',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()},
+                            body='invalid')
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(self._get_error_code(body), 'MalformedACLError')
+
+    def test_handle_acl_header(self):
+        def check_generated_acl_header(acl, targets):
+            req = Request.blank('/bucket',
+                                headers={'X-Amz-Acl': acl})
+            handle_acl_header(req)
+            for target in targets:
+                self.assertTrue(target[0] in req.headers)
+                self.assertEqual(req.headers[target[0]], target[1])
+
+        check_generated_acl_header('public-read',
+                                   [('X-Container-Read', '.r:*,.rlistings')])
+        check_generated_acl_header('public-read-write',
+                                   [('X-Container-Read', '.r:*,.rlistings'),
+                                    ('X-Container-Write', '.r:*')])
+        check_generated_acl_header('private',
+                                   [('X-Container-Read', '.'),
+                                    ('X-Container-Write', '.')])
+
+    @s3acl(s3acl_only=True)
+    def test_handle_acl_header_with_s3acl(self):
+        def check_generated_acl_header(acl, targets):
+            req = Request.blank('/bucket',
+                                headers={'X-Amz-Acl': acl})
+            for target in targets:
+                self.assertTrue(target not in req.headers)
+            self.assertTrue('HTTP_X_AMZ_ACL' in req.environ)
+            # TODO: add transration and assertion for s3acl
+
+        check_generated_acl_header('public-read',
+                                   ['X-Container-Read'])
+        check_generated_acl_header('public-read-write',
+                                   ['X-Container-Read', 'X-Container-Write'])
+        check_generated_acl_header('private',
+                                   ['X-Container-Read', 'X-Container-Write'])
+
+    def test_handle_acl_with_invalid_header_string(self):
+        req = Request.blank('/bucket', headers={'X-Amz-Acl': 'invalid'})
+        with self.assertRaises(InvalidArgument) as cm:
+            handle_acl_header(req)
+        self.assertTrue('argument_name' in cm.exception.info)
+        self.assertEqual(cm.exception.info['argument_name'], 'x-amz-acl')
+        self.assertTrue('argument_value' in cm.exception.info)
+        self.assertEqual(cm.exception.info['argument_value'], 'invalid')
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/middleware/s3api/test_acl_handlers.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from swift.common.middleware.s3api.acl_handlers import S3AclHandler, \
+    BucketAclHandler, ObjectAclHandler, BaseAclHandler, PartAclHandler, \
+    UploadAclHandler, UploadsAclHandler, get_acl_handler
+
+
+class TestAclHandlers(unittest.TestCase):
+    def test_get_acl_handler(self):
+        expected_handlers = (('Bucket', BucketAclHandler),
+                             ('Object', ObjectAclHandler),
+                             ('S3Acl', S3AclHandler),
+                             ('Part', PartAclHandler),
+                             ('Upload', UploadAclHandler),
+                             ('Uploads', UploadsAclHandler),
+                             ('Foo', BaseAclHandler))
+        for name, expected in expected_handlers:
+            handler = get_acl_handler(name)
+            self.assertTrue(issubclass(handler, expected))
+
+    def test_handle_acl(self):
+        # we have already have tests for s3_acl checking at test_s3_acl.py
+        pass
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/middleware/s3api/test_acl_utils.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from swift.common.swob import Request
+from swift.common.middleware.s3api.acl_utils import handle_acl_header
+
+from test.unit.common.middleware.s3api import S3ApiTestCase
+
+
+class TestS3ApiAclUtils(S3ApiTestCase):
+
+    def setUp(self):
+        super(TestS3ApiAclUtils, self).setUp()
+
+    def test_handle_acl_header(self):
+        def check_generated_acl_header(acl, targets):
+            req = Request.blank('/bucket',
+                                headers={'X-Amz-Acl': acl})
+            handle_acl_header(req)
+            for target in targets:
+                self.assertTrue(target[0] in req.headers)
+                self.assertEqual(req.headers[target[0]], target[1])
+
+        check_generated_acl_header('public-read',
+                                   [('X-Container-Read', '.r:*,.rlistings')])
+        check_generated_acl_header('public-read-write',
+                                   [('X-Container-Read', '.r:*,.rlistings'),
+                                    ('X-Container-Write', '.r:*')])
+        check_generated_acl_header('private',
+                                   [('X-Container-Read', '.'),
+                                    ('X-Container-Write', '.')])
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/middleware/s3api/test_bucket.py

@@ -0,0 +1,755 @@
+# Copyright (c) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import cgi
+
+from swift.common import swob
+from swift.common.swob import Request
+from swift.common.utils import json
+
+from swift.common.middleware.s3api.etree import fromstring, tostring, \
+    Element, SubElement
+from swift.common.middleware.s3api.subresource import Owner, encode_acl, \
+    ACLPublicRead
+from swift.common.middleware.s3api.s3request import MAX_32BIT_INT
+
+from test.unit.common.middleware.s3api import S3ApiTestCase
+from test.unit.common.middleware.s3api.test_s3_acl import s3acl
+from test.unit.common.middleware.s3api.helpers import UnreadableInput
+
+
+class TestS3ApiBucket(S3ApiTestCase):
+    def setup_objects(self):
+        self.objects = (('rose', '2011-01-05T02:19:14.275290', 0, 303),
+                        ('viola', '2011-01-05T02:19:14.275290', '0', 3909),
+                        ('lily', '2011-01-05T02:19:14.275290', '0', '3909'),
+                        ('with space', '2011-01-05T02:19:14.275290', 0, 390),
+                        ('with%20space', '2011-01-05T02:19:14.275290', 0, 390))
+
+        objects = map(
+            lambda item: {'name': str(item[0]), 'last_modified': str(item[1]),
+                          'hash': str(item[2]), 'bytes': str(item[3])},
+            list(self.objects))
+        object_list = json.dumps(objects)
+
+        self.prefixes = ['rose', 'viola', 'lily']
+        object_list_subdir = []
+        for p in self.prefixes:
+            object_list_subdir.append({"subdir": p})
+
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments',
+                            swob.HTTPNoContent, {}, json.dumps([]))
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/rose',
+                            swob.HTTPNoContent, {}, json.dumps([]))
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/viola',
+                            swob.HTTPNoContent, {}, json.dumps([]))
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/lily',
+                            swob.HTTPNoContent, {}, json.dumps([]))
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/with'
+                            ' space', swob.HTTPNoContent, {}, json.dumps([]))
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/with%20'
+                            'space', swob.HTTPNoContent, {}, json.dumps([]))
+        self.swift.register('GET', '/v1/AUTH_test/bucket+segments?format=json'
+                            '&marker=with%2520space', swob.HTTPOk, {},
+                            json.dumps([]))
+        self.swift.register('GET', '/v1/AUTH_test/bucket+segments?format=json'
+                            '&marker=', swob.HTTPOk, {}, object_list)
+        self.swift.register('HEAD', '/v1/AUTH_test/junk', swob.HTTPNoContent,
+                            {}, None)
+        self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
+                            {}, None)
+        self.swift.register('GET', '/v1/AUTH_test/junk', swob.HTTPOk, {},
+                            object_list)
+        self.swift.register(
+            'GET',
+            '/v1/AUTH_test/junk?delimiter=a&format=json&limit=3&marker=viola',
+            swob.HTTPOk, {}, json.dumps(objects[2:]))
+        self.swift.register('GET', '/v1/AUTH_test/junk-subdir', swob.HTTPOk,
+                            {}, json.dumps(object_list_subdir))
+        self.swift.register(
+            'GET',
+            '/v1/AUTH_test/subdirs?delimiter=/&format=json&limit=3',
+            swob.HTTPOk, {}, json.dumps([
+                {'subdir': 'nothing/'},
+                {'subdir': 'but/'},
+                {'subdir': 'subdirs/'},
+            ]))
+
+    def setUp(self):
+        super(TestS3ApiBucket, self).setUp()
+        self.setup_objects()
+
+    def test_bucket_HEAD(self):
+        req = Request.blank('/junk',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+    def test_bucket_HEAD_error(self):
+        req = Request.blank('/nojunk',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '404')
+        self.assertEqual(body, '')  # sanity
+
+    def test_bucket_HEAD_slash(self):
+        req = Request.blank('/junk/',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+    def test_bucket_HEAD_slash_error(self):
+        req = Request.blank('/nojunk/',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '404')
+
+    @s3acl
+    def test_bucket_GET_error(self):
+        code = self._test_method_error('GET', '/bucket', swob.HTTPUnauthorized)
+        self.assertEqual(code, 'SignatureDoesNotMatch')
+        code = self._test_method_error('GET', '/bucket', swob.HTTPForbidden)
+        self.assertEqual(code, 'AccessDenied')
+        code = self._test_method_error('GET', '/bucket', swob.HTTPNotFound)
+        self.assertEqual(code, 'NoSuchBucket')
+        code = self._test_method_error('GET', '/bucket', swob.HTTPServerError)
+        self.assertEqual(code, 'InternalError')
+
+    def test_bucket_GET(self):
+        bucket_name = 'junk'
+        req = Request.blank('/%s' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+        elem = fromstring(body, 'ListBucketResult')
+        name = elem.find('./Name').text
+        self.assertEqual(name, bucket_name)
+
+        objects = elem.iterchildren('Contents')
+
+        names = []
+        for o in objects:
+            names.append(o.find('./Key').text)
+            self.assertEqual('2011-01-05T02:19:14.275Z',
+                             o.find('./LastModified').text)
+            self.assertEqual('"0"', o.find('./ETag').text)
+
+        self.assertEqual(len(names), len(self.objects))
+        for i in self.objects:
+            self.assertTrue(i[0] in names)
+
+    def test_bucket_GET_subdir(self):
+        bucket_name = 'junk-subdir'
+        req = Request.blank('/%s' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+        elem = fromstring(body, 'ListBucketResult')
+        name = elem.find('./Name').text
+        self.assertEqual(name, bucket_name)
+
+        prefixes = elem.findall('CommonPrefixes')
+
+        self.assertEqual(len(prefixes), len(self.prefixes))
+        for p in prefixes:
+            self.assertTrue(p.find('./Prefix').text in self.prefixes)
+
+    def test_bucket_GET_is_truncated(self):
+        bucket_name = 'junk'
+
+        req = Request.blank('/%s?max-keys=5' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./IsTruncated').text, 'false')
+
+        req = Request.blank('/%s?max-keys=4' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./IsTruncated').text, 'true')
+
+        req = Request.blank('/subdirs?delimiter=/&max-keys=2',
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./IsTruncated').text, 'true')
+        self.assertEqual(elem.find('./NextMarker').text, 'but/')
+
+    def test_bucket_GET_v2_is_truncated(self):
+        bucket_name = 'junk'
+
+        req = Request.blank('/%s?list-type=2&max-keys=5' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./KeyCount').text, '5')
+        self.assertEqual(elem.find('./IsTruncated').text, 'false')
+
+        req = Request.blank('/%s?list-type=2&max-keys=4' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertIsNotNone(elem.find('./NextContinuationToken'))
+        self.assertEqual(elem.find('./KeyCount').text, '4')
+        self.assertEqual(elem.find('./IsTruncated').text, 'true')
+
+        req = Request.blank('/subdirs?list-type=2&delimiter=/&max-keys=2',
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertIsNotNone(elem.find('./NextContinuationToken'))
+        self.assertEqual(elem.find('./KeyCount').text, '2')
+        self.assertEqual(elem.find('./IsTruncated').text, 'true')
+
+    def test_bucket_GET_max_keys(self):
+        bucket_name = 'junk'
+
+        req = Request.blank('/%s?max-keys=5' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./MaxKeys').text, '5')
+        _, path = self.swift.calls[-1]
+        _, query_string = path.split('?')
+        args = dict(cgi.parse_qsl(query_string))
+        self.assertEqual(args['limit'], '6')
+
+        req = Request.blank('/%s?max-keys=5000' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./MaxKeys').text, '5000')
+        _, path = self.swift.calls[-1]
+        _, query_string = path.split('?')
+        args = dict(cgi.parse_qsl(query_string))
+        self.assertEqual(args['limit'], '1001')
+
+    def test_bucket_GET_str_max_keys(self):
+        bucket_name = 'junk'
+
+        req = Request.blank('/%s?max-keys=invalid' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+
+    def test_bucket_GET_negative_max_keys(self):
+        bucket_name = 'junk'
+
+        req = Request.blank('/%s?max-keys=-1' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+
+    def test_bucket_GET_over_32bit_int_max_keys(self):
+        bucket_name = 'junk'
+
+        req = Request.blank('/%s?max-keys=%s' %
+                            (bucket_name, MAX_32BIT_INT + 1),
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(self._get_error_code(body), 'InvalidArgument')
+
+    def test_bucket_GET_passthroughs(self):
+        bucket_name = 'junk'
+        req = Request.blank('/%s?delimiter=a&marker=b&prefix=c' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./Prefix').text, 'c')
+        self.assertEqual(elem.find('./Marker').text, 'b')
+        self.assertEqual(elem.find('./Delimiter').text, 'a')
+        _, path = self.swift.calls[-1]
+        _, query_string = path.split('?')
+        args = dict(cgi.parse_qsl(query_string))
+        self.assertEqual(args['delimiter'], 'a')
+        self.assertEqual(args['marker'], 'b')
+        self.assertEqual(args['prefix'], 'c')
+
+    def test_bucket_GET_v2_passthroughs(self):
+        bucket_name = 'junk'
+        req = Request.blank(
+            '/%s?list-type=2&delimiter=a&start-after=b&prefix=c' % bucket_name,
+            environ={'REQUEST_METHOD': 'GET'},
+            headers={'Authorization': 'AWS test:tester:hmac',
+                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./Prefix').text, 'c')
+        self.assertEqual(elem.find('./StartAfter').text, 'b')
+        self.assertEqual(elem.find('./Delimiter').text, 'a')
+        _, path = self.swift.calls[-1]
+        _, query_string = path.split('?')
+        args = dict(cgi.parse_qsl(query_string))
+        self.assertEqual(args['delimiter'], 'a')
+        # "start-after" is converted to "marker"
+        self.assertEqual(args['marker'], 'b')
+        self.assertEqual(args['prefix'], 'c')
+
+    def test_bucket_GET_with_nonascii_queries(self):
+        bucket_name = 'junk'
+        req = Request.blank(
+            '/%s?delimiter=\xef\xbc\xa1&marker=\xef\xbc\xa2&'
+            'prefix=\xef\xbc\xa3' % bucket_name,
+            environ={'REQUEST_METHOD': 'GET'},
+            headers={'Authorization': 'AWS test:tester:hmac',
+                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./Prefix').text, '\xef\xbc\xa3')
+        self.assertEqual(elem.find('./Marker').text, '\xef\xbc\xa2')
+        self.assertEqual(elem.find('./Delimiter').text, '\xef\xbc\xa1')
+        _, path = self.swift.calls[-1]
+        _, query_string = path.split('?')
+        args = dict(cgi.parse_qsl(query_string))
+        self.assertEqual(args['delimiter'], '\xef\xbc\xa1')
+        self.assertEqual(args['marker'], '\xef\xbc\xa2')
+        self.assertEqual(args['prefix'], '\xef\xbc\xa3')
+
+    def test_bucket_GET_v2_with_nonascii_queries(self):
+        bucket_name = 'junk'
+        req = Request.blank(
+            '/%s?list-type=2&delimiter=\xef\xbc\xa1&start-after=\xef\xbc\xa2&'
+            'prefix=\xef\xbc\xa3' % bucket_name,
+            environ={'REQUEST_METHOD': 'GET'},
+            headers={'Authorization': 'AWS test:tester:hmac',
+                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./Prefix').text, '\xef\xbc\xa3')
+        self.assertEqual(elem.find('./StartAfter').text, '\xef\xbc\xa2')
+        self.assertEqual(elem.find('./Delimiter').text, '\xef\xbc\xa1')
+        _, path = self.swift.calls[-1]
+        _, query_string = path.split('?')
+        args = dict(cgi.parse_qsl(query_string))
+        self.assertEqual(args['delimiter'], '\xef\xbc\xa1')
+        self.assertEqual(args['marker'], '\xef\xbc\xa2')
+        self.assertEqual(args['prefix'], '\xef\xbc\xa3')
+
+    def test_bucket_GET_with_delimiter_max_keys(self):
+        bucket_name = 'junk'
+        req = Request.blank('/%s?delimiter=a&max-keys=2' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./NextMarker').text, 'viola')
+        self.assertEqual(elem.find('./MaxKeys').text, '2')
+        self.assertEqual(elem.find('./IsTruncated').text, 'true')
+
+    def test_bucket_GET_v2_with_delimiter_max_keys(self):
+        bucket_name = 'junk'
+        req = Request.blank(
+            '/%s?list-type=2&delimiter=a&max-keys=2' % bucket_name,
+            environ={'REQUEST_METHOD': 'GET'},
+            headers={'Authorization': 'AWS test:tester:hmac',
+                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+        elem = fromstring(body, 'ListBucketResult')
+        next_token = elem.find('./NextContinuationToken')
+        self.assertIsNotNone(next_token)
+        self.assertEqual(elem.find('./MaxKeys').text, '2')
+        self.assertEqual(elem.find('./IsTruncated').text, 'true')
+
+        req = Request.blank(
+            '/%s?list-type=2&delimiter=a&max-keys=2&continuation-token=%s' %
+            (bucket_name, next_token.text),
+            environ={'REQUEST_METHOD': 'GET'},
+            headers={'Authorization': 'AWS test:tester:hmac',
+                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+        elem = fromstring(body, 'ListBucketResult')
+        names = [o.find('./Key').text for o in elem.iterchildren('Contents')]
+        self.assertEqual(names[0], 'lily')
+
+    def test_bucket_GET_subdir_with_delimiter_max_keys(self):
+        bucket_name = 'junk-subdir'
+        req = Request.blank('/%s?delimiter=a&max-keys=1' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+        elem = fromstring(body, 'ListBucketResult')
+        self.assertEqual(elem.find('./NextMarker').text, 'rose')
+        self.assertEqual(elem.find('./MaxKeys').text, '1')
+        self.assertEqual(elem.find('./IsTruncated').text, 'true')
+
+    def test_bucket_GET_v2_fetch_owner(self):
+        bucket_name = 'junk'
+        req = Request.blank('/%s?list-type=2' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+        elem = fromstring(body, 'ListBucketResult')
+        name = elem.find('./Name').text
+        self.assertEqual(name, bucket_name)
+
+        objects = elem.iterchildren('Contents')
+        for o in objects:
+            self.assertIsNone(o.find('./Owner'))
+
+        req = Request.blank('/%s?list-type=2&fetch-owner=true' % bucket_name,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+        elem = fromstring(body, 'ListBucketResult')
+        name = elem.find('./Name').text
+        self.assertEqual(name, bucket_name)
+
+        objects = elem.iterchildren('Contents')
+        for o in objects:
+            self.assertIsNotNone(o.find('./Owner'))
+
+    @s3acl
+    def test_bucket_PUT_error(self):
+        code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated,
+                                       headers={'Content-Length': 'a'})
+        self.assertEqual(code, 'InvalidArgument')
+        code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated,
+                                       headers={'Content-Length': '-1'})
+        self.assertEqual(code, 'InvalidArgument')
+        code = self._test_method_error('PUT', '/bucket', swob.HTTPUnauthorized)
+        self.assertEqual(code, 'SignatureDoesNotMatch')
+        code = self._test_method_error('PUT', '/bucket', swob.HTTPForbidden)
+        self.assertEqual(code, 'AccessDenied')
+        code = self._test_method_error('PUT', '/bucket', swob.HTTPAccepted)
+        self.assertEqual(code, 'BucketAlreadyExists')
+        code = self._test_method_error('PUT', '/bucket', swob.HTTPServerError)
+        self.assertEqual(code, 'InternalError')
+        code = self._test_method_error(
+            'PUT', '/bucket+bucket', swob.HTTPCreated)
+        self.assertEqual(code, 'InvalidBucketName')
+        code = self._test_method_error(
+            'PUT', '/192.168.11.1', swob.HTTPCreated)
+        self.assertEqual(code, 'InvalidBucketName')
+        code = self._test_method_error(
+            'PUT', '/bucket.-bucket', swob.HTTPCreated)
+        self.assertEqual(code, 'InvalidBucketName')
+        code = self._test_method_error(
+            'PUT', '/bucket-.bucket', swob.HTTPCreated)
+        self.assertEqual(code, 'InvalidBucketName')
+        code = self._test_method_error('PUT', '/bucket*', swob.HTTPCreated)
+        self.assertEqual(code, 'InvalidBucketName')
+        code = self._test_method_error('PUT', '/b', swob.HTTPCreated)
+        self.assertEqual(code, 'InvalidBucketName')
+        code = self._test_method_error(
+            'PUT', '/%s' % ''.join(['b' for x in xrange(64)]),
+            swob.HTTPCreated)
+        self.assertEqual(code, 'InvalidBucketName')
+
+    @s3acl
+    def test_bucket_PUT(self):
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(body, '')
+        self.assertEqual(status.split()[0], '200')
+        self.assertEqual(headers['Location'], '/bucket')
+
+        # Apparently some clients will include a chunked transfer-encoding
+        # even with no body
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'Transfer-Encoding': 'chunked'})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(body, '')
+        self.assertEqual(status.split()[0], '200')
+        self.assertEqual(headers['Location'], '/bucket')
+
+        with UnreadableInput(self) as fake_input:
+            req = Request.blank(
+                '/bucket',
+                environ={'REQUEST_METHOD': 'PUT',
+                         'wsgi.input': fake_input},
+                headers={'Authorization': 'AWS test:tester:hmac',
+                         'Date': self.get_date_header()})
+            status, headers, body = self.call_s3api(req)
+        self.assertEqual(body, '')
+        self.assertEqual(status.split()[0], '200')
+        self.assertEqual(headers['Location'], '/bucket')
+
+    def _test_bucket_PUT_with_location(self, root_element):
+        elem = Element(root_element)
+        SubElement(elem, 'LocationConstraint').text = 'US'
+        xml = tostring(elem)
+
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()},
+                            body=xml)
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
+    @s3acl
+    def test_bucket_PUT_with_location(self):
+        self._test_bucket_PUT_with_location('CreateBucketConfiguration')
+
+    @s3acl
+    def test_bucket_PUT_with_ami_location(self):
+        # ec2-ami-tools apparently uses CreateBucketConstraint instead?
+        self._test_bucket_PUT_with_location('CreateBucketConstraint')
+
+    @s3acl
+    def test_bucket_PUT_with_strange_location(self):
+        # Even crazier: it doesn't seem to matter
+        self._test_bucket_PUT_with_location('foo')
+
+    def test_bucket_PUT_with_canned_acl(self):
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'X-Amz-Acl': 'public-read'})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+        _, _, headers = self.swift.calls_with_headers[-1]
+        self.assertTrue('X-Container-Read' in headers)
+        self.assertEqual(headers.get('X-Container-Read'), '.r:*,.rlistings')
+        self.assertNotIn('X-Container-Sysmeta-S3api-Acl', headers)
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_PUT_with_canned_s3acl(self):
+        account = 'test:tester'
+        acl = \
+            encode_acl('container', ACLPublicRead(Owner(account, account)))
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'X-Amz-Acl': 'public-read'})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+        _, _, headers = self.swift.calls_with_headers[-1]
+        self.assertNotIn('X-Container-Read', headers)
+        self.assertIn('X-Container-Sysmeta-S3api-Acl', headers)
+        self.assertEqual(headers.get('X-Container-Sysmeta-S3api-Acl'),
+                         acl['x-container-sysmeta-s3api-acl'])
+
+    @s3acl
+    def test_bucket_PUT_with_location_error(self):
+        elem = Element('CreateBucketConfiguration')
+        SubElement(elem, 'LocationConstraint').text = 'XXX'
+        xml = tostring(elem)
+
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()},
+                            body=xml)
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(self._get_error_code(body),
+                         'InvalidLocationConstraint')
+
+    @s3acl
+    def test_bucket_PUT_with_location_invalid_xml(self):
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()},
+                            body='invalid_xml')
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(self._get_error_code(body), 'MalformedXML')
+
+    def _test_method_error_delete(self, path, sw_resp):
+        self.swift.register('HEAD', '/v1/AUTH_test' + path, sw_resp, {}, None)
+        return self._test_method_error('DELETE', path, sw_resp)
+
+    @s3acl
+    def test_bucket_DELETE_error(self):
+        code = self._test_method_error_delete('/bucket', swob.HTTPUnauthorized)
+        self.assertEqual(code, 'SignatureDoesNotMatch')
+        code = self._test_method_error_delete('/bucket', swob.HTTPForbidden)
+        self.assertEqual(code, 'AccessDenied')
+        code = self._test_method_error_delete('/bucket', swob.HTTPNotFound)
+        self.assertEqual(code, 'NoSuchBucket')
+        code = self._test_method_error_delete('/bucket', swob.HTTPServerError)
+        self.assertEqual(code, 'InternalError')
+
+        # bucket not empty is now validated at s3api
+        self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent,
+                            {'X-Container-Object-Count': '1'}, None)
+        code = self._test_method_error('DELETE', '/bucket', swob.HTTPConflict)
+        self.assertEqual(code, 'BucketNotEmpty')
+
+    @s3acl
+    def test_bucket_DELETE(self):
+        # overwrite default HEAD to return x-container-object-count
+        self.swift.register(
+            'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent,
+            {'X-Container-Object-Count': 0}, None)
+
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'DELETE'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '204')
+
+    @s3acl
+    def test_bucket_DELETE_error_while_segment_bucket_delete(self):
+        # An error occurred while deleting segment objects
+        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/lily',
+                            swob.HTTPServiceUnavailable, {}, json.dumps([]))
+        # overwrite default HEAD to return x-container-object-count
+        self.swift.register(
+            'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent,
+            {'X-Container-Object-Count': 0}, None)
+
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': 'DELETE'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '503')
+        called = [(method, path) for method, path, _ in
+                  self.swift.calls_with_headers]
+        # Don't delete original bucket when error occurred in segment container
+        self.assertNotIn(('DELETE', '/v1/AUTH_test/bucket'), called)
+
+    def _test_bucket_for_s3acl(self, method, account):
+        req = Request.blank('/bucket',
+                            environ={'REQUEST_METHOD': method},
+                            headers={'Authorization': 'AWS %s:hmac' % account,
+                                     'Date': self.get_date_header()})
+
+        return self.call_s3api(req)
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_GET_without_permission(self):
+        status, headers, body = self._test_bucket_for_s3acl('GET',
+                                                            'test:other')
+        self.assertEqual(self._get_error_code(body), 'AccessDenied')
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_GET_with_read_permission(self):
+        status, headers, body = self._test_bucket_for_s3acl('GET',
+                                                            'test:read')
+        self.assertEqual(status.split()[0], '200')
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_GET_with_fullcontrol_permission(self):
+        status, headers, body = \
+            self._test_bucket_for_s3acl('GET', 'test:full_control')
+        self.assertEqual(status.split()[0], '200')
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_GET_with_owner_permission(self):
+        status, headers, body = self._test_bucket_for_s3acl('GET',
+                                                            'test:tester')
+        self.assertEqual(status.split()[0], '200')
+
+    def _test_bucket_GET_canned_acl(self, bucket):
+        req = Request.blank('/%s' % bucket,
+                            environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
+
+        return self.call_s3api(req)
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_GET_authenticated_users(self):
+        status, headers, body = \
+            self._test_bucket_GET_canned_acl('authenticated')
+        self.assertEqual(status.split()[0], '200')
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_GET_all_users(self):
+        status, headers, body = self._test_bucket_GET_canned_acl('public')
+        self.assertEqual(status.split()[0], '200')
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_DELETE_without_permission(self):
+        status, headers, body = self._test_bucket_for_s3acl('DELETE',
+                                                            'test:other')
+        self.assertEqual(self._get_error_code(body), 'AccessDenied')
+        # Don't delete anything in backend Swift
+        called = [method for method, _, _ in self.swift.calls_with_headers]
+        self.assertNotIn('DELETE', called)
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_DELETE_with_write_permission(self):
+        status, headers, body = self._test_bucket_for_s3acl('DELETE',
+                                                            'test:write')
+        self.assertEqual(self._get_error_code(body), 'AccessDenied')
+        # Don't delete anything in backend Swift
+        called = [method for method, _, _ in self.swift.calls_with_headers]
+        self.assertNotIn('DELETE', called)
+
+    @s3acl(s3acl_only=True)
+    def test_bucket_DELETE_with_fullcontrol_permission(self):
+        status, headers, body = \
+            self._test_bucket_for_s3acl('DELETE', 'test:full_control')
+        self.assertEqual(self._get_error_code(body), 'AccessDenied')
+        # Don't delete anything in backend Swift
+        called = [method for method, _, _ in self.swift.calls_with_headers]
+        self.assertNotIn('DELETE', called)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/middleware/s3api/test_cfg.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from swift.common.middleware.s3api.utils import Config
+
+
+class TestS3ApiCfg(unittest.TestCase):
+    def test_config(self):
+        conf = Config(
+            {
+                'a': 'str',
+                'b': 10,
+                'c': True,
+            }
+        )
+
+        conf.update(
+            {
+                'a': 'str2',
+                'b': '100',
+                'c': 'false',
+            }
+        )
+
+        self.assertEqual(conf['a'], 'str2')
+        self.assertEqual(conf['b'], 100)
+        self.assertEqual(conf['c'], False)
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/middleware/s3api/test_etree.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from swift.common.middleware.s3api import etree
+
+
+class TestS3ApiEtree(unittest.TestCase):
+    def test_xml_namespace(self):
+        def test_xml(ns, prefix):
+            return '<A %(ns)s><%(prefix)sB>C</%(prefix)sB></A>' % \
+                ({'ns': ns, 'prefix': prefix})
+
+        # No namespace is same as having the S3 namespace.
+        xml = test_xml('', '')
+        elem = etree.fromstring(xml)
+        self.assertEqual(elem.find('./B').text, 'C')
+
+        # The S3 namespace is handled as no namespace.
+        xml = test_xml('xmlns="%s"' % etree.XMLNS_S3, '')
+        elem = etree.fromstring(xml)
+        self.assertEqual(elem.find('./B').text, 'C')
+
+        xml = test_xml('xmlns:s3="%s"' % etree.XMLNS_S3, 's3:')
+        elem = etree.fromstring(xml)
+        self.assertEqual(elem.find('./B').text, 'C')
+
+        # Any namespaces without a prefix work as no namespace.
+        xml = test_xml('xmlns="http://example.com/"', '')
+        elem = etree.fromstring(xml)
+        self.assertEqual(elem.find('./B').text, 'C')
+
+        xml = test_xml('xmlns:s3="http://example.com/"', 's3:')
+        elem = etree.fromstring(xml)
+        self.assertIsNone(elem.find('./B'))
+
+    def test_xml_with_comments(self):
+        xml = '<A><!-- comment --><B>C</B></A>'
+        elem = etree.fromstring(xml)
+        self.assertEqual(elem.find('./B').text, 'C')
+
+    def test_tostring_with_nonascii_text(self):
+        elem = etree.Element('Test')
+        sub = etree.SubElement(elem, 'FOO')
+        sub.text = '\xef\xbc\xa1'
+        self.assertTrue(isinstance(sub.text, str))
+        xml_string = etree.tostring(elem)
+        self.assertTrue(isinstance(xml_string, str))
+
+    def test_fromstring_with_nonascii_text(self):
+        input_str = '<?xml version="1.0" encoding="UTF-8"?>\n' \
+                    '<Test><FOO>\xef\xbc\xa1</FOO></Test>'
+        elem = etree.fromstring(input_str)
+        text = elem.find('FOO').text
+        self.assertEqual(text, '\xef\xbc\xa1')
+        self.assertTrue(isinstance(text, str))
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/middleware/s3api/test_helpers.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This stuff can't live in test/unit/__init__.py due to its swob dependency.
+
+import unittest
+from test.unit.common.middleware.s3api.helpers import FakeSwift
+from swift.common.middleware.s3api.utils import sysmeta_header
+from swift.common.swob import HeaderKeyDict
+from mock import MagicMock
+
+
+class S3ApiHelperTestCase(unittest.TestCase):
+    def setUp(self):
+        self.method = 'HEAD'
+        self.path = '/v1/AUTH_test/bucket'
+
+    def _check_headers(self, swift, method, path, headers):
+        _, response_headers, _ = swift._responses[(method, path)]
+        self.assertEqual(headers, response_headers)
+
+    def test_fake_swift_sysmeta(self):
+        swift = FakeSwift()
+        orig_headers = HeaderKeyDict()
+        orig_headers.update({sysmeta_header('container', 'acl'): 'test',
+                             'x-container-meta-foo': 'bar'})
+
+        swift.register(self.method, self.path, MagicMock(), orig_headers, None)
+
+        self._check_headers(swift, self.method, self.path, orig_headers)
+
+        new_headers = orig_headers.copy()
+        del new_headers[sysmeta_header('container', 'acl').title()]
+        swift.register(self.method, self.path, MagicMock(), new_headers, None)
+
+        self._check_headers(swift, self.method, self.path, orig_headers)
+
+    def test_fake_swift_sysmeta_overwrite(self):
+        swift = FakeSwift()
+        orig_headers = HeaderKeyDict()
+        orig_headers.update({sysmeta_header('container', 'acl'): 'test',
+                             'x-container-meta-foo': 'bar'})
+        swift.register(self.method, self.path, MagicMock(), orig_headers, None)
+
+        self._check_headers(swift, self.method, self.path, orig_headers)
+
+        new_headers = orig_headers.copy()
+        new_headers[sysmeta_header('container', 'acl').title()] = 'bar'
+
+        swift.register(self.method, self.path, MagicMock(), new_headers, None)
+
+        self.assertFalse(orig_headers == new_headers)
+        self._check_headers(swift, self.method, self.path, new_headers)
+
+
+if __name__ == '__main__':
+    unittest.main()