openstack
/
swift
Code Issues Proposed changes

Merge branch 'master' into feature/losf 

Change-Id: I4150fd0152a6a3fd3f31b86315f212316f27ec39
This commit is contained in:
Tim Burke 2019-08-21 15:13:12 -07:00
parent ea5cbf633c ff5ea003b3
commit 6204a2bc57
88 changed files with 3225 additions and 880 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.mailmap
View File

@ -127,3 +127,6 @@ zhangdebo1987 <zhangdebo@inspur.com> zhangdebo
Thomas Goirand <thomas@goirand.fr> <zigo@debian.org>
Thiago da Silva <thiagodasilva@gmail.com> <thiago@redhat.com>
Kota Tsuyuzaki <kota.tsuyuzaki.pc@hco.ntt.co.jp> <tsuyuzaki.kota@lab.ntt.co.jp>
Ehud Kaldor <ehud@unfairfunction.org> <ehud@UnfairFunction.org>
Takashi Kajinami <tkajinam@redhat.com> <kajinamit@nttdata.co.jp>
Yuxin Wang <wang.yuxin@ostorage.com.cn> Wang Yuxin

101
.zuul.yaml
View File

@ -96,6 +96,30 @@
bindep_profile: test py37
python_version: 3.7
- job:
name: swift-tox-func-encryption-py37
parent: swift-tox-func-py37
description: |
Run functional tests for swift under cPython version 3.7.
Uses tox with the ``func-encryption-py3`` environment.
It sets TMPDIR to an XFS mount point created via
tools/test-setup.sh.
vars:
tox_envlist: func-encryption-py3
- job:
name: swift-tox-func-ec-py37
parent: swift-tox-func-py37
description: |
Run functional tests for swift under cPython version 3.7.
Uses tox with the ``func-ec-py3`` environment.
It sets TMPDIR to an XFS mount point created via
tools/test-setup.sh.
vars:
tox_envlist: func-ec-py3
- job:
name: swift-tox-func-domain-remap-staticweb-py37
parent: swift-tox-func-py37
@ -108,6 +132,18 @@
vars:
tox_envlist: func-domain-remap-staticweb-py3
- job:
name: swift-tox-func-s3api-py37
parent: swift-tox-func-py37
description: |
Run functional tests for swift under cPython version 3.7.
Uses tox with the ``func-s3api`` environment.
It sets TMPDIR to an XFS mount point created via
tools/test-setup.sh.
vars:
tox_envlist: func-s3api-py3
- job:
name: swift-tox-func-centos-7
parent: swift-tox-func
@ -244,8 +280,7 @@
vars:
devstack_localrc:
SERVICE_IP_VERSION: 6
HOST_IPV6: '[::1]'
SERVICE_HOST: '[::1]'
SERVICE_HOST: ""
- job:
name: swift-tox-func-s3api-ceph-s3tests-tempauth
@ -408,12 +443,15 @@
- periodic-stable-jobs
- check-requirements
- release-notes-jobs-python3
- integrated-gate-object-storage
check:
jobs:
- swift-build-image:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/(functional|probe)/.*$
# Unit tests
- swift-tox-py27:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
@ -422,21 +460,17 @@
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/(functional|probe)/.*$
voting: false
- swift-tox-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/(functional|probe)/.*$
# Functional tests
- swift-tox-func:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- swift-tox-func-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- swift-tox-func-encryption:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
@ -462,11 +496,35 @@
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
# py3 functional tests
- swift-tox-func-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- swift-tox-func-encryption-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- swift-tox-func-domain-remap-staticweb-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- swift-tox-func-ec-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- swift-tox-func-s3api-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
# Other tests
- swift-tox-func-s3api-ceph-s3tests-tempauth:
irrelevant-files:
- ^(api-ref|releasenotes)/.*$
@ -511,7 +569,16 @@
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
voting: false
- tempest-integrated-object-storage:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- grenade-py3:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
gate:
jobs:
# For gate jobs, err towards running more jobs (so, generally avoid
@ -520,14 +587,18 @@
# in-tree definitions) dsvm jobs.
- swift-upload-image
- swift-tox-py27
- swift-tox-py36
- swift-tox-py37
- swift-tox-func
- swift-tox-func-py37
- swift-tox-func-encryption
- swift-tox-func-domain-remap-staticweb
- swift-tox-func-ec
- swift-tox-func-s3api
- swift-tox-func-py37
- swift-tox-func-encryption
- swift-tox-func-domain-remap-staticweb-py37
- swift-tox-func-ec-py37
- swift-tox-func-s3api-py37
- swift-probetests-centos-7:
irrelevant-files:
- ^(api-ref|releasenotes)/.*$
@ -559,6 +630,16 @@
- ^(api-ref|etc|examples|releasenotes)/.*$
# Keep doc/manpages -- we want to syntax check them
- ^doc/(requirements.txt|(saio|s3api|source)/.*)$
- tempest-integrated-object-storage:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- grenade-py3:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
experimental:
jobs:
- swift-tox-py27-centos-7

13
AUTHORS
View File

@ -143,7 +143,7 @@ Drew Balfour (andrew.balfour@oracle.com)
Eamonn O'Toole (eamonn.otoole@hpe.com)
Ed Leafe (ed.leafe@rackspace.com)
Edward Hope-Morley (opentastic@gmail.com)
Ehud Kaldor (ehud@UnfairFunction.org)
Ehud Kaldor (ehud@unfairfunction.org)
Ellen Leahy (ellen.mar.leahy@hpe.com)
Emett Speer (speer.emett@gmail.com)
Emile Snyder (emile.snyder@gmail.com)
@ -174,6 +174,7 @@ gengchc2 (geng.changcai2@zte.com.cn)
Gerard Gine (ggine@swiftstack.com)
Gerry Drudy (gerry.drudy@hpe.com)
Gil Vernik (gilv@il.ibm.com)
Gilles Biannic (gilles.biannic@corp.ovh.com)
Gleb Samsonov (sams-gleb@yandex.ru)
Gonéri Le Bouder (goneri.lebouder@enovance.com)
Graham Hayes (graham.hayes@hpe.com)
@ -216,6 +217,7 @@ Jing Liuqing (jing.liuqing@99cloud.net)
Joanna H. Huang (joanna.huitzu.huang@gmail.com)
Joe Arnold (joe@swiftstack.com)
Joe Gordon (jogo@cloudscaling.com)
Joe Yang (jyang@swiftstack.com)
Joel Wright (joel.wright@sohonet.com)
John Leach (john@johnleach.co.uk)
Jola Mirecka (jola.mirecka@hp.com)
@ -238,6 +240,7 @@ Kiyoung Jung (kiyoung.jung@kt.com)
Koert van der Veer (koert@cloudvps.com)
Kota Tsuyuzaki (kota.tsuyuzaki.pc@hco.ntt.co.jp)
Ksenia Demina (kdemina@mirantis.com)
Kuan-Lin Chen (kuanlinchen@synology.com)
Kun Huang (gareth@unitedstack.com)
Larry Rensing (lr699s@att.com)
Leah Klearman (lklrmn@gmail.com)
@ -259,6 +262,7 @@ M V P Nitesh (m.nitesh@nectechnologies.in)
Madhuri Kumari (madhuri.rai07@gmail.com)
Mahati Chamarthy (mahati.chamarthy@gmail.com)
malei (malei@maleideMacBook-Pro.local)
Mandell Degerness (mdegerness@swiftstack.com)
maoshuai (fwsakura@163.com)
Marcelo Martins (btorch@gmail.com)
Maria Malyarova (savoreux69@gmail.com)
@ -348,6 +352,7 @@ Sarvesh Ranjan (saranjan@cisco.com)
Sascha Peilicke (saschpe@gmx.de)
Saverio Proto (saverio.proto@switch.ch)
Scott Simpson (sasimpson@gmail.com)
Sean McGinnis (sean.mcginnis@gmail.com)
Sergey Kraynev (skraynev@mirantis.com)
Sergey Lukjanov (slukjanov@mirantis.com)
Shane Wang (shane.wang@intel.com)
@ -357,6 +362,7 @@ Shashank Kumar Shankar (shashank.kumar.shankar@intel.com)
Shashirekha Gundur (shashirekha.j.gundur@intel.com)
Shilla Saebi (shilla.saebi@gmail.com)
Shri Javadekar (shrinand@maginatics.com)
Simeon Gourlin (simeon.gourlin@infomaniak.com)
Sivasathurappan Radhakrishnan (siva.radhakrishnan@intel.com)
Soren Hansen (soren@linux2go.dk)
Stefan Majewsky (stefan.majewsky@sap.com)
@ -365,7 +371,7 @@ Steve Kowalik (steven@wedontsleep.org)
Steve Martinelli (stevemar@ca.ibm.com)
Steven Lang (Steven.Lang@hgst.com)
Sushil Kumar (sushil.kumar2@globallogic.com)
Takashi Kajinami (kajinamit@nttdata.co.jp)
Takashi Kajinami (tkajinam@redhat.com)
Takashi Natsume (natsume.takashi@lab.ntt.co.jp)
TheSriram (sriram@klusterkloud.com)
Thiago da Silva (thiagodasilva@gmail.com)
@ -420,6 +426,7 @@ Yushiro FURUKAWA (y.furukawa_2@jp.fujitsu.com)
Yuxin Wang (wang.yuxin@ostorage.com.cn)
Zack M. Davis (zdavis@swiftstack.com)
Zap Chang (zapchang@gmail.com)
zengjia (zengjia@awcloud.com)
Zhang Guoqing (zhang.guoqing@99cloud.net)
Zhang Jinnan (ben.os@99cloud.net)
zhang.lei (zhang.lei@99cloud.net)
@ -432,6 +439,8 @@ Zhenguo Niu (zhenguo@unitedstack.com)
zhengwei6082 (zhengwei6082@fiberhome.com)
ZhijunWei (wzj334965317@outlook.com)
ZhiQiang Fan (aji.zqfan@gmail.com)
ZhongShengping (chdzsp@163.com)
Zhongyue Luo (zhongyue.nah@intel.com)
zhufl (zhu.fanglei@zte.com.cn)
zhulingjie (easyzlj@gmail.com)
翟小君 (zhaixiaojun@gohighsec.com)

83
CHANGELOG
View File

@ -1,3 +1,78 @@
swift (2.22.0)
* Experimental support for Python 3.6 and 3.7 is now available.
Note that this requires eventlet>=0.25.0. All unit tests pass,
and running functional tests under Python 2 will pass against
services running under Python 3. Expect full support in the
next minor release.
* Log formats are now more configurable and include support for
anonymization. See the log_msg_template option in proxy-server.conf
and https://docs.openstack.org/swift/latest/logs.html#proxy-logs
for more information.
* Added an operator tool, swift-container-deleter, to asynchronously
delete some or all objects in a container using the object expirers.
* Swift-all-in-one Docker images are now built and published to
https://hub.docker.com/r/openstackswift/saio. These are intended
for use as development targets, but will hopefully be useful as a
starting point for other work involving containerizing Swift.
* The object-expirer may now be configured in object-server.conf.
This is in anticipation of a future change to allow the
object-expirer to be deployed on all nodes that run object-servers.
* Correctness improvements
* The proxy-server now ignores 404 responses from handoffs without
databases when deciding on the correct response for account and
container requests.
* Object writes to a container whose existence cannot be verified
now 503 instead of 404.
* Sharding improvements
* The container-replicator now only attempts to fetch shard ranges if
the remote indicates that it has shard ranges. Further, it does so
with a timeout to prevent the process from hanging in certain cases.
* The proxy-server now caches 'updating' shards, improving write
performance for sharded containers. A new config option,
`recheck_updating_shard_ranges`, controls the cache time; set it to
0 to disable caching.
* The container-replicator now correctly enqueues container-reconciler
work for sharded containers.
* S3 API improvements
* Unsigned payloads work with v4 signatures once more.
* Multipart upload parts may now be copied from other multipart uploads.
* CompleteMultipartUpload requests with a Content-MD5 now work.
* Content-Type can now be updated when copying an object.
* Fixed v1 listings that end with a non-ASCII object name.
* Background corruption-detection improvements
* Detect and remove invalid entries from hashes.pkl
* When object path is not a directory, just quarantine it,
rather than the whole suffix.
* Dependency updates: we've increased our minimum supported version
of cryptography to 2.0.2 and netifaces to 0.8. This is largely due
to the difficulty of continuing to test with the old versions.
If running Swift under Python 3, eventlet must be at least 0.25.0.
* Various other minor bug fixes and improvements.
swift (2.21.0, OpenStack Stein release)
* Change the behavior of the EC reconstructor to perform a
@ -162,7 +237,7 @@ swift (2.20.0)
* Various other minor bug fixes and improvements.
2.19.1 (rocky stable backports)
swift (2.19.1, rocky stable backports)
* Prevent PyKMIP's kmip_protocol logger from logging at DEBUG.
Previously, some versions of PyKMIP would include all wire
@ -339,7 +414,7 @@ swift (2.18.0)
* Various other minor bug fixes and improvements.
2.17.1 (queens stable backports)
swift (2.17.1, queens stable backports)
* Fix SLO delete for accounts with non-ASCII names.
@ -541,7 +616,7 @@ swift (2.16.0)
* Various other minor bug fixes and improvements.
2.15.2 (pike stable backports)
swift (2.15.2, pike stable backports)
* Fixed a cache invalidation issue related to GET and PUT requests to
containers that would occasionally cause object PUTs to a container to
@ -979,7 +1054,7 @@ swift (2.10.0, OpenStack Newton)
improved in clusters that are not completely healthy.
* Significant improvements to the api-ref doc available at
https://developer.openstack.org/api-ref/object-storage/.
https://docs.openstack.org/api-ref/object-store/.
* A PUT or POST to a container will now update the container's
Last-Modified time, and that value will be included in a

9
Dockerfile
View File

@ -1,10 +1,10 @@
################################################
#
# Alpine 3.7 Swift-All-In-One
# Alpine 3.10.1 Swift-All-In-One
#
################################################
FROM alpine:3.7
FROM alpine:3.10.1
MAINTAINER Openstack Swift
ENV S6_LOGGING 1
@ -35,6 +35,11 @@ RUN mkdir /etc/swift && \
echo && \
echo && \
echo && \
echo "================ starting apk_install_py2 ===================" && \
/opt/swift/docker/install_scripts/20_apk_install_py2.sh && \
echo && \
echo && \
echo && \
echo "================ starting swift_install ===================" && \
/opt/swift/docker/install_scripts/50_swift_install.sh && \
echo && \

76
Dockerfile-py3 Normal file
View File

@ -0,0 +1,76 @@
################################################
#
# Alpine 3.10.1 Swift-All-In-One
#
################################################
FROM alpine:3.10.1
MAINTAINER Openstack Swift
ENV S6_LOGGING 1
ENV S6_VERSION 1.21.4.0
ENV SOCKLOG_VERSION 3.0.1-1
ENV ARCH amd64
ENV BUILD_DIR "/tmp"
ENV ENV="/etc/profile"
#COPY docker/install_scripts /install_scripts
COPY . /opt/swift
ADD https://github.com/just-containers/s6-overlay/releases/download/v$S6_VERSION/s6-overlay-$ARCH.tar.gz /tmp/
ADD https://github.com/just-containers/s6-overlay/releases/download/v$S6_VERSION/s6-overlay-$ARCH.tar.gz.sig /tmp/
ADD https://github.com/just-containers/socklog-overlay/releases/download/v$SOCKLOG_VERSION/socklog-overlay-$ARCH.tar.gz /tmp/
RUN mkdir /etc/swift && \
echo && \
echo && \
echo && \
echo "================ starting swift_needs ===================" && \
/opt/swift/docker/install_scripts/00_swift_needs.sh && \
echo && \
echo && \
echo && \
echo "================ starting apk_install_prereqs ===================" && \
/opt/swift/docker/install_scripts/10_apk_install_prereqs.sh && \
echo && \
echo && \
echo && \
echo "================ starting apk_install_py3 ===================" && \
/opt/swift/docker/install_scripts/21_apk_install_py3.sh && \
echo && \
echo && \
echo && \
echo "================ starting swift_install ===================" && \
/opt/swift/docker/install_scripts/50_swift_install.sh && \
echo && \
echo && \
echo && \
echo "================ installing s6-overlay ===================" && \
curl https://keybase.io/justcontainers/key.asc | gpg --import && \
gpg --verify /tmp/s6-overlay-$ARCH.tar.gz.sig /tmp/s6-overlay-$ARCH.tar.gz && \
gunzip -c /tmp/s6-overlay-$ARCH.tar.gz | tar -xf - -C / && \
gunzip -c /tmp/socklog-overlay-amd64.tar.gz | tar -xf - -C / && \
rm -rf /tmp/s6-overlay* && \
rm -rf /tmp/socklog-overlay* && \
echo && \
echo && \
echo && \
echo "================ starting pip_uninstall_dev ===================" && \
/opt/swift/docker/install_scripts/60_pip_uninstall_dev.sh && \
echo && \
echo && \
echo && \
echo "================ starting apk_uninstall_dev ===================" && \
/opt/swift/docker/install_scripts/99_apk_uninstall_dev.sh && \
echo && \
echo && \
echo && \
echo "================ clean up ===================" && \
echo "TODO: cleanup"
#rm -rf /opt/swift
# Add Swift required configuration files
COPY docker/rootfs /
ENTRYPOINT ["/init"]

2
README.rst
View File

@ -147,7 +147,7 @@ For client applications, official Python language bindings are provided
at https://github.com/openstack/python-swiftclient.
Complete API documentation at
https://developer.openstack.org/api-ref/object-store/
https://docs.openstack.org/api-ref/object-store/
There is a large ecosystem of applications and libraries that support and
work with OpenStack Swift. Several are listed on the

28
bin/swift-orphans
View File

@ -15,6 +15,7 @@
from __future__ import print_function
import optparse
import os
import re
import signal
import subprocess
import sys
@ -56,27 +57,30 @@ Example (sends SIGTERM to all orphaned Swift processes older than two hours):
pids.append(open(os.path.join(root, name)).read().strip())
pids.extend(subprocess.Popen(
['ps', '--ppid', pids[-1], '-o', 'pid', '--no-headers'],
stdout=subprocess.PIPE).communicate()[0].split())
stdout=subprocess.PIPE).communicate()[0].decode().split())
listing = []
swift_cmd_re = re.compile(
'^/usr/bin/python[23]? /usr(?:/local)?/bin/swift-')
for line in subprocess.Popen(
['ps', '-eo', 'etime,pid,args', '--no-headers'],
stdout=subprocess.PIPE).communicate()[0].split('\n'):
stdout=subprocess.PIPE).communicate()[0].split(b'\n'):
if not line:
continue
hours = 0
try:
etime, pid, args = line.split(None, 2)
etime, pid, args = line.decode('ascii').split(None, 2)
except ValueError:
sys.exit('Could not process ps line %r' % line)
if pid in pids:
continue
if (not args.startswith('/usr/bin/python /usr/bin/swift-') and
not args.startswith('/usr/bin/python /usr/local/bin/swift-')) or \
'swift-orphans' in args or \
'once' in args.split():
if any([
not swift_cmd_re.match(args),
'swift-orphans' in args,
'once' in args.split(),
]):
continue
args = args.split('-', 1)[1]
args = args.split('swift-', 1)[1]
etime = etime.split('-')
if len(etime) == 2:
hours = int(etime[0]) * 24
@ -105,11 +109,11 @@ Example (sends SIGTERM to all orphaned Swift processes older than two hours):
args_len = max(args_len, len(args))
args_len = min(args_len, 78 - hours_len - pid_len)
print(('%%%ds %%%ds %%s' % (hours_len, pid_len)) %
('Hours', 'PID', 'Command'))
print('%*s %*s %s' %
(hours_len, 'Hours', pid_len, 'PID', 'Command'))
for hours, pid, args in listing:
print(('%%%ds %%%ds %%s' % (hours_len, pid_len)) %
(hours, pid, args[:args_len]))
print('%*s %*s %s' %
(hours_len, hours, pid_len, pid, args[:args_len]))
if options.signal:
try:

2
doc/source/admin/objectstorage-monitoring.rst
View File

@ -113,7 +113,7 @@ Swift StatsD logging
~~~~~~~~~~~~~~~~~~~~
StatsD (see `Measure Anything, Measure Everything
<http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/>`_)
<https://codeascraft.com/2011/02/15/measure-anything-measure-everything/>`_)
was designed for application code to be deeply instrumented. Meters are
sent in real-time by the code that just noticed or did something. The
overhead of sending a meter is extremely low: a ``sendto`` of one UDP

2
doc/source/admin_guide.rst
View File

@ -833,7 +833,7 @@ this is unnecessary since the port is specified separately). If a hostname
resolves to an IPv4 address, an IPv4 socket will be used to send StatsD UDP
packets, even if the hostname would also resolve to an IPv6 address.
.. _StatsD: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/
.. _StatsD: https://codeascraft.com/2011/02/15/measure-anything-measure-everything/
.. _Graphite: http://graphiteapp.org/
.. _Ganglia: http://ganglia.sourceforge.net/

6
doc/source/api/object_api_v1_overview.rst
View File

@ -171,14 +171,14 @@ The API Reference describes the operations that you can perform with the
Object Storage API:
- `Storage
accounts <https://developer.openstack.org/api-ref/object-store/index.html#accounts>`__:
accounts <https://docs.openstack.org/api-ref/object-store/index.html#accounts>`__:
Use to perform account-level tasks.
Lists containers for a specified account. Creates, updates, and
deletes account metadata. Shows account metadata.
- `Storage
containers <https://developer.openstack.org/api-ref/object-store/index.html#containers>`__:
containers <https://docs.openstack.org/api-ref/object-store/index.html#containers>`__:
Use to perform container-level tasks.
Lists objects in a specified container. Creates, shows details for,
@ -186,7 +186,7 @@ Object Storage API:
container metadata.
- `Storage
objects <https://developer.openstack.org/api-ref/object-store/index.html#objects>`__:
objects <https://docs.openstack.org/api-ref/object-store/index.html#objects>`__:
Use to perform object-level tasks.
Creates, replaces, shows details for, and deletes objects. Copies

2
doc/source/first_contribution_swift.rst
View File

@ -21,7 +21,7 @@ A source tarball for the latest release of Swift is available on the
Prebuilt packages for Ubuntu and RHEL variants are available.
* `Swift Ubuntu Packages <https://launchpad.net/ubuntu/+source/swift>`_
* `Swift RDO Packages <https://www.rdoproject.org/Repositories>`_
* `Swift RDO Packages <https://www.rdoproject.org/documentation/repositories/>`_
--------------------
Source Control Setup

2
doc/source/index.rst
View File

@ -99,7 +99,7 @@ Administrator Documentation
Object Storage v1 REST API Documentation
========================================
See `Complete Reference for the Object Storage REST API <http://developer.openstack.org/api-ref/object-storage/>`_
See `Complete Reference for the Object Storage REST API <https://docs.openstack.org/api-ref/object-store/>`_
The following provides supporting information for the REST API:

2
doc/source/overview_encryption.rst
View File

@ -318,7 +318,7 @@ in Barbican::
--mode ctr --secret-type symmetric --payload <base64_encoded_root_secret>
Alternatively, the existing root secret can also be stored in Barbican using
`curl <http://developer.openstack.org/api-guide/key-manager/secrets.html>`__.
`curl <https://docs.openstack.org/api-guide/key-manager/secrets.html>`__.
.. note::

10
doc/source/overview_erasure_code.rst
View File

@ -646,16 +646,16 @@ multi-phase conversation, that the other nodes have landed enough for a quorum.
The basic flow looks like this:
* The Proxy Server erasure codes and streams the object fragments
#. The Proxy Server erasure codes and streams the object fragments
(ec_ndata + ec_nparity) to the storage nodes.
* The storage nodes store objects as EC archives and upon finishing object
#. The storage nodes store objects as EC archives and upon finishing object
data/metadata write, send a 1st-phase response to proxy.
* Upon quorum of storage nodes responses, the proxy initiates 2nd-phase by
#. Upon quorum of storage nodes responses, the proxy initiates 2nd-phase by
sending commit confirmations to object servers.
* Upon receipt of commit message, object servers rename ``.data`` files to
#. Upon receipt of commit message, object servers rename ``.data`` files to
include the ``#d`` substring, indicating successful PUT, and send a final
response to the proxy server.
* The proxy waits for `ec_ndata + 1` object servers to respond with a
#. The proxy waits for `ec_ndata + 1` object servers to respond with a
success (2xx) status before responding to the client with a successful
status.

3
docker/install_scripts/10_apk_install_prereqs.sh
View File

@ -25,6 +25,3 @@ apk add --update \
libxslt-dev \
libxml2 \
libxml2-dev \
python \
python-dev \
py-pip

57
docker/install_scripts/15_install_python_and_pip_src.sh
View File

@ -1,57 +0,0 @@
#!/bin/sh
set -e
echo
echo
echo
echo "building python and pip"
# export PATH=$PATH:/usr/include
mkdir $BUILD_DIR/python27
mkdir $BUILD_DIR/python36
echo
echo
echo
echo "building python 2.7.15"
cd $BUILD_DIR/python27
wget https://www.python.org/ftp/python/2.7.15/Python-2.7.15.tgz
tar -zxvf Python-2.7.15.tgz
cd Python-2.7.15
./configure --enable-optimizations
make
make DESTDIR=/opt/python27 install
echo
echo
echo
echo "building python 3.6.5"
cd $BUILD_DIR/python36
wget https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz
tar -zxvf Python-3.6.5.tgz
cd Python-3.6.5
./configure --enable-optimizations
make
make DESTDIR=/opt/python36 install
export PATH=$PATH:/opt/python27/usr/local/bin:/opt/python36/usr/local/bin
echo "export PATH=$PATH:/opt/python27/usr/local/bin:/opt/python36/usr/local/bin" >> /etc/profile
echo
echo
echo
echo "building pip"
wget https://bootstrap.pypa.io/get-pip.py
python ./get-pip.py
echo
echo
echo
echo "deleting python internal test dirs"
for f in `cat /opt/swift/docker/install_scripts/python_test_dirs` ; do rm -rf $f; done
rm -rf $BUILD_DIR/python27
rm -rf $BUILD_DIR/python36

7
docker/install_scripts/20_apk_install_py2.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
set -e
apk add --update \
python \
python-dev \
py-pip

32
docker/install_scripts/20_pip_install_prereqs.sh
View File

@ -1,32 +0,0 @@
#!/bin/sh
set -e
echo "+ + + + + + + + + + upgrading pip" && \
pip install -U pip && \
cd /opt/swift && \
pip install -r requirements.txt
#echo "+ + + + + + + + + + installing pastedeploy" && \
#pip install pastedeploy && \
#echo "+ + + + + + + + + + installing eventlet" && \
#pip install eventlet && \
#echo "+ + + + + + + + + + installing greenlet" && \
#pip install greenlet && \
#echo "+ + + + + + + + + + installing netifaces" && \
#pip install netifaces && \
#echo "+ + + + + + + + + + installing setuptools" && \
#pip install setuptools && \
#echo "+ + + + + + + + + + installing requests" && \
#pip install requests && \
#echo "+ + + + + + + + + + installing six" && \
#pip install six && \
#echo "+ + + + + + + + + + installing cryptography" && \
#pip install cryptography && \
#echo "+ + + + + + + + + + installing dnspython" && \
#pip install dnspython
#echo "+ + + + + + + + + + installing xattr" && \
#pip install xattr
#echo "+ + + + + + + + + + installing pyeclib" && \
#pip install pyeclib
#echo "+ + + + + + + + + + installing lxml" && \
#pip install lxml

10
docker/install_scripts/21_apk_install_py3.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/sh
set -e
apk add --update \
python3 \
python3-dev \
py3-pip
if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi

8
lower-constraints.txt
View File

@ -14,18 +14,18 @@ cmd2==0.8.1
coverage==3.6
cryptography==2.0.2
debtcollector==1.19.0
dnspython==1.14.0
dnspython==1.15.0
docutils==0.11
dulwich==0.19.0
enum-compat==0.0.2
eventlet==0.17.4
eventlet==0.25.0
extras==1.0.0
fixtures==3.0.0
flake8==2.5.5
future==0.16.0
gitdb2==2.0.3
GitPython==2.1.8
greenlet==0.3.1
greenlet==0.3.2
hacking==0.11.0
idna==2.6
imagesize==1.0.0
@ -76,7 +76,7 @@ reno==1.8.0
requests==2.14.2
requests-mock==1.2.0
rfc3986==1.1.0
six==1.9.0
six==1.10.0
smmap2==2.0.3
snowballstemmer==1.2.1
Sphinx==1.6.2

87
releasenotes/notes/2_22_0_release-f60d29508b3c1283.yaml Normal file
View File

@ -0,0 +1,87 @@
---
features:
- |
Experimental support for Python 3.6 and 3.7 is now available.
Note that this requires ``eventlet>=0.25.0``. All unit tests pass,
and running functional tests under Python 2 will pass against
services running under Python 3. Expect full support in the
next minor release.
- |
Log formats are now more configurable and include support for
anonymization. See the ``log_msg_template`` option in ``proxy-server.conf``
and `the Swift documentation <https://docs.openstack.org/swift/latest/logs.html#proxy-logs>`__
for more information.
- |
Added an operator tool, ``swift-container-deleter``, to asynchronously
delete some or all objects in a container using the object expirers.
- |
Swift-all-in-one Docker images are now built and published to
https://hub.docker.com/r/openstackswift/saio. These are intended
for use as development targets, but will hopefully be useful as a
starting point for other work involving containerizing Swift.
upgrade:
- |
The ``object-expirer`` may now be configured in ``object-server.conf``.
This is in anticipation of a future change to allow the ``object-expirer``
to be deployed on all nodes that run the ``object-server``.
- |
**Dependency updates**: we've increased our minimum supported version
of ``cryptography`` to 2.0.2 and ``netifaces`` to 0.8. This is largely due
to the difficulty of continuing to test with the old versions.
If running Swift under Python 3, ``eventlet`` must be at least 0.25.0.
fixes:
- |
Correctness improvements
* The ``proxy-server`` now ignores 404 responses from handoffs without
databases when deciding on the correct response for account and
container requests.
* Object writes to a container whose existence cannot be verified
now 503 instead of 404.
- |
Sharding improvements
* The ``container-replicator`` now only attempts to fetch shard ranges if
the remote indicates that it has shard ranges. Further, it does so
with a timeout to prevent the process from hanging in certain cases.
* The ``proxy-server`` now caches 'updating' shards, improving write
performance for sharded containers. A new config option,
``recheck_updating_shard_ranges``, controls the cache time; set it to
0 to disable caching.
* The ``container-replicator`` now correctly enqueues
``container-reconciler`` work for sharded containers.
- |
S3 API improvements
* Unsigned payloads work with v4 signatures once more.
* Multipart upload parts may now be copied from other multipart uploads.
* CompleteMultipartUpload requests with a ``Content-MD5`` now work.
* ``Content-Type`` can now be updated when copying an object.
* Fixed v1 listings that end with a non-ASCII object name.
- |
Background corruption-detection improvements
* Detect and remove invalid entries from ``hashes.pkl``
* When object path is not a directory, just quarantine it,
rather than the whole suffix.
- |
Various other minor bug fixes and improvements.

8
requirements.txt
View File

@ -2,14 +2,14 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
dnspython>=1.14.0;python_version=='2.7' # http://www.dnspython.org/LICENSE
eventlet>=0.17.4,!=0.23.0 # MIT
greenlet>=0.3.1
dnspython>=1.15.0;python_version=='2.7' # http://www.dnspython.org/LICENSE
eventlet>=0.25.0 # MIT
greenlet>=0.3.2
netifaces>=0.8,!=0.10.0,!=0.10.1
PasteDeploy>=1.3.3
lxml>=3.4.1
requests>=2.14.2 # Apache-2.0
six>=1.9.0
six>=1.10.0
xattr>=0.4;sys_platform!='win32' # MIT
PyECLib>=1.3.1 # BSD
cryptography>=2.0.2 # BSD/Apache-2.0

3
setup.cfg
View File

@ -16,6 +16,9 @@ classifier =
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
[pbr]
skip_authors = True

36
swift/account/server.py
View File

@ -21,8 +21,6 @@ from swift import gettext_ as _
from eventlet import Timeout
import six
import swift.common.db
from swift.account.backend import AccountBroker, DATADIR
from swift.account.utils import account_listing_response, get_response_headers
@ -110,6 +108,14 @@ class AccountController(BaseStorageServer):
broker.delete_db(req_timestamp.internal)
return self._deleted_response(broker, req, HTTPNoContent)
def _update_metadata(self, req, broker, req_timestamp):
metadata = {
wsgi_to_str(key): (wsgi_to_str(value), req_timestamp.internal)
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key)}
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
@public
@timing_stats()
def PUT(self, req):
@ -169,24 +175,7 @@ class AccountController(BaseStorageServer):
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
if six.PY2:
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
else:
for key, value in req.headers.items():
if is_sys_or_user_meta('account', key):
# Cast to native strings, so that json inside
# updata_metadata eats the data.
try:
value = value.encode('latin-1').decode('utf-8')
except UnicodeDecodeError:
raise HTTPBadRequest(
'Metadata must be valid UTF-8')
metadata[key] = (value, timestamp.internal)
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
self._update_metadata(req, broker, timestamp)
if created:
return HTTPCreated(request=req)
else:
@ -287,12 +276,7 @@ class AccountController(BaseStorageServer):
broker = self._get_account_broker(drive, part, account)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
metadata = {}
metadata.update((key, (value, req_timestamp.internal))
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
self._update_metadata(req, broker, req_timestamp)
return HTTPNoContent(request=req)
def __call__(self, env, start_response):

6
swift/account/utils.py
View File

@ -18,7 +18,7 @@ import json
import six
from swift.common.middleware import listing_formats
from swift.common.swob import HTTPOk, HTTPNoContent
from swift.common.swob import HTTPOk, HTTPNoContent, str_to_wsgi
from swift.common.utils import Timestamp
from swift.common.storage_policy import POLICIES
@ -64,8 +64,8 @@ def get_response_headers(broker):
for key, value in stats.items():
header_name = header_prefix % key.replace('_', '-')
resp_headers[header_name] = value
resp_headers.update((key, value)
for key, (value, timestamp) in
resp_headers.update((str_to_wsgi(key), str_to_wsgi(value))
for key, (value, _timestamp) in
broker.metadata.items() if value != '')
return resp_headers

4
swift/common/constraints.py
View File

@ -100,10 +100,6 @@ def reload_constraints():
reload_constraints()
# Maximum slo segments in buffer
MAX_BUFFERED_SLO_SEGMENTS = 10000
# By default the maximum number of allowed headers depends on the number of max
# allowed metadata settings plus a default value of 36 for swift internally
# generated headers and regular http headers. If for some reason this is not

32
swift/common/header_key_dict.py
View File

@ -16,10 +16,20 @@
import six
def _title(s):
if six.PY2:
return s.title()
else:
return s.encode('latin1').title().decode('latin1')
class HeaderKeyDict(dict):
"""
A dict that title-cases all keys on the way in, so as to be
case-insensitive.
Note that all keys and values are expected to be wsgi strings,
though some allowances are made when setting values.
"""
def __init__(self, base_headers=None, **kwargs):
if base_headers:
@ -29,32 +39,32 @@ class HeaderKeyDict(dict):
def update(self, other):
if hasattr(other, 'keys'):
for key in other.keys():
self[key.title()] = other[key]
self[_title(key)] = other[key]
else:
for key, value in other:
self[key.title()] = value
self[_title(key)] = value
def __getitem__(self, key):
return dict.get(self, key.title())
return dict.get(self, _title(key))
def __setitem__(self, key, value):
if value is None:
self.pop(key.title(), None)
self.pop(_title(key), None)
elif six.PY2 and isinstance(value, six.text_type):
return dict.__setitem__(self, key.title(), value.encode('utf-8'))
return dict.__setitem__(self, _title(key), value.encode('utf-8'))
elif six.PY3 and isinstance(value, six.binary_type):
return dict.__setitem__(self, key.title(), value.decode('latin-1'))
return dict.__setitem__(self, _title(key), value.decode('latin-1'))
else:
return dict.__setitem__(self, key.title(), str(value))
return dict.__setitem__(self, _title(key), str(value))
def __contains__(self, key):
return dict.__contains__(self, key.title())
return dict.__contains__(self, _title(key))
def __delitem__(self, key):
return dict.__delitem__(self, key.title())
return dict.__delitem__(self, _title(key))
def get(self, key, default=None):
return dict.get(self, key.title(), default)
return dict.get(self, _title(key), default)
def setdefault(self, key, value=None):
if key not in self:
@ -62,4 +72,4 @@ class HeaderKeyDict(dict):
return self[key]
def pop(self, key, default=None):
return dict.pop(self, key.title(), default)
return dict.pop(self, _title(key), default)

20
swift/common/internal_client.py
View File

@ -229,7 +229,7 @@ class InternalClient(object):
def _get_metadata(
self, path, metadata_prefix='', acceptable_statuses=(2,),
headers=None):
headers=None, params=None):
"""
Gets metadata by doing a HEAD on a path and using the metadata_prefix
to get values from the headers returned.
@ -252,7 +252,8 @@ class InternalClient(object):
"""
headers = headers or {}
resp = self.make_request('HEAD', path, headers, acceptable_statuses)
resp = self.make_request('HEAD', path, headers, acceptable_statuses,
params=params)
metadata_prefix = metadata_prefix.lower()
metadata = {}
for k, v in resp.headers.items():
@ -406,7 +407,8 @@ class InternalClient(object):
int(resp.headers.get('x-account-object-count', 0)))
def get_account_metadata(
self, account, metadata_prefix='', acceptable_statuses=(2,)):
self, account, metadata_prefix='', acceptable_statuses=(2,),
params=None):
"""Gets account metadata.
:param account: Account on which to get the metadata.
@ -425,7 +427,8 @@ class InternalClient(object):
"""
path = self.make_path(account)
return self._get_metadata(path, metadata_prefix, acceptable_statuses)
return self._get_metadata(path, metadata_prefix, acceptable_statuses,
headers=None, params=params)
def set_account_metadata(
self, account, metadata, metadata_prefix='',
@ -516,7 +519,7 @@ class InternalClient(object):
def get_container_metadata(
self, account, container, metadata_prefix='',
acceptable_statuses=(2,)):
acceptable_statuses=(2,), params=None):
"""Gets container metadata.
:param account: The container's account.
@ -536,7 +539,8 @@ class InternalClient(object):
"""
path = self.make_path(account, container)
return self._get_metadata(path, metadata_prefix, acceptable_statuses)
return self._get_metadata(path, metadata_prefix, acceptable_statuses,
params=params)
def iter_objects(
self, account, container, marker='', end_marker='', prefix='',
@ -618,7 +622,7 @@ class InternalClient(object):
def get_object_metadata(
self, account, container, obj, metadata_prefix='',
acceptable_statuses=(2,), headers=None):
acceptable_statuses=(2,), headers=None, params=None):
"""Gets object metadata.
:param account: The object's account.
@ -641,7 +645,7 @@ class InternalClient(object):
path = self.make_path(account, container, obj)
return self._get_metadata(path, metadata_prefix, acceptable_statuses,
headers=headers)
headers=headers, params=params)
def get_object(self, account, container, obj, headers,
acceptable_statuses=(2,), params=None):

4
swift/common/middleware/copy.py
View File

@ -123,7 +123,7 @@ from swift.common.http import HTTP_MULTIPLE_CHOICES, is_success, HTTP_OK
from swift.common.constraints import check_account_format, MAX_FILE_SIZE
from swift.common.request_helpers import copy_header_subset, remove_items, \
is_sys_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \
check_path_header
check_path_header, OBJECT_SYSMETA_CONTAINER_UPDATE_OVERRIDE_PREFIX
from swift.common.wsgi import WSGIContext, make_subrequest
@ -405,7 +405,7 @@ class ServerSideCopyMiddleware(object):
# since we're not copying the source etag, make sure that any
# container update override values are not copied.
remove_items(sink_req.headers, lambda k: k.startswith(
'X-Object-Sysmeta-Container-Update-Override-'))
OBJECT_SYSMETA_CONTAINER_UPDATE_OVERRIDE_PREFIX.title()))
# We no longer need these headers
sink_req.headers.pop('X-Copy-From', None)

5
swift/common/middleware/crypto/decrypter.py
View File

@ -23,7 +23,8 @@ from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
load_crypto_meta, extract_crypto_meta, Crypto
from swift.common.exceptions import EncryptionException, UnknownSecretIdError
from swift.common.request_helpers import get_object_transient_sysmeta, \
get_sys_meta_prefix, get_user_meta_prefix
get_sys_meta_prefix, get_user_meta_prefix, \
get_container_update_override_key
from swift.common.swob import Request, HTTPException, \
HTTPInternalServerError, wsgi_to_bytes, bytes_to_wsgi
from swift.common.utils import get_logger, config_true_value, \
@ -220,7 +221,7 @@ class DecrypterObjContext(BaseDecrypterContext):
required=True)
mod_hdr_pairs.append(('Etag', decrypted_etag))
etag_header = 'X-Object-Sysmeta-Container-Update-Override-Etag'
etag_header = get_container_update_override_key('etag')
encrypted_etag = self._response_header_value(etag_header)
if encrypted_etag:
decrypted_etag = self._decrypt_header(

12
swift/common/middleware/crypto/encrypter.py
View File

@ -22,7 +22,8 @@ from swift.common.http import is_success
from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
dump_crypto_meta, append_crypto_meta, Crypto
from swift.common.request_helpers import get_object_transient_sysmeta, \
strip_user_meta_prefix, is_user_meta, update_etag_is_at_header
strip_user_meta_prefix, is_user_meta, update_etag_is_at_header, \
get_container_update_override_key
from swift.common.swob import Request, Match, HTTPException, \
HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi
from swift.common.utils import get_logger, config_true_value, \
@ -100,8 +101,8 @@ class EncInputWrapper(object):
# remove any Etag from headers, it won't be valid for ciphertext and
# we'll send the ciphertext Etag later in footer metadata
client_etag = req.headers.pop('etag', None)
container_listing_etag_header = req.headers.get(
'X-Object-Sysmeta-Container-Update-Override-Etag')
override_header = get_container_update_override_key('etag')
container_listing_etag_header = req.headers.get(override_header)
def footers_callback(footers):
if inner_callback:
@ -152,8 +153,7 @@ class EncInputWrapper(object):
# This may be None if no override was set and no data was read. An
# override value of '' will be passed on.
container_listing_etag = footers.get(
'X-Object-Sysmeta-Container-Update-Override-Etag',
container_listing_etag_header)
override_header, container_listing_etag_header)
if container_listing_etag is None:
container_listing_etag = plaintext_etag
@ -174,7 +174,7 @@ class EncInputWrapper(object):
self.crypto, container_listing_etag,
self.keys['container'])
crypto_meta['key_id'] = self.keys['id']
footers['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
footers[override_header] = \
append_crypto_meta(val, crypto_meta)
# else: no override was set and no data was read

2
swift/common/middleware/dlo.py
View File

@ -304,6 +304,8 @@ class GetContext(WSGIContext):
actual_content_length = None
content_length_for_swob_range = None
req.range = None
else:
req.range = None
response_headers = [
(h, v) for h, v in response_headers

5
swift/common/middleware/s3api/controllers/multi_upload.py
View File

@ -70,6 +70,7 @@ import six
from swift.common.swob import Range, bytes_to_wsgi
from swift.common.utils import json, public, reiterate
from swift.common.db import utf8encode
from swift.common.request_helpers import get_container_update_override_key
from six.moves.urllib.parse import quote, urlparse
@ -182,7 +183,7 @@ class PartController(Controller):
'X-Object-Sysmeta-Swift3-Etag': '', # for legacy data
'X-Object-Sysmeta-Slo-Etag': '',
'X-Object-Sysmeta-Slo-Size': '',
'X-Object-Sysmeta-Container-Update-Override-Etag': '',
get_container_update_override_key('etag'): '',
})
resp = req.get_response(self.app)
@ -634,7 +635,7 @@ class UploadController(Controller):
headers[sysmeta_header('object', 'etag')] = s3_etag
# Leave base header value blank; SLO will populate
c_etag = '; s3_etag=%s' % s3_etag
headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = c_etag
headers[get_container_update_override_key('etag')] = c_etag
too_small_message = ('s3api requires that each segment be at least '
'%d bytes' % self.conf.min_segment_size)

2
swift/common/middleware/s3api/controllers/obj.py
View File

@ -133,7 +133,7 @@ class ObjectController(Controller):
# delete object metadata from response
for key in list(resp.headers.keys()):
if key.startswith('x-amz-meta-'):
if key.lower().startswith('x-amz-meta-'):
del resp.headers[key]
resp.status = HTTP_OK

8
swift/common/middleware/s3api/s3request.py
View File

@ -33,7 +33,8 @@ from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
HTTP_CONFLICT, HTTP_UNPROCESSABLE_ENTITY, HTTP_REQUEST_ENTITY_TOO_LARGE, \
HTTP_PARTIAL_CONTENT, HTTP_NOT_MODIFIED, HTTP_PRECONDITION_FAILED, \
HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, HTTP_LENGTH_REQUIRED, \
HTTP_BAD_REQUEST, HTTP_REQUEST_TIMEOUT, is_success
HTTP_BAD_REQUEST, HTTP_REQUEST_TIMEOUT, HTTP_SERVICE_UNAVAILABLE, \
is_success
from swift.common.constraints import check_utf8
from swift.proxy.controllers.base import get_container_info, \
@ -52,7 +53,8 @@ from swift.common.middleware.s3api.s3response import AccessDenied, \
InternalError, NoSuchBucket, NoSuchKey, PreconditionFailed, InvalidRange, \
MissingContentLength, InvalidStorageClass, S3NotImplemented, InvalidURI, \
MalformedXML, InvalidRequest, RequestTimeout, InvalidBucketName, \
BadDigest, AuthorizationHeaderMalformed, AuthorizationQueryParametersError
BadDigest, AuthorizationHeaderMalformed, \
AuthorizationQueryParametersError, ServiceUnavailable
from swift.common.middleware.s3api.exception import NotS3Request, \
BadSwiftRequest
from swift.common.middleware.s3api.utils import utf8encode, \
@ -1369,6 +1371,8 @@ class S3Request(swob.Request):
**self.signature_does_not_match_kwargs())
if status == HTTP_FORBIDDEN:
raise AccessDenied()
if status == HTTP_SERVICE_UNAVAILABLE:
raise ServiceUnavailable()
raise InternalError('unexpected status code %d' % status)

79
swift/common/middleware/slo.py
View File

@ -330,15 +330,18 @@ from swift.common.middleware.listing_formats import \
from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \
HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \
HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \
HTTPUnauthorized, HTTPConflict, HTTPUnprocessableEntity, Response, Range, \
HTTPUnauthorized, HTTPConflict, HTTPUnprocessableEntity, \
HTTPServiceUnavailable, Response, Range, \
RESPONSE_REASONS, str_to_wsgi, wsgi_to_str, wsgi_quote
from swift.common.utils import get_logger, config_true_value, \
get_valid_utf8_str, override_bytes_from_content_type, split_path, \
register_swift_info, RateLimitedIterator, quote, close_if_possible, \
closing_if_possible, LRUCache, StreamingPile, strict_b64decode
closing_if_possible, LRUCache, StreamingPile, strict_b64decode, \
Timestamp
from swift.common.request_helpers import SegmentedIterable, \
get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header
from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS
get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header, \
get_container_update_override_key
from swift.common.constraints import check_utf8
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success
from swift.common.wsgi import WSGIContext, make_subrequest
from swift.common.middleware.bulk import get_response_body, \
@ -500,10 +503,7 @@ def parse_and_validate_input(req_body, req_path):
% (seg_index,))
continue
# re-encode to normalize padding
if six.PY2:
seg_dict['data'] = base64.b64encode(data)
else:
seg_dict['data'] = base64.b64encode(data).decode('ascii')
seg_dict['data'] = base64.b64encode(data).decode('ascii')
if parsed_data and all('data' in d for d in parsed_data):
errors.append(b"Inline data segments require at least one "
@ -734,7 +734,7 @@ class SloGetContext(WSGIContext):
content_range = value
break
# e.g. Content-Range: bytes 0-14289/14290
match = re.match('bytes (\d+)-(\d+)/(\d+)$', content_range)
match = re.match(r'bytes (\d+)-(\d+)/(\d+)$', content_range)
if not match:
# Malformed or missing, so we don't know what we got.
return True
@ -767,7 +767,7 @@ class SloGetContext(WSGIContext):
resp_iter = self._app_call(req.environ)
# make sure this response is for a static large object manifest
slo_marker = slo_etag = slo_size = None
slo_marker = slo_etag = slo_size = slo_timestamp = None
for header, value in self._response_headers:
header = header.lower()
if header == SYSMETA_SLO_ETAG:
@ -777,8 +777,10 @@ class SloGetContext(WSGIContext):
elif (header == 'x-static-large-object' and
config_true_value(value)):
slo_marker = value
elif header == 'x-backend-timestamp':
slo_timestamp = value
if slo_marker and slo_etag and slo_size:
if slo_marker and slo_etag and slo_size and slo_timestamp:
break
if not slo_marker:
@ -822,6 +824,7 @@ class SloGetContext(WSGIContext):
conditional_response=True)
resp.headers.update({
'Etag': '"%s"' % slo_etag,
'X-Manifest-Etag': self._response_header_value('etag'),
'Content-Length': slo_size,
})
return resp(req.environ, start_response)
@ -836,6 +839,35 @@ class SloGetContext(WSGIContext):
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent='%(orig)s SLO MultipartGET', swift_source='SLO')
resp_iter = self._app_call(get_req.environ)
slo_marker = config_true_value(self._response_header_value(
'x-static-large-object'))
if not slo_marker: # will also catch non-2xx responses
got_timestamp = self._response_header_value(
'x-backend-timestamp') or '0'
if Timestamp(got_timestamp) >= Timestamp(slo_timestamp):
# We've got a newer response available, so serve that.
# Note that if there's data, it's going to be a 200 now,
# not a 206, and we're not going to drop bytes in the
# proxy on the client's behalf. Fortunately, the RFC is
# pretty forgiving for a server; there's no guarantee that
# a Range header will be respected.
resp = Response(
status=self._response_status,
headers=self._response_headers,
app_iter=resp_iter,
request=req,
conditional_etag=resolve_etag_is_at_header(
req, self._response_headers),
conditional_response=is_success(
int(self._response_status[:3])))
return resp(req.environ, start_response)
else:
# We saw newer data that indicated it's an SLO, but
# couldn't fetch the whole thing; 503 seems reasonable?
close_if_possible(resp_iter)
raise HTTPServiceUnavailable(request=req)
# NB: we might have gotten an out-of-date manifest -- that's OK;
# we'll just try to serve the old data
# Any Content-Range from a manifest is almost certainly wrong for the
# full large object.
@ -897,7 +929,9 @@ class SloGetContext(WSGIContext):
response_headers = []
for header, value in resp_headers:
lheader = header.lower()
if lheader not in ('etag', 'content-length'):
if lheader == 'etag':
response_headers.append(('X-Manifest-Etag', value))
elif lheader != 'content-length':
response_headers.append((header, value))
if lheader == SYSMETA_SLO_ETAG:
@ -926,7 +960,7 @@ class SloGetContext(WSGIContext):
r = '%s:%s;' % (seg_dict['hash'], seg_dict['range'])
else:
r = seg_dict['hash']
calculated_etag.update(r.encode('ascii') if six.PY3 else r)
calculated_etag.update(r.encode('ascii'))
if content_length is None:
if config_true_value(seg_dict.get('sub_slo')):
@ -1062,7 +1096,10 @@ class StaticLargeObject(object):
delete_concurrency = int(self.conf.get(
'delete_concurrency', self.concurrency))
self.bulk_deleter = Bulk(
app, {}, delete_concurrency=delete_concurrency, logger=self.logger)
app, {},
max_deletes_per_request=float('inf'),
delete_concurrency=delete_concurrency,
logger=self.logger)
def handle_multipart_get_or_head(self, req, start_response):
"""
@ -1264,9 +1301,7 @@ class StaticLargeObject(object):
resp_dict = {}
if heartbeat:
resp_dict['Response Status'] = err.status
err_body = err.body
if six.PY3:
err_body = err_body.decode('utf-8', errors='replace')
err_body = err.body.decode('utf-8')
resp_dict['Response Body'] = err_body or '\n'.join(
RESPONSE_REASONS.get(err.status_int, ['']))
else:
@ -1321,7 +1356,7 @@ class StaticLargeObject(object):
# Ensure container listings have both etags. However, if any
# middleware to the left of us touched the base value, trust them.
override_header = 'X-Object-Sysmeta-Container-Update-Override-Etag'
override_header = get_container_update_override_key('etag')
val, sep, params = req.headers.get(
override_header, '').partition(';')
req.headers[override_header] = '%s; slo_etag=%s' % (
@ -1380,7 +1415,13 @@ class StaticLargeObject(object):
'sub_slo': True,
'name': obj_path}]
while segments:
if len(segments) > MAX_BUFFERED_SLO_SEGMENTS:
# We chose not to set the limit at max_manifest_segments
# in the case this value was decreased by operators.
# Still it is important to set a limit to avoid this list
# growing too large and causing OOM failures.
# x10 is a best guess as to how much operators would change
# the value of max_manifest_segments.
if len(segments) > self.max_manifest_segments * 10:
raise HTTPBadRequest(
'Too many buffered slo segments to delete.')
seg_data = segments.pop(0)

272
swift/common/middleware/symlink.py
View File

@ -30,12 +30,20 @@ symlink, the header ``X-Symlink-Target-Account: <account>`` must be included.
If omitted, it is inserted automatically with the account of the symlink
object in the PUT request process.
Symlinks must be zero-byte objects. Attempting to PUT a symlink
with a non-empty request body will result in a 400-series error. Also, POST
with X-Symlink-Target header always results in a 400-series error. The target
object need not exist at symlink creation time. It is suggested to set the
``Content-Type`` of symlink objects to a distinct value such as
``application/symlink``.
Symlinks must be zero-byte objects. Attempting to PUT a symlink with a
non-empty request body will result in a 400-series error. Also, POST with
``X-Symlink-Target`` header always results in a 400-series error. The target
object need not exist at symlink creation time.
Clients may optionally include a ``X-Symlink-Target-Etag: <etag>`` header
during the PUT. If present, this will create a "static symlink" instead of a
"dynamic symlink". Static symlinks point to a specific object rather than a
specific name. They do this by using the value set in their
``X-Symlink-Target-Etag`` header when created to verify it still matches the
ETag of the object they're pointing at on a GET. In contrast to a dynamic
symlink the target object referenced in the ``X-Symlink-Target`` header must
exist and its ETag must match the ``X-Symlink-Target-Etag`` or the symlink
creation will return a client error.
A GET/HEAD request to a symlink will result in a request to the target
object referenced by the symlink's ``X-Symlink-Target-Account`` and
@ -45,12 +53,22 @@ GET/HEAD request to a symlink with the query parameter ``?symlink=get`` will
result in the request targeting the symlink itself.
A symlink can point to another symlink. Chained symlinks will be traversed
until target is not a symlink. If the number of chained symlinks exceeds the
limit ``symloop_max`` an error response will be produced. The value of
until the target is not a symlink. If the number of chained symlinks exceeds
the limit ``symloop_max`` an error response will be produced. The value of
``symloop_max`` can be defined in the symlink config section of
`proxy-server.conf`. If not specified, the default ``symloop_max`` value is 2.
If a value less than 1 is specified, the default value will be used.
If a static symlink (i.e. a symlink created with a ``X-Symlink-Target-Etag``
header) targets another static symlink, both of the ``X-Symlink-Target-Etag``
headers must match the target object for the GET to succeed. If a static
symlink targets a dynamic symlink (i.e. a symlink created without a
``X-Symlink-Target-Etag`` header) then the ``X-Symlink-Target-Etag`` header of
the static symlink must be the Etag of the zero-byte object. If a symlink with
a ``X-Symlink-Target-Etag`` targets a large object manifest it must match the
ETag of the manifest (e.g. the ETag as returned by ``multipart-manifest=get``
or value in the ``X-Manifest-Etag`` header).
A HEAD/GET request to a symlink object behaves as a normal HEAD/GET request
to the target object. Therefore issuing a HEAD request to the symlink will
return the target metadata, and issuing a GET request to the symlink will
@ -58,13 +76,22 @@ return the data and metadata of the target object. To return the symlink
metadata (with its empty body) a GET/HEAD request with the ``?symlink=get``
query parameter must be sent to a symlink object.
A POST request to a symlink will result in a 307 TemporaryRedirect response.
A POST request to a symlink will result in a 307 Temporary Redirect response.
The response will contain a ``Location`` header with the path of the target
object as the value. The request is never redirected to the target object by
Swift. Nevertheless, the metadata in the POST request will be applied to the
symlink because object servers cannot know for sure if the current object is a
symlink or not in eventual consistency.
A symlink's ``Content-Type`` is completely independent from its target. As a
convenience Swift will automatically set the ``Content-Type`` on a symlink PUT
if not explicitly set by the client. If the client sends a
``X-Symlink-Target-Etag`` Swift will set the symlink's ``Content-Type`` to that
of the target, otherwise it will be set to ``application/symlink``. You can
review a symlink's ``Content-Type`` using the ``?symlink=get`` interface. You
can change a symlink's ``Content-Type`` using a POST request. The symlink's
``Content-Type`` will appear in the container listing.
A DELETE request to a symlink will delete the symlink itself. The target
object will not be deleted.
@ -73,7 +100,7 @@ will copy the target object. The same request to a symlink with the query
parameter ``?symlink=get`` will copy the symlink itself.
An OPTIONS request to a symlink will respond with the options for the symlink
only, the request will not be redirected to the target object. Please note that
only; the request will not be redirected to the target object. Please note that
if the symlink's target object is in another container with CORS settings, the
response will not reflect the settings.
@ -82,7 +109,8 @@ will result in a 400-series error. The GET/HEAD tempurls honor the scope of
the tempurl key. Container tempurl will only work on symlinks where the target
container is the same as the symlink. In case a symlink targets an object
in a different container, a GET/HEAD request will result in a 401 Unauthorized
error. The account level tempurl will allow cross container symlinks.
error. The account level tempurl will allow cross-container symlinks, but not
cross-account symlinks.
If a symlink object is overwritten while it is in a versioned container, the
symlink object itself is versioned, not the referenced object.
@ -91,8 +119,19 @@ A GET request with query parameter ``?format=json`` to a container which
contains symlinks will respond with additional information ``symlink_path``
for each symlink object in the container listing. The ``symlink_path`` value
is the target path of the symlink. Clients can differentiate symlinks and
other objects by this function. Note that responses of any other format
(e.g.``?format=xml``) won't include ``symlink_path`` info.
other objects by this function. Note that responses in any other format
(e.g. ``?format=xml``) won't include ``symlink_path`` info. If a
``X-Symlink-Target-Etag`` header was included on the symlink, JSON container
listings will include that value in a ``symlink_etag`` key and the target
object's ``Content-Length`` will be included in the key ``symlink_bytes``.
If a static symlink targets a static large object manifest it will carry
forward the SLO's size and slo_etag in the container listing using the
``symlink_bytes`` and ``slo_etag`` keys. However, manifests created before
swift v2.12.0 (released Dec 2016) do not contain enough metadata to propagate
the extra SLO information to the listing. Clients may recreate the manifest
(COPY w/ ``?multipart-manfiest=get``) before creating a static symlink to add
the requisite metadata.
Errors
@ -105,7 +144,10 @@ Errors
* GET/HEAD traversing more than ``symloop_max`` chained symlinks will
produce a 409 Conflict error.
* POSTs will produce a 307 TemporaryRedirect error.
* PUT/GET/HEAD on a symlink that inclues a ``X-Symlink-Target-Etag`` header
that does not match the target will poduce a 409 Conflict error.
* POSTs will produce a 307 Temporary Redirect error.
----------
Deployment
@ -160,15 +202,15 @@ import os
from cgi import parse_header
from swift.common.utils import get_logger, register_swift_info, split_path, \
MD5_OF_EMPTY_STRING, closing_if_possible
MD5_OF_EMPTY_STRING, close_if_possible, closing_if_possible
from swift.common.constraints import check_account_format
from swift.common.wsgi import WSGIContext, make_subrequest
from swift.common.request_helpers import get_sys_meta_prefix, \
check_path_header
check_path_header, get_container_update_override_key
from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \
HTTPException, HTTPConflict, HTTPPreconditionFailed, wsgi_quote, \
wsgi_unquote
from swift.common.http import is_success
from swift.common.http import is_success, HTTP_NOT_FOUND
from swift.common.exceptions import LinkIterError
from swift.common.header_key_dict import HeaderKeyDict
@ -176,22 +218,33 @@ DEFAULT_SYMLOOP_MAX = 2
# Header values for symlink target path strings will be quoted values.
TGT_OBJ_SYMLINK_HDR = 'x-symlink-target'
TGT_ACCT_SYMLINK_HDR = 'x-symlink-target-account'
TGT_ETAG_SYMLINK_HDR = 'x-symlink-target-etag'
TGT_BYTES_SYMLINK_HDR = 'x-symlink-target-bytes'
TGT_OBJ_SYSMETA_SYMLINK_HDR = get_sys_meta_prefix('object') + 'symlink-target'
TGT_ACCT_SYSMETA_SYMLINK_HDR = \
get_sys_meta_prefix('object') + 'symlink-target-account'
TGT_ETAG_SYSMETA_SYMLINK_HDR = \
get_sys_meta_prefix('object') + 'symlink-target-etag'
TGT_BYTES_SYSMETA_SYMLINK_HDR = \
get_sys_meta_prefix('object') + 'symlink-target-bytes'
def _check_symlink_header(req):
def _validate_and_prep_request_headers(req):
"""
Validate that the value from x-symlink-target header is
well formatted. We assume the caller ensures that
Validate that the value from x-symlink-target header is well formatted
and that the x-symlink-target-etag header (if present) does not contain
problematic characters. We assume the caller ensures that
x-symlink-target header is present in req.headers.
:param req: HTTP request object
:returns: a tuple, the full versioned path to the object (as a WSGI string)
and the X-Symlink-Target-Etag header value which may be None
:raise: HTTPPreconditionFailed if x-symlink-target value
is not well formatted.
:raise: HTTPBadRequest if the x-symlink-target value points to the request
path.
:raise: HTTPBadRequest if the x-symlink-target-etag value contains
a semicolon, double-quote, or backslash.
"""
# N.B. check_path_header doesn't assert the leading slash and
# copy middleware may accept the format. In the symlink, API
@ -228,43 +281,48 @@ def _check_symlink_header(req):
raise HTTPBadRequest(
body='Symlink cannot target itself',
request=req, content_type='text/plain')
etag = req.headers.get(TGT_ETAG_SYMLINK_HDR, None)
if etag and any(c in etag for c in ';"\\'):
# See cgi.parse_header for why the above chars are problematic
raise HTTPBadRequest(
body='Bad %s format' % TGT_ETAG_SYMLINK_HDR.title(),
request=req, content_type='text/plain')
if not (etag or req.headers.get('Content-Type')):
req.headers['Content-Type'] = 'application/symlink'
return '/v1/%s/%s/%s' % (account, container, obj), etag
def symlink_usermeta_to_sysmeta(headers):
"""
Helper function to translate from X-Symlink-Target and
X-Symlink-Target-Account to X-Object-Sysmeta-Symlink-Target
and X-Object-Sysmeta-Symlink-Target-Account.
Helper function to translate from client-facing X-Symlink-* headers
to cluster-facing X-Object-Sysmeta-Symlink-* headers.
:param headers: request headers dict. Note that the headers dict
will be updated directly.
"""
# To preseve url-encoded value in the symlink header, use raw value
if TGT_OBJ_SYMLINK_HDR in headers:
headers[TGT_OBJ_SYSMETA_SYMLINK_HDR] = headers.pop(
TGT_OBJ_SYMLINK_HDR)
if TGT_ACCT_SYMLINK_HDR in headers:
headers[TGT_ACCT_SYSMETA_SYMLINK_HDR] = headers.pop(
TGT_ACCT_SYMLINK_HDR)
for user_hdr, sysmeta_hdr in (
(TGT_OBJ_SYMLINK_HDR, TGT_OBJ_SYSMETA_SYMLINK_HDR),
(TGT_ACCT_SYMLINK_HDR, TGT_ACCT_SYSMETA_SYMLINK_HDR)):
if user_hdr in headers:
headers[sysmeta_hdr] = headers.pop(user_hdr)
def symlink_sysmeta_to_usermeta(headers):
"""
Helper function to translate from X-Object-Sysmeta-Symlink-Target and
X-Object-Sysmeta-Symlink-Target-Account to X-Symlink-Target and
X-Sysmeta-Symlink-Target-Account
Helper function to translate from cluster-facing
X-Object-Sysmeta-Symlink-* headers to client-facing X-Symlink-* headers.
:param headers: request headers dict. Note that the headers dict
will be updated directly.
"""
if TGT_OBJ_SYSMETA_SYMLINK_HDR in headers:
headers[TGT_OBJ_SYMLINK_HDR] = headers.pop(
TGT_OBJ_SYSMETA_SYMLINK_HDR)
if TGT_ACCT_SYSMETA_SYMLINK_HDR in headers:
headers[TGT_ACCT_SYMLINK_HDR] = headers.pop(
TGT_ACCT_SYSMETA_SYMLINK_HDR)
for user_hdr, sysmeta_hdr in (
(TGT_OBJ_SYMLINK_HDR, TGT_OBJ_SYSMETA_SYMLINK_HDR),
(TGT_ACCT_SYMLINK_HDR, TGT_ACCT_SYSMETA_SYMLINK_HDR),
(TGT_ETAG_SYMLINK_HDR, TGT_ETAG_SYSMETA_SYMLINK_HDR),
(TGT_BYTES_SYMLINK_HDR, TGT_BYTES_SYSMETA_SYMLINK_HDR)):
if sysmeta_hdr in headers:
headers[user_hdr] = headers.pop(sysmeta_hdr)
class SymlinkContainerContext(WSGIContext):
@ -308,9 +366,10 @@ class SymlinkContainerContext(WSGIContext):
def _extract_symlink_path_json(self, obj_dict, swift_version, account):
"""
Extract the symlink path from the hash value
:return: object dictionary with additional key:value pair if object
is a symlink. The new key is symlink_path.
Extract the symlink info from the hash value
:return: object dictionary with additional key:value pairs when object
is a symlink. i.e. new symlink_path, symlink_etag and
symlink_bytes keys
"""
if 'hash' in obj_dict:
hash_value, meta = parse_header(obj_dict['hash'])
@ -321,6 +380,10 @@ class SymlinkContainerContext(WSGIContext):
target = meta[key]
elif key == 'symlink_target_account':
account = meta[key]
elif key == 'symlink_target_etag':
obj_dict['symlink_etag'] = meta[key]
elif key == 'symlink_target_bytes':
obj_dict['symlink_bytes'] = int(meta[key])
else:
# make sure to add all other (key, values) back in place
obj_dict['hash'] += '; %s=%s' % (key, meta[key])
@ -370,10 +433,11 @@ class SymlinkObjectContext(WSGIContext):
except LinkIterError:
errmsg = 'Too many levels of symbolic links, ' \
'maximum allowed is %d' % self.symloop_max
raise HTTPConflict(
body=errmsg, request=req, content_type='text/plain')
raise HTTPConflict(body=errmsg, request=req,
content_type='text/plain')
def _recursive_get_head(self, req):
def _recursive_get_head(self, req, target_etag=None,
follow_softlinks=True):
resp = self._app_call(req.environ)
def build_traversal_req(symlink_target):
@ -396,14 +460,35 @@ class SymlinkObjectContext(WSGIContext):
symlink_target = self._response_header_value(
TGT_OBJ_SYSMETA_SYMLINK_HDR)
if symlink_target:
resp_etag = self._response_header_value(
TGT_ETAG_SYSMETA_SYMLINK_HDR)
if symlink_target and (resp_etag or follow_softlinks):
close_if_possible(resp)
found_etag = resp_etag or self._response_header_value('etag')
if target_etag and target_etag != found_etag:
raise HTTPConflict(
body='X-Symlink-Target-Etag headers do not match',
headers={
'Content-Type': 'text/plain',
'Content-Location': self._last_target_path})
if self._loop_count >= self.symloop_max:
raise LinkIterError()
# format: /<account name>/<container name>/<object name>
new_req = build_traversal_req(symlink_target)
self._loop_count += 1
return self._recursive_get_head(new_req)
return self._recursive_get_head(new_req, target_etag=resp_etag)
else:
final_etag = self._response_header_value('etag')
if final_etag and target_etag and target_etag != final_etag:
close_if_possible(resp)
body = ('Object Etag %r does not match '
'X-Symlink-Target-Etag header %r')
raise HTTPConflict(
body=body % (final_etag, target_etag),
headers={
'Content-Type': 'text/plain',
'Content-Location': self._last_target_path})
if self._last_target_path:
# Content-Location will be applied only when one or more
# symlink recursion occurred.
@ -417,6 +502,47 @@ class SymlinkObjectContext(WSGIContext):
return resp
def _validate_etag_and_update_sysmeta(self, req, symlink_target_path,
etag):
# next we'll make sure the E-Tag matches a real object
new_req = make_subrequest(
req.environ, path=wsgi_quote(symlink_target_path), method='HEAD',
swift_source='SYM')
self._last_target_path = symlink_target_path
resp = self._recursive_get_head(new_req, target_etag=etag,
follow_softlinks=False)
if self._get_status_int() == HTTP_NOT_FOUND:
raise HTTPConflict(
body='X-Symlink-Target does not exist',
headers={
'Content-Type': 'text/plain',
'Content-Location': self._last_target_path})
if not is_success(self._get_status_int()):
return resp
response_headers = HeaderKeyDict(self._response_headers)
# carry forward any etag update params (e.g. "slo_etag"), we'll append
# symlink_target_* params to this header after this method returns
override_header = get_container_update_override_key('etag')
if override_header in response_headers and \
override_header not in req.headers:
sep, params = response_headers[override_header].partition(';')[1:]
req.headers[override_header] = MD5_OF_EMPTY_STRING + sep + params
# It's troublesome that there's so much leakage with SLO
if 'X-Object-Sysmeta-Slo-Etag' in response_headers and \
override_header not in req.headers:
req.headers[override_header] = '%s; slo_etag=%s' % (
MD5_OF_EMPTY_STRING,
response_headers['X-Object-Sysmeta-Slo-Etag'])
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = (
response_headers.get('x-object-sysmeta-slo-size') or
response_headers['Content-Length'])
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
if not req.headers.get('Content-Type'):
req.headers['Content-Type'] = response_headers['Content-Type']
def handle_put(self, req):
"""
Handle put request when it contains X-Symlink-Target header.
@ -425,13 +551,23 @@ class SymlinkObjectContext(WSGIContext):
:param req: HTTP PUT object request
:returns: Response Iterator
"""
if req.content_length != 0:
if req.content_length is None:
has_body = (req.body_file.read(1) != b'')
else:
has_body = (req.content_length != 0)
if has_body:
raise HTTPBadRequest(
body='Symlink requests require a zero byte body',
request=req,
content_type='text/plain')
_check_symlink_header(req)
symlink_target_path, etag = _validate_and_prep_request_headers(req)
if etag:
resp = self._validate_etag_and_update_sysmeta(
req, symlink_target_path, etag)
if resp is not None:
return resp
# N.B. TGT_ETAG_SYMLINK_HDR was converted as part of verifying it
symlink_usermeta_to_sysmeta(req.headers)
# Store info in container update that this object is a symlink.
# We have a design decision to use etag space to store symlink info for
@ -441,17 +577,31 @@ class SymlinkObjectContext(WSGIContext):
# listing result for clients.
# To create override etag easily, we have a constraint that the symlink
# must be 0 byte so we can add etag of the empty string + symlink info
# here, simply. Note that this override etag may be encrypted in the
# container db by encryption middleware.
# here, simply (if no other override etag was provided). Note that this
# override etag may be encrypted in the container db by encryption
# middleware.
etag_override = [
MD5_OF_EMPTY_STRING,
req.headers.get(get_container_update_override_key('etag'),
MD5_OF_EMPTY_STRING),
'symlink_target=%s' % req.headers[TGT_OBJ_SYSMETA_SYMLINK_HDR]
]
if TGT_ACCT_SYSMETA_SYMLINK_HDR in req.headers:
etag_override.append(
'symlink_target_account=%s' %
req.headers[TGT_ACCT_SYSMETA_SYMLINK_HDR])
req.headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
if TGT_ETAG_SYSMETA_SYMLINK_HDR in req.headers:
# if _validate_etag_and_update_sysmeta or a middleware sets
# TGT_ETAG_SYSMETA_SYMLINK_HDR then they need to also set
# TGT_BYTES_SYSMETA_SYMLINK_HDR. If they forget, they get a
# KeyError traceback and client gets a ServerError
etag_override.extend([
'symlink_target_etag=%s' %
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR],
'symlink_target_bytes=%s' %
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR],
])
req.headers[get_container_update_override_key('etag')] = \
'; '.join(etag_override)
return self._app_call(req.environ)
@ -491,11 +641,16 @@ class SymlinkObjectContext(WSGIContext):
TGT_ACCT_SYSMETA_SYMLINK_HDR) or wsgi_quote(account)
location_hdr = os.path.join(
'/', version, target_acc, tgt_co)
headers = {'location': location_hdr}
tgt_etag = self._response_header_value(
TGT_ETAG_SYSMETA_SYMLINK_HDR)
if tgt_etag:
headers[TGT_ETAG_SYMLINK_HDR] = tgt_etag
req.environ['swift.leave_relative_location'] = True
errmsg = 'The requested POST was applied to a symlink. POST ' +\
'directly to the target to apply requested metadata.'
raise HTTPTemporaryRedirect(
body=errmsg, headers={'location': location_hdr})
body=errmsg, headers=headers)
else:
return resp
@ -508,10 +663,7 @@ class SymlinkObjectContext(WSGIContext):
:returns: Response Iterator after start_response has been called
"""
if req.method in ('GET', 'HEAD'):
# if GET request came from versioned writes, then it should get
# the symlink only, not the referenced target
if req.params.get('symlink') == 'get' or \
req.environ.get('swift.source') == 'VW':
if req.params.get('symlink') == 'get':
resp = self.handle_get_head_symlink(req)
else:
resp = self.handle_get_head(req)
@ -578,7 +730,7 @@ def filter_factory(global_conf, **local_conf):
symloop_max = int(conf.get('symloop_max', DEFAULT_SYMLOOP_MAX))
if symloop_max < 1:
symloop_max = int(DEFAULT_SYMLOOP_MAX)
register_swift_info('symlink', symloop_max=symloop_max)
register_swift_info('symlink', symloop_max=symloop_max, static_links=True)
def symlink_mw(app):
return SymlinkMiddleware(app, conf, symloop_max)

2
swift/common/middleware/versioned_writes.py
View File

@ -371,7 +371,7 @@ class VersionedWritesContext(WSGIContext):
# to container, but not READ. This was allowed in previous version
# (i.e., before middleware) so keeping the same behavior here
get_req = make_pre_authed_request(
req.environ, path=wsgi_quote(path_info),
req.environ, path=wsgi_quote(path_info) + '?symlink=get',
headers={'X-Newest': 'True'}, method='GET', swift_source='VW')
source_resp = get_req.get_response(self.app)

13
swift/common/request_helpers.py
View File

@ -44,6 +44,8 @@ from swift.common.wsgi import make_subrequest
OBJECT_TRANSIENT_SYSMETA_PREFIX = 'x-object-transient-sysmeta-'
OBJECT_SYSMETA_CONTAINER_UPDATE_OVERRIDE_PREFIX = \
'x-object-sysmeta-container-update-override-'
def get_param(req, name, default=None):
@ -260,6 +262,17 @@ def get_object_transient_sysmeta(key):
return '%s%s' % (OBJECT_TRANSIENT_SYSMETA_PREFIX, key)
def get_container_update_override_key(key):
"""
Returns the full X-Object-Sysmeta-Container-Update-Override-* header key.
:param key: the key you want to override in the container update
:returns: the full header key
"""
header = '%s%s' % (OBJECT_SYSMETA_CONTAINER_UPDATE_OVERRIDE_PREFIX, key)
return header.title()
def remove_items(headers, condition):
"""
Removes items from a dict whose keys satisfy

24
swift/common/utils.py
View File

@ -3254,6 +3254,9 @@ class GreenAsyncPileWaitallTimeout(Timeout):
pass
DEAD = object()
class GreenAsyncPile(object):
"""
Runs jobs in a pool of green threads, and the results can be retrieved by
@ -3282,6 +3285,8 @@ class GreenAsyncPile(object):
def _run_func(self, func, args, kwargs):
try:
self._responses.put(func(*args, **kwargs))
except Exception:
self._responses.put(DEAD)
finally:
self._inflight -= 1
@ -3332,14 +3337,17 @@ class GreenAsyncPile(object):
return self
def next(self):
try:
rv = self._responses.get_nowait()
except eventlet.queue.Empty:
if self._inflight == 0:
raise StopIteration()
rv = self._responses.get()
self._pending -= 1
return rv
while True:
try:
rv = self._responses.get_nowait()
except eventlet.queue.Empty:
if self._inflight == 0:
raise StopIteration()
rv = self._responses.get()
self._pending -= 1
if rv is DEAD:
continue
return rv
__next__ = next

2
swift/common/wsgi.py
View File

@ -619,7 +619,7 @@ def run_server(conf, logger, sock, global_conf=None):
try:
wsgi.server(sock, app, wsgi_logger, **server_kwargs)
except socket.error as err:
if err[0] != errno.EINVAL:
if err.errno != errno.EINVAL:
raise
pool.waitall()

2
swift/container/replicator.py
View File

@ -138,7 +138,7 @@ class ContainerReplicator(db_replicator.Replicator):
def _fetch_and_merge_shard_ranges(self, http, broker):
with Timeout(self.node_timeout):
response = http.replicate('get_shard_ranges')
if is_success(response.status):
if response and is_success(response.status):
broker.merge_shard_ranges(json.loads(
response.data.decode('ascii')))

11
swift/container/server.py
View File

@ -415,12 +415,11 @@ class ContainerController(BaseStorageServer):
return created
def _update_metadata(self, req, broker, req_timestamp, method):
metadata = {}
metadata.update(
(wsgi_to_str(key), (wsgi_to_str(value), req_timestamp.internal))
metadata = {
wsgi_to_str(key): (wsgi_to_str(value), req_timestamp.internal)
for key, value in req.headers.items()
if key.lower() in self.save_headers or
is_sys_or_user_meta('container', key))
if key.lower() in self.save_headers
or is_sys_or_user_meta('container', key)}
if metadata:
if 'X-Container-Sync-To' in metadata:
if 'X-Container-Sync-To' not in broker.metadata or \
@ -706,7 +705,7 @@ class ContainerController(BaseStorageServer):
def create_listing(self, req, out_content_type, info, resp_headers,
metadata, container_list, container):
for key, (value, timestamp) in metadata.items():
for key, (value, _timestamp) in metadata.items():
if value and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)):
resp_headers[str_to_wsgi(key)] = str_to_wsgi(value)

30
swift/obj/reconstructor.py
View File

@ -400,6 +400,7 @@ class ObjectReconstructor(Daemon):
path, headers, full_get_path)
buckets = defaultdict(dict)
durable_buckets = {}
etag_buckets = {}
error_resp_count = 0
for resp in pile:
@ -443,6 +444,10 @@ class ObjectReconstructor(Daemon):
continue
timestamp = Timestamp(timestamp)
durable = resp.headers.get('X-Backend-Durable-Timestamp')
if durable:
durable_buckets[Timestamp(durable)] = True
etag = resp.headers.get('X-Object-Sysmeta-Ec-Etag')
if not etag:
self.logger.warning('Invalid resp from %s, frag index %s '
@ -468,26 +473,29 @@ class ObjectReconstructor(Daemon):
% (fi_to_rebuild, list(buckets[timestamp])))
break
else:
path = _full_path(node, job['partition'],
datafile_metadata['name'],
job['policy'])
for timestamp, resp in sorted(buckets.items()):
etag = etag_buckets[timestamp]
durable = durable_buckets.get(timestamp)
self.logger.error(
'Unable to get enough responses (%s/%s) '
'to reconstruct %s frag#%s with ETag %s' % (
'Unable to get enough responses (%s/%s) to reconstruct '
'%s %s frag#%s with ETag %s and timestamp %s' % (
len(resp), job['policy'].ec_ndata,
_full_path(node, job['partition'],
datafile_metadata['name'],
job['policy']),
fi_to_rebuild, etag))
'durable' if durable else 'non-durable',
path, fi_to_rebuild, etag, timestamp.internal))
if error_resp_count:
durable = durable_buckets.get(Timestamp(
datafile_metadata['X-Timestamp']))
self.logger.error(
'Unable to get enough responses (%s error responses) '
'to reconstruct %s frag#%s' % (
'to reconstruct %s %s frag#%s' % (
error_resp_count,
_full_path(node, job['partition'],
datafile_metadata['name'],
job['policy']),
fi_to_rebuild))
'durable' if durable else 'non-durable',
path, fi_to_rebuild))
raise DiskFileError('Unable to reconstruct EC archive')

2
swift/obj/replicator.py
View File

@ -393,7 +393,7 @@ class ObjectReplicator(Daemon):
return 1 # failure response code
total_time = time.time() - start_time
for result in results.split('\n'):
for result in results.decode('utf8').split('\n'):
if result == '':
continue
if result.startswith('cd+'):

4
swift/obj/server.py
View File

@ -43,6 +43,8 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \
DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout, \
ChunkReadError, DiskFileXattrNotSupported
from swift.common.request_helpers import \
OBJECT_SYSMETA_CONTAINER_UPDATE_OVERRIDE_PREFIX
from swift.obj import ssync_receiver
from swift.common.http import is_success, HTTP_MOVED_PERMANENTLY
from swift.common.base_storage_server import BaseStorageServer
@ -583,7 +585,7 @@ class ObjectController(BaseStorageServer):
# x-object-sysmeta-container-update-override-* headers take precedence
# over x-backend-container-update-override-* headers
override_prefixes = ['x-backend-container-update-override-',
'x-object-sysmeta-container-update-override-']
OBJECT_SYSMETA_CONTAINER_UPDATE_OVERRIDE_PREFIX]
for override_prefix in override_prefixes:
for key, val in metadata.items():
if key.lower().startswith(override_prefix):

12
swift/obj/ssync_receiver.py
View File

@ -130,17 +130,7 @@ class Receiver(object):
# raised during processing because otherwise the sender could send for
# quite some time before realizing it was all in vain.
self.disconnect = True
try:
self.initialize_request()
except swob.HTTPException:
# Old (pre-0.18.0) eventlet would try to drain the request body
# in a way that's prone to blowing up when the client has
# disconnected. Trick it into skipping that so we don't trip
# ValueError: invalid literal for int() with base 16
# in tests. Note we disconnect shortly after receiving a non-200
# response in the sender code, so this is not *so* crazy to do.
request.environ['wsgi.input'].chunked_input = False
raise
self.initialize_request()
def __call__(self):
"""

4
swift/proxy/controllers/base.py
View File

@ -1282,10 +1282,10 @@ class ResumingGetter(object):
if not self.newest: # one good source is enough
return True
else:
if self.server_type != 'Object' and 'handoff_index' in node and \
if 'handoff_index' in node and \
possible_source.status == HTTP_NOT_FOUND and \
not Timestamp(src_headers.get('x-backend-timestamp', 0)):
# throw out 404s from handoff nodes unless the db is really
# throw out 404s from handoff nodes unless the data is really
# on disk and had been DELETEd
return False
self.statuses.append(possible_source.status)

53
swift/proxy/controllers/obj.py
View File

@ -38,7 +38,7 @@ from hashlib import md5
from swift import gettext_ as _
from greenlet import GreenletExit
from eventlet import GreenPile
from eventlet import GreenPile, sleep
from eventlet.queue import Queue
from eventlet.timeout import Timeout
@ -1066,6 +1066,7 @@ class ECAppIter(object):
# executing the internal_parts_iters.
if self.stashed_iter:
self.stashed_iter.close()
sleep() # Give the per-frag threads a chance to clean up
for it in self.internal_parts_iters:
close_if_possible(it)
@ -2002,6 +2003,10 @@ class ECGetResponseBucket(object):
def set_durable(self):
self._durable = True
@property
def durable(self):
return self._durable
def add_response(self, getter, parts_iter):
if not self.gets:
self.status = getter.last_status
@ -2017,7 +2022,8 @@ class ECGetResponseBucket(object):
# metadata headers for self.headers by selecting the source with
# the latest X-Timestamp.
self.headers = getter.last_headers
elif (getter.last_headers.get('X-Object-Sysmeta-Ec-Etag') !=
elif (self.timestamp_str is not None and # ie, not bad_bucket
getter.last_headers.get('X-Object-Sysmeta-Ec-Etag') !=
self.headers.get('X-Object-Sysmeta-Ec-Etag')):
# Fragments at the same timestamp with different etags are never
# expected. If somehow it happens then ignore those fragments
@ -2054,9 +2060,8 @@ class ECGetResponseBucket(object):
@property
def shortfall(self):
# A non-durable bucket always has a shortfall of at least 1
result = self.policy.ec_ndata - len(self.get_responses())
return max(result, 0 if self._durable else 1)
return max(result, 0)
@property
def shortfall_with_alts(self):
@ -2064,7 +2069,7 @@ class ECGetResponseBucket(object):
# for frags on the alt nodes.
alts = set(self.alt_nodes.keys()).difference(set(self.gets.keys()))
result = self.policy.ec_ndata - (len(self.get_responses()) + len(alts))
return max(result, 0 if self._durable else 1)
return max(result, 0)
def __str__(self):
# return a string summarising bucket state, useful for debugging.
@ -2141,12 +2146,14 @@ class ECGetResponseCollection(object):
def _sort_buckets(self):
def key_fn(bucket):
# Returns a tuple to use for sort ordering:
# buckets with no shortfall sort higher,
# durable buckets with no shortfall sort higher,
# then durable buckets with no shortfall_with_alts,
# then non-durable buckets with no shortfall,
# otherwise buckets with lowest shortfall_with_alts sort higher,
# finally buckets with newer timestamps sort higher.
return (bucket.shortfall <= 0,
(not (bucket.shortfall <= 0) and
(-1 * bucket.shortfall_with_alts)),
return (bucket.durable,
bucket.shortfall <= 0,
-1 * bucket.shortfall_with_alts,
bucket.timestamp_str)
return sorted(self.buckets.values(), key=key_fn, reverse=True)
@ -2196,7 +2203,7 @@ class ECGetResponseCollection(object):
return None
bucket = self.best_bucket
if (bucket is None) or (bucket.shortfall <= 0):
if (bucket is None) or (bucket.shortfall <= 0) or not bucket.durable:
return None
alt_frags = set(bucket.alt_nodes.keys())
@ -2381,7 +2388,11 @@ class ECObjectController(BaseObjectController):
shortfall = bad_bucket.shortfall
best_bucket = buckets.best_bucket
if best_bucket:
shortfall = min(best_bucket.shortfall, shortfall)
shortfall = best_bucket.shortfall
if not best_bucket.durable and shortfall <= 0:
# be willing to go a *little* deeper, slowly
shortfall = 1
shortfall = min(shortfall, bad_bucket.shortfall)
if (extra_requests < max_extra_requests and
shortfall > pile._pending and
(node_iter.nodes_left > 0 or
@ -2395,7 +2406,7 @@ class ECObjectController(BaseObjectController):
buckets.get_extra_headers)
req.range = orig_range
if best_bucket and best_bucket.shortfall <= 0:
if best_bucket and best_bucket.shortfall <= 0 and best_bucket.durable:
# headers can come from any of the getters
resp_headers = best_bucket.headers
resp_headers.pop('Content-Range', None)
@ -2435,10 +2446,28 @@ class ECObjectController(BaseObjectController):
bodies = []
headers = []
for getter, _parts_iter in bad_bucket.get_responses():
if best_bucket and best_bucket.durable:
headers = HeaderKeyDict(getter.last_headers)
t_data_file = headers.get('X-Backend-Data-Timestamp')
t_obj = headers.get('X-Backend-Timestamp',
headers.get('X-Timestamp'))
bad_ts = Timestamp(t_data_file or t_obj or '0')
if bad_ts <= Timestamp(best_bucket.timestamp_str):
# We have reason to believe there's still good data
# out there, it's just currently unavailable
continue
statuses.extend(getter.statuses)
reasons.extend(getter.reasons)
bodies.extend(getter.bodies)
headers.extend(getter.source_headers)
if not statuses and best_bucket and not best_bucket.durable:
# pretend that non-durable bucket was 404s
statuses.append(404)
reasons.append('404 Not Found')
bodies.append(b'')
headers.append({})
resp = self.best_response(
req, statuses, reasons, bodies, 'Object',
headers=headers)

6
test/functional/__init__.py
View File

@ -16,6 +16,7 @@
from __future__ import print_function
import mock
import os
import six
from six.moves.urllib.parse import urlparse, urlsplit, urlunsplit
import sys
import pickle
@ -25,6 +26,7 @@ import eventlet
import eventlet.debug
import functools
import random
import base64
from time import time, sleep
from contextlib import closing
@ -319,7 +321,9 @@ def _load_encryption(proxy_conf_file, swift_conf_file, **kwargs):
"proxy-logging proxy-server",
"keymaster encryption proxy-logging proxy-server")
conf.set(section, 'pipeline', pipeline)
root_secret = os.urandom(32).encode("base64")
root_secret = base64.b64encode(os.urandom(32))
if not six.PY2:
root_secret = root_secret.decode('ascii')
conf.set('filter:keymaster', 'encryption_root_secret', root_secret)
except NoSectionError as err:
msg = 'Error problem with proxy conf file %s: %s' % \

8
test/functional/s3api/s3_test_client.py
View File

@ -81,7 +81,7 @@ class Connection(object):
break
for bucket in buckets:
if not isinstance(bucket.name, six.binary_type):
if six.PY2 and not isinstance(bucket.name, bytes):
bucket.name = bucket.name.encode('utf-8')
try:
@ -103,7 +103,7 @@ class Connection(object):
exceptions.insert(0, 'Too many errors to continue:')
raise Exception('\n========\n'.join(exceptions))
def make_request(self, method, bucket='', obj='', headers=None, body='',
def make_request(self, method, bucket='', obj='', headers=None, body=b'',
query=None):
"""
Wrapper method of S3Connection.make_request.
@ -123,7 +123,9 @@ class Connection(object):
query_args=query, sender=None,
override_num_retries=RETRY_COUNT,
retry_handler=None)
return response.status, dict(response.getheaders()), response.read()
return (response.status,
{h.lower(): v for h, v in response.getheaders()},
response.read())
def generate_url_and_headers(self, method, bucket='', obj='',
expires_in=3600):

3
test/functional/s3api/test_acl.py
View File

@ -40,7 +40,8 @@ class TestS3Acl(S3ApiBase):
raise tf.SkipTest(
'TestS3Acl requires s3_access_key3 and s3_secret_key3 '
'configured for reduced-access user')
self.conn.make_request('PUT', self.bucket)
status, headers, body = self.conn.make_request('PUT', self.bucket)
self.assertEqual(status, 200, body)
access_key3 = tf.config['s3_access_key3']
secret_key3 = tf.config['s3_secret_key3']
self.conn3 = Connection(access_key3, secret_key3, access_key3)

61
test/functional/s3api/test_multi_upload.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
import base64
import binascii
import unittest2
import os
import boto
@ -23,7 +24,7 @@ import boto
from distutils.version import StrictVersion
from hashlib import md5
from itertools import izip, izip_longest
from six.moves import zip, zip_longest
import test.functional as tf
from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
@ -67,7 +68,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
headers = [None] * len(keys)
self.conn.make_request('PUT', bucket)
query = 'uploads'
for key, key_headers in izip_longest(keys, headers):
for key, key_headers in zip_longest(keys, headers):
for i in range(trials):
status, resp_headers, body = \
self.conn.make_request('POST', bucket, key,
@ -76,7 +77,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
def _upload_part(self, bucket, key, upload_id, content=None, part_num=1):
query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id)
content = content if content else 'a' * self.min_segment_size
content = content if content else b'a' * self.min_segment_size
status, headers, body = \
self.conn.make_request('PUT', bucket, key, body=content,
query=query)
@ -108,8 +109,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
def test_object_multi_upload(self):
bucket = 'bucket'
keys = ['obj1', 'obj2', 'obj3']
bad_content_md5 = base64.b64encode(b'a' * 16).strip().decode('ascii')
headers = [None,
{'Content-MD5': base64.b64encode('a' * 16).strip()},
{'Content-MD5': bad_content_md5},
{'Etag': 'nonsense'}]
uploads = []
@ -118,20 +120,20 @@ class TestS3ApiMultiUpload(S3ApiBase):
# Initiate Multipart Upload
for expected_key, (status, headers, body) in \
izip(keys, results_generator):
self.assertEqual(status, 200)
zip(keys, results_generator):
self.assertEqual(status, 200, body)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertIn('content-type', headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertIn('content-length', headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'InitiateMultipartUploadResult')
self.assertEqual(elem.find('Bucket').text, bucket)
key = elem.find('Key').text
self.assertEqual(expected_key, key)
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None)
self.assertTrue((key, upload_id) not in uploads)
self.assertIsNotNone(upload_id)
self.assertNotIn((key, upload_id), uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
@ -157,7 +159,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
self.assertEqual(elem.find('IsTruncated').text, 'false')
self.assertEqual(len(elem.findall('Upload')), 3)
for (expected_key, expected_upload_id), u in \
izip(uploads, elem.findall('Upload')):
zip(uploads, elem.findall('Upload')):
key = u.find('Key').text
upload_id = u.find('UploadId').text
self.assertEqual(expected_key, key)
@ -174,7 +176,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
# Upload Part
key, upload_id = uploads[0]
content = 'a' * self.min_segment_size
content = b'a' * self.min_segment_size
etag = md5(content).hexdigest()
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
@ -190,7 +192,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
key, upload_id = uploads[1]
src_bucket = 'bucket2'
src_obj = 'obj3'
src_content = 'b' * self.min_segment_size
src_content = b'b' * self.min_segment_size
etag = md5(src_content).hexdigest()
# prepare src obj
@ -266,7 +268,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
# etags will be used to generate xml for Complete Multipart Upload
etags = []
for (expected_etag, expected_date), p in \
izip(expected_parts_list, elem.findall('Part')):
zip(expected_parts_list, elem.findall('Part')):
last_modified = p.find('LastModified').text
self.assertTrue(last_modified is not None)
# TODO: sanity check
@ -295,9 +297,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
else:
self.assertIn('transfer-encoding', headers)
self.assertEqual(headers['transfer-encoding'], 'chunked')
lines = body.split('\n')
self.assertTrue(lines[0].startswith('<?xml'), body)
self.assertTrue(lines[0].endswith('?>'), body)
lines = body.split(b'\n')
self.assertTrue(lines[0].startswith(b'<?xml'), body)
self.assertTrue(lines[0].endswith(b'?>'), body)
elem = fromstring(body, 'CompleteMultipartUploadResult')
# TODO: use tf.config value
self.assertEqual(
@ -305,9 +307,10 @@ class TestS3ApiMultiUpload(S3ApiBase):
elem.find('Location').text)
self.assertEqual(elem.find('Bucket').text, bucket)
self.assertEqual(elem.find('Key').text, key)
concatted_etags = ''.join(etag.strip('"') for etag in etags)
concatted_etags = b''.join(
etag.strip('"').encode('ascii') for etag in etags)
exp_etag = '"%s-%s"' % (
md5(concatted_etags.decode('hex')).hexdigest(), len(etags))
md5(binascii.unhexlify(concatted_etags)).hexdigest(), len(etags))
etag = elem.find('ETag').text
self.assertEqual(etag, exp_etag)
@ -332,7 +335,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
last_modified = elem.find('LastModified').text
self.assertIsNotNone(last_modified)
exp_content = 'a' * self.min_segment_size
exp_content = b'a' * self.min_segment_size
etag = md5(exp_content).hexdigest()
self.assertEqual(resp_etag, etag)
@ -723,7 +726,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query,
body='A' * body_size[i])
body=b'A' * body_size[i])
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
@ -747,7 +750,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query,
body='A' * body_size[i])
body=b'A' * body_size[i])
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
@ -770,9 +773,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
etags = []
for i in range(1, 4):
query = 'partNumber=%s&uploadId=%s' % (2 * i - 1, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key,
body='A' * 1024 * 1024 * 5, query=query)
status, headers, body = self.conn.make_request(
'PUT', bucket, key, body=b'A' * 1024 * 1024 * 5,
query=query)
etags.append(headers['etag'])
query = 'uploadId=%s' % upload_id
xml = self._gen_comp_xml(etags[:-1], step=2)
@ -791,7 +794,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
# Initiate Multipart Upload
for expected_key, (status, headers, body) in \
izip(keys, results_generator):
zip(keys, results_generator):
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
@ -813,7 +816,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
key, upload_id = uploads[0]
src_bucket = 'bucket2'
src_obj = 'obj4'
src_content = 'y' * (self.min_segment_size / 2) + 'z' * \
src_content = b'y' * (self.min_segment_size // 2) + b'z' * \
self.min_segment_size
src_range = 'bytes=0-%d' % (self.min_segment_size - 1)
etag = md5(src_content[:self.min_segment_size]).hexdigest()
@ -901,7 +904,7 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
# Initiate Multipart Upload
for expected_key, (status, _, body) in \
izip(keys, results_generator):
zip(keys, results_generator):
self.assertEqual(status, 200) # sanity
elem = fromstring(body, 'InitiateMultipartUploadResult')
key = elem.find('Key').text
@ -915,7 +918,7 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
# Upload Part
key, upload_id = uploads[0]
content = 'a' * self.min_segment_size
content = b'a' * self.min_segment_size
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
self.assertEqual(status, 200)

52
test/functional/s3api/test_object.py
View File

@ -25,7 +25,8 @@ import email.parser
from email.utils import formatdate, parsedate
from time import mktime
from hashlib import md5
from urllib import quote
import six
from six.moves.urllib.parse import quote
import test.functional as tf
@ -59,7 +60,7 @@ class TestS3ApiObject(S3ApiBase):
def test_object(self):
obj = 'object name with %-sign'
content = 'abc123'
content = b'abc123'
etag = md5(content).hexdigest()
# PUT Object
@ -219,19 +220,19 @@ class TestS3ApiObject(S3ApiBase):
status, headers, body = \
auth_error_conn.make_request('HEAD', self.bucket, obj)
self.assertEqual(status, 403)
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
self.assertEqual(headers['content-type'], 'application/xml')
status, headers, body = \
self.conn.make_request('HEAD', self.bucket, 'invalid')
self.assertEqual(status, 404)
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
self.assertEqual(headers['content-type'], 'application/xml')
status, headers, body = \
self.conn.make_request('HEAD', 'invalid', obj)
self.assertEqual(status, 404)
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
self.assertEqual(headers['content-type'], 'application/xml')
def test_delete_object_error(self):
@ -265,7 +266,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_content_md5(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'Content-MD5': calculate_md5(content)}
status, headers, body = \
@ -276,7 +277,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_content_type(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'Content-Type': 'text/plain'}
status, headers, body = \
@ -290,7 +291,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_conditional_requests(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
headers = {'If-None-Match': '*'}
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, headers, content)
@ -318,7 +319,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_expect(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'Expect': '100-continue'}
status, headers, body = \
@ -331,7 +332,7 @@ class TestS3ApiObject(S3ApiBase):
if expected_headers is None:
expected_headers = req_headers
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj,
@ -387,7 +388,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_storage_class(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'X-Amz-Storage-Class': 'STANDARD'}
status, headers, body = \
@ -399,7 +400,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_copy_source_params(self):
obj = 'object'
src_headers = {'X-Amz-Meta-Test': 'src'}
src_body = 'some content'
src_body = b'some content'
dst_bucket = 'dst-bucket'
dst_obj = 'dst_object'
self.conn.make_request('PUT', self.bucket, obj, src_headers, src_body)
@ -433,7 +434,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_copy_source(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
self.conn.make_request('PUT', self.bucket, obj, body=content)
@ -648,7 +649,7 @@ class TestS3ApiObject(S3ApiBase):
def test_get_object_range(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
headers = {'x-amz-meta-test': 'swift'}
self.conn.make_request(
'PUT', self.bucket, obj, headers=headers, body=content)
@ -662,7 +663,7 @@ class TestS3ApiObject(S3ApiBase):
self.assertEqual(headers['content-length'], '5')
self.assertTrue('x-amz-meta-test' in headers)
self.assertEqual('swift', headers['x-amz-meta-test'])
self.assertEqual(body, 'bcdef')
self.assertEqual(body, b'bcdef')
headers = {'Range': 'bytes=5-'}
status, headers, body = \
@ -673,7 +674,7 @@ class TestS3ApiObject(S3ApiBase):
self.assertEqual(headers['content-length'], '5')
self.assertTrue('x-amz-meta-test' in headers)
self.assertEqual('swift', headers['x-amz-meta-test'])
self.assertEqual(body, 'fghij')
self.assertEqual(body, b'fghij')
headers = {'Range': 'bytes=-5'}
status, headers, body = \
@ -684,7 +685,7 @@ class TestS3ApiObject(S3ApiBase):
self.assertEqual(headers['content-length'], '5')
self.assertTrue('x-amz-meta-test' in headers)
self.assertEqual('swift', headers['x-amz-meta-test'])
self.assertEqual(body, 'fghij')
self.assertEqual(body, b'fghij')
ranges = ['1-2', '4-5']
@ -693,9 +694,9 @@ class TestS3ApiObject(S3ApiBase):
self.conn.make_request('GET', self.bucket, obj, headers=headers)
self.assertEqual(status, 206)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-length' in headers)
self.assertIn('content-length', headers)
self.assertTrue('content-type' in headers) # sanity
self.assertIn('content-type', headers) # sanity
content_type, boundary = headers['content-type'].split(';')
self.assertEqual('multipart/byteranges', content_type)
@ -704,10 +705,13 @@ class TestS3ApiObject(S3ApiBase):
# TODO: Using swift.common.utils.multipart_byteranges_to_document_iters
# could be easy enough.
parser = email.parser.FeedParser()
if six.PY2:
parser = email.parser.FeedParser()
else:
parser = email.parser.BytesFeedParser()
parser.feed(
"Content-Type: multipart/byterange; boundary=%s\r\n\r\n" %
boundary_str)
b"Content-Type: multipart/byterange; boundary=%s\r\n\r\n" %
boundary_str.encode('ascii'))
parser.feed(body)
message = parser.close()
@ -727,7 +731,7 @@ class TestS3ApiObject(S3ApiBase):
self.assertEqual(
expected_range, part.get('Content-Range'))
# rest
payload = part.get_payload().strip()
payload = part.get_payload(decode=True).strip()
self.assertEqual(content[start:end + 1], payload)
def test_get_object_if_modified_since(self):
@ -783,7 +787,7 @@ class TestS3ApiObject(S3ApiBase):
def test_head_object_range(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
self.conn.make_request('PUT', self.bucket, obj, body=content)
headers = {'Range': 'bytes=1-5'}

8
test/functional/s3api/test_presigned.py
View File

@ -190,7 +190,7 @@ class TestS3ApiPresignedUrls(S3ApiBase):
# PUT empty object
put_url, headers = self.conn.generate_url_and_headers(
'PUT', bucket, obj)
resp = requests.put(put_url, data='', headers=headers)
resp = requests.put(put_url, data=b'', headers=headers)
self.assertEqual(resp.status_code, 200,
'Got %d %s' % (resp.status_code, resp.content))
# GET empty object
@ -199,10 +199,10 @@ class TestS3ApiPresignedUrls(S3ApiBase):
resp = requests.get(get_url, headers=headers)
self.assertEqual(resp.status_code, 200,
'Got %d %s' % (resp.status_code, resp.content))
self.assertEqual(resp.content, '')
self.assertEqual(resp.content, b'')
# PUT over object
resp = requests.put(put_url, data='foobar', headers=headers)
resp = requests.put(put_url, data=b'foobar', headers=headers)
self.assertEqual(resp.status_code, 200,
'Got %d %s' % (resp.status_code, resp.content))
@ -210,7 +210,7 @@ class TestS3ApiPresignedUrls(S3ApiBase):
resp = requests.get(get_url, headers=headers)
self.assertEqual(resp.status_code, 200,
'Got %d %s' % (resp.status_code, resp.content))
self.assertEqual(resp.content, 'foobar')
self.assertEqual(resp.content, b'foobar')
# DELETE Object
delete_url, headers = self.conn.generate_url_and_headers(

4
test/functional/s3api/test_service.py
View File

@ -80,8 +80,8 @@ class TestS3ApiService(S3ApiBase):
'GET', headers={'Date': '', 'x-amz-date': ''})
self.assertEqual(status, 403)
self.assertEqual(get_error_code(body), 'AccessDenied')
self.assertIn('AWS authentication requires a valid Date '
'or x-amz-date header', body)
self.assertIn(b'AWS authentication requires a valid Date '
b'or x-amz-date header', body)
class TestS3ApiServiceSigV4(TestS3ApiService):

3
test/functional/s3api/utils.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from base64 import b64encode
from hashlib import md5
from swift.common.middleware.s3api.etree import fromstring
@ -28,4 +29,4 @@ def get_error_msg(body):
def calculate_md5(body):
return md5(body).digest().encode('base64').strip()
return b64encode(md5(body).digest()).strip().decode('ascii')

2
test/functional/test_access_control.py
View File

@ -15,7 +15,7 @@
# limitations under the License.
import unittest
from urlparse import urlparse, urlunparse
from six.moves.urllib.parse import urlparse, urlunparse
import uuid
from random import shuffle

23
test/functional/test_account.py
View File

@ -20,8 +20,10 @@ import json
from uuid import uuid4
from string import ascii_letters
import six
from six.moves import range
from swift.common.middleware.acl import format_acl
from swift.common.utils import distribute_evenly
from test.functional import check_response, retry, requires_acls, \
load_constraint, SkipTest
@ -57,8 +59,8 @@ class TestAccount(unittest2.TestCase):
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
for i in range(0, len(remove_metadata_keys), 90):
batch = remove_metadata_keys[i:i + 90]
buckets = (len(remove_metadata_keys) - 1) // 90 + 1
for batch in distribute_evenly(remove_metadata_keys, buckets):
resp = retry(post, batch)
resp.read()
@ -148,7 +150,7 @@ class TestAccount(unittest2.TestCase):
# needs to be an acceptable header size
num_keys = 8
max_key_size = load_constraint('max_header_size') / num_keys
max_key_size = load_constraint('max_header_size') // num_keys
acl = {'admin': [c * max_key_size for c in ascii_letters[:num_keys]]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
@ -717,7 +719,9 @@ class TestAccount(unittest2.TestCase):
return check_response(conn)
uni_key = u'X-Account-Meta-uni\u0E12'
uni_value = u'uni\u0E12'
if (tf.web_front_end == 'integral'):
# Note that py3 has issues with non-ascii header names; see
# https://bugs.python.org/issue37093
if (tf.web_front_end == 'integral' and six.PY2):
resp = retry(post, uni_key, '1')
resp.read()
self.assertIn(resp.status, (201, 204))
@ -731,9 +735,14 @@ class TestAccount(unittest2.TestCase):
resp = retry(head)
resp.read()
self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('X-Account-Meta-uni'),
uni_value.encode('utf-8'))
if (tf.web_front_end == 'integral'):
if six.PY2:
self.assertEqual(resp.getheader('X-Account-Meta-uni'),
uni_value.encode('utf8'))
else:
self.assertEqual(resp.getheader('X-Account-Meta-uni'),
uni_value)
# See above note about py3 and non-ascii header names
if (tf.web_front_end == 'integral' and six.PY2):
resp = retry(post, uni_key, uni_value)
resp.read()
self.assertEqual(resp.status, 204)

47
test/functional/test_container.py
View File

@ -23,6 +23,7 @@ from test.functional import check_response, cluster_info, retry, \
requires_acls, load_constraint, requires_policies, SkipTest
import test.functional as tf
import six
from six.moves import range
@ -71,8 +72,11 @@ class TestContainer(unittest2.TestCase):
return check_response(conn)
def delete(url, token, parsed, conn, container, obj):
path = '/'.join([parsed.path, container,
obj['name'].encode('utf8')])
if six.PY2:
obj_name = obj['name'].encode('utf8')
else:
obj_name = obj['name']
path = '/'.join([parsed.path, container, obj_name])
conn.request('DELETE', path, '', {'X-Auth-Token': token})
return check_response(conn)
@ -153,7 +157,9 @@ class TestContainer(unittest2.TestCase):
uni_key = u'X-Container-Meta-uni\u0E12'
uni_value = u'uni\u0E12'
if (tf.web_front_end == 'integral'):
# Note that py3 has issues with non-ascii header names; see
# https://bugs.python.org/issue37093
if (tf.web_front_end == 'integral' and six.PY2):
resp = retry(post, uni_key, '1')
resp.read()
self.assertEqual(resp.status, 204)
@ -167,9 +173,14 @@ class TestContainer(unittest2.TestCase):
resp = retry(head)
resp.read()
self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('X-Container-Meta-uni'),
uni_value.encode('utf-8'))
if (tf.web_front_end == 'integral'):
if six.PY2:
self.assertEqual(resp.getheader('X-Container-Meta-uni'),
uni_value.encode('utf-8'))
else:
self.assertEqual(resp.getheader('X-Container-Meta-uni'),
uni_value)
# See above note about py3 and non-ascii header names
if (tf.web_front_end == 'integral' and six.PY2):
resp = retry(post, uni_key, uni_value)
resp.read()
self.assertEqual(resp.status, 204)
@ -763,6 +774,8 @@ class TestContainer(unittest2.TestCase):
# read-only can list containers
resp = retry(get, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(self.name, listing)
@ -778,6 +791,8 @@ class TestContainer(unittest2.TestCase):
self.assertEqual(resp.status, 201)
resp = retry(get, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(new_container_name, listing)
@ -878,6 +893,8 @@ class TestContainer(unittest2.TestCase):
# can list containers
resp = retry(get, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(self.name, listing)
@ -888,6 +905,8 @@ class TestContainer(unittest2.TestCase):
self.assertIn(resp.status, (201, 202))
resp = retry(get, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(new_container_name, listing)
@ -897,6 +916,8 @@ class TestContainer(unittest2.TestCase):
self.assertIn(resp.status, (204, 404))
resp = retry(get, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertNotIn(new_container_name, listing)
@ -1020,6 +1041,8 @@ class TestContainer(unittest2.TestCase):
# can list containers
resp = retry(get, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(self.name, listing)
@ -1030,6 +1053,8 @@ class TestContainer(unittest2.TestCase):
self.assertEqual(resp.status, 201)
resp = retry(get, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(new_container_name, listing)
@ -1039,6 +1064,8 @@ class TestContainer(unittest2.TestCase):
self.assertEqual(resp.status, 204)
resp = retry(get, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertNotIn(new_container_name, listing)
@ -1408,7 +1435,7 @@ class TestContainer(unittest2.TestCase):
if (tf.web_front_end == 'apache2'):
self.assertEqual(resp.status, 404)
else:
self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL')
self.assertEqual(resp.read(), b'Invalid UTF8 or contains NULL')
self.assertEqual(resp.status, 412)
def test_create_container_gets_default_policy_by_default(self):
@ -1604,12 +1631,12 @@ class TestContainer(unittest2.TestCase):
return check_response(conn)
# upload 11 bytes object
resp = retry(put, '01234567890')
resp = retry(put, b'01234567890')
resp.read()
self.assertEqual(resp.status, 413)
# upload 10 bytes object
resp = retry(put, '0123456789')
resp = retry(put, b'0123456789')
resp.read()
self.assertEqual(resp.status, 201)
@ -1622,7 +1649,7 @@ class TestContainer(unittest2.TestCase):
resp = retry(get)
body = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(body, '0123456789')
self.assertEqual(body, b'0123456789')
class BaseTestContainerACLs(unittest2.TestCase):

87
test/functional/test_dlo.py
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from swift.common.swob import str_to_wsgi
import test.functional as tf
from test.functional.tests import Utils, Base, Base2, BaseEnv
from test.functional.swift_test_client import Connection, ResponseError
@ -86,25 +86,25 @@ class TestDlo(Base):
file_contents = file_item.read()
self.assertEqual(
file_contents,
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
file_item = self.env.container.file('man2')
file_contents = file_item.read()
self.assertEqual(
file_contents,
"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")
b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")
file_item = self.env.container.file('manall')
file_contents = file_item.read()
self.assertEqual(
file_contents,
("aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" +
"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE"))
(b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" +
b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE"))
def test_get_manifest_document_itself(self):
file_item = self.env.container.file('man1')
file_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(file_contents, "man1-contents")
self.assertEqual(file_contents, b"man1-contents")
self.assertEqual(file_item.info()['x_object_manifest'],
"%s/%s/seg_lower" %
(self.env.container.name, self.env.segment_prefix))
@ -112,10 +112,19 @@ class TestDlo(Base):
def test_get_range(self):
file_item = self.env.container.file('man1')
file_contents = file_item.read(size=25, offset=8)
self.assertEqual(file_contents, "aabbbbbbbbbbccccccccccddd")
self.assertEqual(file_contents, b"aabbbbbbbbbbccccccccccddd")
file_contents = file_item.read(size=1, offset=47)
self.assertEqual(file_contents, "e")
self.assertEqual(file_contents, b"e")
def test_get_multiple_ranges(self):
file_item = self.env.container.file('man1')
file_contents = file_item.read(
hdrs={'Range': 'bytes=0-4,10-14'})
self.assert_status(200) # *not* 206
self.assertEqual(
file_contents,
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
def test_get_range_out_of_range(self):
file_item = self.env.container.file('man1')
@ -130,7 +139,7 @@ class TestDlo(Base):
# segments and not just a manifest.
f_segment = self.env.container.file("%s/seg_lowerf" %
(self.env.segment_prefix))
f_segment.write('ffffffffff')
f_segment.write(b'ffffffffff')
try:
man1_item = self.env.container.file('man1')
man1_item.copy(self.env.container.name, "copied-man1")
@ -142,7 +151,7 @@ class TestDlo(Base):
file_contents = file_item.read()
self.assertEqual(
file_contents,
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
# The copied object must not have X-Object-Manifest
self.assertNotIn("x_object_manifest", file_item.info())
@ -154,7 +163,7 @@ class TestDlo(Base):
# segments and not just a manifest.
f_segment = self.env.container.file("%s/seg_lowerf" %
(self.env.segment_prefix))
f_segment.write('ffffffffff')
f_segment.write(b'ffffffffff')
try:
man1_item = self.env.container.file('man1')
man1_item.copy_account(acct,
@ -168,7 +177,7 @@ class TestDlo(Base):
file_contents = file_item.read()
self.assertEqual(
file_contents,
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
# The copied object must not have X-Object-Manifest
self.assertNotIn("x_object_manifest", file_item.info())
@ -182,12 +191,12 @@ class TestDlo(Base):
copied = self.env.container.file("copied-man1")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
self.assertEqual(copied_contents, "man1-contents")
self.assertEqual(copied_contents, b"man1-contents")
copied_contents = copied.read()
self.assertEqual(
copied_contents,
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
self.assertEqual(man1_item.info()['x_object_manifest'],
copied.info()['x_object_manifest'])
finally:
@ -239,8 +248,9 @@ class TestDlo(Base):
manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
self.assert_status(200)
@unittest.skipIf('username3' not in tf.config, "Requires user 3")
def test_dlo_referer_on_segment_container(self):
if 'username3' not in tf.config:
raise tf.SkipTest('Requires user 3')
# First the account2 (test3) should fail
config2 = tf.config.copy()
config2['username'] = tf.config['username3']
@ -268,7 +278,7 @@ class TestDlo(Base):
contents = dlo_file.read(hdrs=headers)
self.assertEqual(
contents,
"ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj")
b"ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj")
def test_dlo_post_with_manifest_header(self):
# verify that performing a POST to a DLO manifest
@ -278,12 +288,13 @@ class TestDlo(Base):
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
file_item = self.env.container.file(Utils.create_name())
file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})
file_item.write(b'manifest-contents',
hdrs={"X-Object-Manifest": x_o_m})
# sanity checks
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde'])
self.assertEqual(b'manifest-contents', manifest_contents)
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
@ -294,16 +305,18 @@ class TestDlo(Base):
# verify that x-object-manifest was updated
file_item.info()
resp_headers = file_item.conn.response.getheaders()
self.assertIn(('x-object-manifest', new_x_o_m), resp_headers)
resp_headers = [(h.lower(), v)
for h, v in file_item.conn.response.getheaders()]
self.assertIn(('x-object-manifest', str_to_wsgi(new_x_o_m)),
resp_headers)
self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
self.assertEqual(b'manifest-contents', manifest_contents)
# verify that updated manifest points to new content
expected_contents = ''.join([(c * 10) for c in 'ABCDE'])
expected_contents = ''.join((c * 10) for c in 'ABCDE').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
@ -319,10 +332,10 @@ class TestDlo(Base):
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
self.assertEqual(b'manifest-contents', manifest_contents)
# verify that updated manifest points new content
expected_contents = ''.join([(c * 10) for c in 'abcde'])
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
@ -334,12 +347,13 @@ class TestDlo(Base):
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
file_item = self.env.container.file(Utils.create_name())
file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})
file_item.write(b'manifest-contents',
hdrs={"X-Object-Manifest": x_o_m})
# sanity checks
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde'])
self.assertEqual(b'manifest-contents', manifest_contents)
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
@ -352,11 +366,11 @@ class TestDlo(Base):
# verify that object content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
self.assertEqual(b'manifest-contents', manifest_contents)
# verify that object is no longer a manifest
contents = file_item.read(parms={})
self.assertEqual('manifest-contents', contents)
self.assertEqual(b'manifest-contents', contents)
def test_dlo_post_with_manifest_regular_object(self):
# verify that performing a POST to a regular object
@ -364,11 +378,11 @@ class TestDlo(Base):
# Put a regular object
file_item = self.env.container.file(Utils.create_name())
file_item.write('file contents', hdrs={})
file_item.write(b'file contents', hdrs={})
# sanity checks
file_contents = file_item.read(parms={})
self.assertEqual('file contents', file_contents)
self.assertEqual(b'file contents', file_contents)
# get the path associated with man1
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
@ -378,13 +392,14 @@ class TestDlo(Base):
# verify that the file is now a manifest
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('file contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde'])
self.assertEqual(b'file contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde']).encode()
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
file_item.info()
resp_headers = file_item.conn.response.getheaders()
self.assertIn(('x-object-manifest', x_o_m), resp_headers)
resp_headers = [(h.lower(), v)
for h, v in file_item.conn.response.getheaders()]
self.assertIn(('x-object-manifest', str_to_wsgi(x_o_m)), resp_headers)
class TestDloUTF8(Base2, TestDlo):

100
test/functional/test_object.py
View File

@ -22,6 +22,7 @@ from uuid import uuid4
import time
from xml.dom import minidom
import six
from six.moves import range
from test.functional import check_response, retry, requires_acls, \
@ -105,8 +106,11 @@ class TestObject(unittest2.TestCase):
# delete an object
def delete(url, token, parsed, conn, container, obj):
path = '/'.join([parsed.path, container,
obj['name'].encode('utf8')])
if six.PY2:
obj_name = obj['name'].encode('utf8')
else:
obj_name = obj['name']
path = '/'.join([parsed.path, container, obj_name])
conn.request('DELETE', path, '', {'X-Auth-Token': token})
return check_response(conn)
@ -176,7 +180,7 @@ class TestObject(unittest2.TestCase):
resp.read()
self.assertEqual(resp.status, 201)
resp = retry(get)
self.assertEqual('', resp.read())
self.assertEqual(b'', resp.read())
self.assertEqual(resp.status, 200)
self.assertEqual(metadata(resp), {})
# empty post
@ -184,7 +188,7 @@ class TestObject(unittest2.TestCase):
resp.read()
self.assertEqual(resp.status, 202)
resp = retry(get)
self.assertEqual('', resp.read())
self.assertEqual(b'', resp.read())
self.assertEqual(resp.status, 200)
self.assertEqual(metadata(resp), {})
@ -197,7 +201,7 @@ class TestObject(unittest2.TestCase):
resp.read()
self.assertEqual(resp.status, 201)
resp = retry(get)
self.assertEqual('', resp.read())
self.assertEqual(b'', resp.read())
self.assertEqual(resp.status, 200)
self.assertEqual(metadata(resp), {
'X-Object-Meta-Color': 'blUe',
@ -209,7 +213,7 @@ class TestObject(unittest2.TestCase):
resp.read()
self.assertEqual(resp.status, 202)
resp = retry(get)
self.assertEqual('', resp.read())
self.assertEqual(b'', resp.read())
self.assertEqual(resp.status, 200)
self.assertEqual(metadata(resp), {
'X-Object-Meta-Color': 'oraNge'
@ -225,7 +229,7 @@ class TestObject(unittest2.TestCase):
resp.read()
self.assertEqual(resp.status, 201)
resp = retry(get)
self.assertEqual('', resp.read())
self.assertEqual(b'', resp.read())
self.assertEqual(resp.status, 200)
self.assertEqual(metadata(resp), {
'X-Object-Meta-Color': 'Red',
@ -241,7 +245,7 @@ class TestObject(unittest2.TestCase):
resp.read()
self.assertEqual(resp.status, 202)
resp = retry(get)
self.assertEqual('', resp.read())
self.assertEqual(b'', resp.read())
self.assertEqual(resp.status, 200)
self.assertEqual(metadata(resp), {
'X-Object-Meta-Food': 'Burger',
@ -256,7 +260,7 @@ class TestObject(unittest2.TestCase):
resp.read()
self.assertEqual(resp.status, 201)
resp = retry(get)
self.assertEqual('', resp.read())
self.assertEqual(b'', resp.read())
self.assertEqual(resp.status, 200)
self.assertEqual(metadata(resp), {
'X-Object-Meta-Foo': 'B\xc3\xa2r',
@ -269,7 +273,7 @@ class TestObject(unittest2.TestCase):
resp.read()
self.assertEqual(resp.status, 202)
resp = retry(get)
self.assertEqual('', resp.read())
self.assertEqual(b'', resp.read())
self.assertEqual(resp.status, 200)
self.assertEqual(metadata(resp), {
'X-Object-Meta-Foo': 'B\xc3\xa5z',
@ -341,7 +345,7 @@ class TestObject(unittest2.TestCase):
'X-Timestamp should be a UNIX timestamp float value', body)
else:
self.assertEqual(resp.status, 201)
self.assertEqual(body, '')
self.assertEqual(body, b'')
resp = retry(head)
resp.read()
self.assertGreater(float(resp.headers['x-timestamp']), ts_before)
@ -374,7 +378,7 @@ class TestObject(unittest2.TestCase):
'X-Timestamp should be a UNIX timestamp float value', body)
else:
self.assertEqual(resp.status, 201)
self.assertEqual(body, '')
self.assertEqual(body, b'')
resp = retry(head)
resp.read()
self.assertGreater(float(resp.headers['x-timestamp']), ts_before)
@ -470,7 +474,7 @@ class TestObject(unittest2.TestCase):
resp = retry(put)
body = resp.read()
self.assertEqual(resp.status, 400)
self.assertEqual(body, 'Non-integer X-Delete-After')
self.assertEqual(body, b'Non-integer X-Delete-After')
def test_non_integer_x_delete_at(self):
def put(url, token, parsed, conn):
@ -483,7 +487,7 @@ class TestObject(unittest2.TestCase):
resp = retry(put)
body = resp.read()
self.assertEqual(resp.status, 400)
self.assertEqual(body, 'Non-integer X-Delete-At')
self.assertEqual(body, b'Non-integer X-Delete-At')
def test_x_delete_at_in_the_past(self):
def put(url, token, parsed, conn):
@ -496,7 +500,7 @@ class TestObject(unittest2.TestCase):
resp = retry(put)
body = resp.read()
self.assertEqual(resp.status, 400)
self.assertEqual(body, 'X-Delete-At in past')
self.assertEqual(body, b'X-Delete-At in past')
def test_copy_object(self):
if tf.skip:
@ -514,7 +518,7 @@ class TestObject(unittest2.TestCase):
resp = retry(get_source)
source_contents = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(source_contents, 'test')
self.assertEqual(source_contents, b'test')
# copy source to dest with X-Copy-From
def put(url, token, parsed, conn):
@ -605,7 +609,7 @@ class TestObject(unittest2.TestCase):
resp = retry(get_source)
source_contents = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(source_contents, 'test')
self.assertEqual(source_contents, b'test')
acct = tf.parsed[0].path.split('/', 2)[2]
@ -964,14 +968,16 @@ class TestObject(unittest2.TestCase):
# can list objects
resp = retry(get_listing, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(self.obj, listing)
self.assertIn(self.obj, listing.split('\n'))
# can get object
resp = retry(get, self.obj, use_account=3)
body = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(body, 'test')
self.assertEqual(body, b'test')
# can not put an object
obj_name = str(uuid4())
@ -987,9 +993,11 @@ class TestObject(unittest2.TestCase):
# sanity with account1
resp = retry(get_listing, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertNotIn(obj_name, listing)
self.assertIn(self.obj, listing)
self.assertNotIn(obj_name, listing.split('\n'))
self.assertIn(self.obj, listing.split('\n'))
@requires_acls
def test_read_write(self):
@ -1045,14 +1053,16 @@ class TestObject(unittest2.TestCase):
# can list objects
resp = retry(get_listing, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(self.obj, listing)
self.assertIn(self.obj, listing.split('\n'))
# can get object
resp = retry(get, self.obj, use_account=3)
body = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(body, 'test')
self.assertEqual(body, b'test')
# can put an object
obj_name = str(uuid4())
@ -1068,9 +1078,11 @@ class TestObject(unittest2.TestCase):
# sanity with account1
resp = retry(get_listing, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(obj_name, listing)
self.assertNotIn(self.obj, listing)
self.assertIn(obj_name, listing.split('\n'))
self.assertNotIn(self.obj, listing.split('\n'))
@requires_acls
def test_admin(self):
@ -1126,14 +1138,16 @@ class TestObject(unittest2.TestCase):
# can list objects
resp = retry(get_listing, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(self.obj, listing)
self.assertIn(self.obj, listing.split('\n'))
# can get object
resp = retry(get, self.obj, use_account=3)
body = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(body, 'test')
self.assertEqual(body, b'test')
# can put an object
obj_name = str(uuid4())
@ -1149,17 +1163,19 @@ class TestObject(unittest2.TestCase):
# sanity with account1
resp = retry(get_listing, use_account=3)
listing = resp.read()
if not six.PY2:
listing = listing.decode('utf8')
self.assertEqual(resp.status, 200)
self.assertIn(obj_name, listing)
self.assertIn(obj_name, listing.split('\n'))
self.assertNotIn(self.obj, listing)
def test_manifest(self):
if tf.skip:
raise SkipTest
# Data for the object segments
segments1 = ['one', 'two', 'three', 'four', 'five']
segments2 = ['six', 'seven', 'eight']
segments3 = ['nine', 'ten', 'eleven']
segments1 = [b'one', b'two', b'three', b'four', b'five']
segments2 = [b'six', b'seven', b'eight']
segments3 = [b'nine', b'ten', b'eleven']
# Upload the first set of segments
def put(url, token, parsed, conn, objnum):
@ -1190,7 +1206,7 @@ class TestObject(unittest2.TestCase):
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get)
self.assertEqual(resp.read(), ''.join(segments1))
self.assertEqual(resp.read(), b''.join(segments1))
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('content-type'), 'text/jibberish')
@ -1201,7 +1217,7 @@ class TestObject(unittest2.TestCase):
'X-Auth-Token': token, 'Range': 'bytes=3-'})
return check_response(conn)
resp = retry(get)
self.assertEqual(resp.read(), ''.join(segments1[1:]))
self.assertEqual(resp.read(), b''.join(segments1[1:]))
self.assertEqual(resp.status, 206)
# Get with a range in the middle of the second segment
@ -1211,7 +1227,7 @@ class TestObject(unittest2.TestCase):
'X-Auth-Token': token, 'Range': 'bytes=5-'})
return check_response(conn)
resp = retry(get)
self.assertEqual(resp.read(), ''.join(segments1)[5:])
self.assertEqual(resp.read(), b''.join(segments1)[5:])
self.assertEqual(resp.status, 206)
# Get with a full start and stop range
@ -1221,7 +1237,7 @@ class TestObject(unittest2.TestCase):
'X-Auth-Token': token, 'Range': 'bytes=5-10'})
return check_response(conn)
resp = retry(get)
self.assertEqual(resp.read(), ''.join(segments1)[5:11])
self.assertEqual(resp.read(), b''.join(segments1)[5:11])
self.assertEqual(resp.status, 206)
# Upload the second set of segments
@ -1241,7 +1257,7 @@ class TestObject(unittest2.TestCase):
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get)
self.assertEqual(resp.read(), ''.join(segments1))
self.assertEqual(resp.read(), b''.join(segments1))
self.assertEqual(resp.status, 200)
# Update the manifest
@ -1262,7 +1278,7 @@ class TestObject(unittest2.TestCase):
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get)
self.assertEqual(resp.read(), ''.join(segments2))
self.assertEqual(resp.read(), b''.join(segments2))
self.assertEqual(resp.status, 200)
if not tf.skip3:
@ -1292,7 +1308,7 @@ class TestObject(unittest2.TestCase):
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get, use_account=3)
self.assertEqual(resp.read(), ''.join(segments2))
self.assertEqual(resp.read(), b''.join(segments2))
self.assertEqual(resp.status, 200)
# Create another container for the third set of segments
@ -1335,7 +1351,7 @@ class TestObject(unittest2.TestCase):
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get)
self.assertEqual(resp.read(), ''.join(segments3))
self.assertEqual(resp.read(), b''.join(segments3))
self.assertEqual(resp.status, 200)
if not tf.skip3:
@ -1368,7 +1384,7 @@ class TestObject(unittest2.TestCase):
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get, use_account=3)
self.assertEqual(resp.read(), ''.join(segments3))
self.assertEqual(resp.read(), b''.join(segments3))
self.assertEqual(resp.status, 200)
# Delete the manifest
@ -1480,7 +1496,7 @@ class TestObject(unittest2.TestCase):
if (tf.web_front_end == 'apache2'):
self.assertEqual(resp.status, 404)
else:
self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL')
self.assertEqual(resp.read(), b'Invalid UTF8 or contains NULL')
self.assertEqual(resp.status, 412)
def test_cors(self):
@ -1645,6 +1661,8 @@ class TestObject(unittest2.TestCase):
for c, o, body in validate_requests:
resp = retry(get_obj, c, o)
self.assertEqual(resp.status, 200)
if not six.PY2:
body = body.encode('utf8')
self.assertEqual(body, resp.read())
@requires_bulk

369
test/functional/test_slo.py
View File

@ -21,6 +21,8 @@ import itertools
import json
from copy import deepcopy
import six
import test.functional as tf
from test.functional import cluster_info, SkipTest
from test.functional.tests import Utils, Base, Base2, BaseEnv
@ -35,6 +37,16 @@ def tearDownModule():
tf.teardown_package()
def group_file_contents(file_contents):
# This looks a little funny, but iterating through a byte string on py3
# yields a sequence of ints, not a sequence of single-byte byte strings
# as it did on py2.
byte_iter = (file_contents[i:i + 1] for i in range(len(file_contents)))
return [
(char, sum(1 for _ in grp))
for char, grp in itertools.groupby(byte_iter)]
class TestSloEnv(BaseEnv):
slo_enabled = None # tri-state: None initially, then True/False
@ -250,33 +262,36 @@ class TestSlo(Base):
(self.env.slo_enabled,))
manifest_abcde_hash = hashlib.md5()
manifest_abcde_hash.update(hashlib.md5('a' * 1024 * 1024).hexdigest())
manifest_abcde_hash.update(hashlib.md5('b' * 1024 * 1024).hexdigest())
manifest_abcde_hash.update(hashlib.md5('c' * 1024 * 1024).hexdigest())
manifest_abcde_hash.update(hashlib.md5('d' * 1024 * 1024).hexdigest())
manifest_abcde_hash.update(hashlib.md5('e').hexdigest())
for letter in (b'a', b'b', b'c', b'd'):
manifest_abcde_hash.update(hashlib.md5(
letter * 1024 * 1024).hexdigest().encode('ascii'))
manifest_abcde_hash.update(hashlib.md5(
b'e').hexdigest().encode('ascii'))
self.manifest_abcde_etag = manifest_abcde_hash.hexdigest()
def test_slo_get_simple_manifest(self):
file_item = self.env.container.file('manifest-abcde')
file_contents = file_item.read()
self.assertEqual(file_item.conn.response.status, 200)
headers = dict(file_item.conn.response.getheaders())
headers = dict(
(h.lower(), v)
for h, v in file_item.conn.response.getheaders())
self.assertIn('etag', headers)
self.assertEqual(headers['etag'], '"%s"' % self.manifest_abcde_etag)
self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents))
self.assertEqual('a', file_contents[0])
self.assertEqual('a', file_contents[1024 * 1024 - 1])
self.assertEqual('b', file_contents[1024 * 1024])
self.assertEqual('d', file_contents[-2])
self.assertEqual('e', file_contents[-1])
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_file_contents(file_contents))
def test_slo_container_listing(self):
# the listing object size should equal the sum of the size of the
# segments, not the size of the manifest body
file_item = self.env.container.file(Utils.create_name())
file_item.write(
json.dumps([self.env.seg_info['seg_a']]),
json.dumps([self.env.seg_info['seg_a']]).encode('ascii'),
parms={'multipart-manifest': 'put'})
# The container listing exposes BOTH the MD5 of the manifest content
# and the SLO MD5-of-MD5s by splitting the latter out into a separate
@ -346,93 +361,85 @@ class TestSlo(Base):
file_item = self.env.container.file('manifest-abcde-submanifest')
file_contents = file_item.read()
self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents))
self.assertEqual('a', file_contents[0])
self.assertEqual('a', file_contents[1024 * 1024 - 1])
self.assertEqual('b', file_contents[1024 * 1024])
self.assertEqual('d', file_contents[-2])
self.assertEqual('e', file_contents[-1])
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_file_contents(file_item.read()))
def test_slo_get_ranged_manifest(self):
file_item = self.env.container.file('ranged-manifest')
grouped_file_contents = [
(char, sum(1 for _char in grp))
for char, grp in itertools.groupby(file_item.read())]
self.assertEqual([
('c', 1),
('d', 1024 * 1024),
('e', 1),
('a', 512 * 1024),
('b', 512 * 1024),
('c', 1),
('d', 1)], grouped_file_contents)
(b'c', 1),
(b'd', 1024 * 1024),
(b'e', 1),
(b'a', 512 * 1024),
(b'b', 512 * 1024),
(b'c', 1),
(b'd', 1),
], group_file_contents(file_item.read()))
def test_slo_get_ranged_manifest_repeated_segment(self):
file_item = self.env.container.file('ranged-manifest-repeated-segment')
grouped_file_contents = [
(char, sum(1 for _char in grp))
for char, grp in itertools.groupby(file_item.read())]
self.assertEqual(
[('a', 2097152), ('b', 1048576)],
grouped_file_contents)
[(b'a', 2097152), (b'b', 1048576)],
group_file_contents(file_item.read()))
def test_slo_get_ranged_submanifest(self):
file_item = self.env.container.file('ranged-submanifest')
grouped_file_contents = [
(char, sum(1 for _char in grp))
for char, grp in itertools.groupby(file_item.read())]
self.assertEqual([
('c', 1024 * 1024 + 1),
('d', 1024 * 1024),
('e', 1),
('a', 512 * 1024),
('b', 512 * 1024),
('c', 1),
('d', 512 * 1024 + 1),
('e', 1),
('a', 512 * 1024),
('b', 1),
('c', 1),
('d', 1)], grouped_file_contents)
(b'c', 1024 * 1024 + 1),
(b'd', 1024 * 1024),
(b'e', 1),
(b'a', 512 * 1024),
(b'b', 512 * 1024),
(b'c', 1),
(b'd', 512 * 1024 + 1),
(b'e', 1),
(b'a', 512 * 1024),
(b'b', 1),
(b'c', 1),
(b'd', 1),
], group_file_contents(file_item.read()))
def test_slo_ranged_get(self):
file_item = self.env.container.file('manifest-abcde')
file_contents = file_item.read(size=1024 * 1024 + 2,
offset=1024 * 1024 - 1)
self.assertEqual(file_item.conn.response.status, 206)
headers = dict(file_item.conn.response.getheaders())
headers = dict(
(h.lower(), v)
for h, v in file_item.conn.response.getheaders())
self.assertIn('etag', headers)
self.assertEqual(headers['etag'], '"%s"' % self.manifest_abcde_etag)
self.assertEqual('a', file_contents[0])
self.assertEqual('b', file_contents[1])
self.assertEqual('b', file_contents[-2])
self.assertEqual('c', file_contents[-1])
self.assertEqual([
(b'a', 1),
(b'b', 1048576),
(b'c', 1),
], group_file_contents(file_contents))
def test_slo_ranged_get_half_open_on_right(self):
file_item = self.env.container.file('manifest-abcde')
file_contents = file_item.read(
hdrs={"Range": "bytes=1048571-"})
grouped_file_contents = [
(char, sum(1 for _char in grp))
for char, grp in itertools.groupby(file_contents)]
self.assertEqual([
('a', 5),
('b', 1048576),
('c', 1048576),
('d', 1048576),
('e', 1)
], grouped_file_contents)
(b'a', 5),
(b'b', 1048576),
(b'c', 1048576),
(b'd', 1048576),
(b'e', 1)
], group_file_contents(file_contents))
def test_slo_ranged_get_half_open_on_left(self):
file_item = self.env.container.file('manifest-abcde')
file_contents = file_item.read(
hdrs={"Range": "bytes=-123456"})
grouped_file_contents = [
(char, sum(1 for _char in grp))
for char, grp in itertools.groupby(file_contents)]
self.assertEqual([
('d', 123455),
('e', 1),
], grouped_file_contents)
(b'd', 123455),
(b'e', 1),
], group_file_contents(file_contents))
def test_slo_multi_ranged_get(self):
file_item = self.env.container.file('manifest-abcde')
@ -440,8 +447,12 @@ class TestSlo(Base):
hdrs={"Range": "bytes=1048571-1048580,2097147-2097156"})
# See testMultiRangeGets for explanation
parser = email.parser.FeedParser()
parser.feed("Content-Type: %s\r\n\r\n" % file_item.content_type)
if six.PY2:
parser = email.parser.FeedParser()
else:
parser = email.parser.BytesFeedParser()
parser.feed((
"Content-Type: %s\r\n\r\n" % file_item.content_type).encode())
parser.feed(file_contents)
root_message = parser.close()
@ -454,22 +465,23 @@ class TestSlo(Base):
"application/octet-stream")
self.assertEqual(
byteranges[0]['Content-Range'], "bytes 1048571-1048580/4194305")
self.assertEqual(byteranges[0].get_payload(), "aaaaabbbbb")
self.assertEqual(byteranges[0].get_payload(decode=True), b"aaaaabbbbb")
self.assertEqual(byteranges[1]['Content-Type'],
"application/octet-stream")
self.assertEqual(
byteranges[1]['Content-Range'], "bytes 2097147-2097156/4194305")
self.assertEqual(byteranges[1].get_payload(), "bbbbbccccc")
self.assertEqual(byteranges[1].get_payload(decode=True), b"bbbbbccccc")
def test_slo_ranged_submanifest(self):
file_item = self.env.container.file('manifest-abcde-submanifest')
file_contents = file_item.read(size=1024 * 1024 + 2,
offset=1024 * 1024 * 2 - 1)
self.assertEqual('b', file_contents[0])
self.assertEqual('c', file_contents[1])
self.assertEqual('c', file_contents[-2])
self.assertEqual('d', file_contents[-1])
self.assertEqual([
(b'b', 1),
(b'c', 1024 * 1024),
(b'd', 1),
], group_file_contents(file_contents))
def test_slo_etag_is_quote_wrapped_hash_of_etags(self):
# we have this check in test_slo_get_simple_manifest, too,
@ -481,16 +493,17 @@ class TestSlo(Base):
def test_slo_etag_is_quote_wrapped_hash_of_etags_submanifests(self):
def hd(x):
return hashlib.md5(x).hexdigest()
return hashlib.md5(x).hexdigest().encode('ascii')
expected_etag = hd(hd('a' * 1024 * 1024) +
hd(hd('b' * 1024 * 1024) +
hd(hd('c' * 1024 * 1024) +
hd('d' * 1024 * 1024))) +
hd('e'))
expected_etag = hd(hd(b'a' * 1024 * 1024) +
hd(hd(b'b' * 1024 * 1024) +
hd(hd(b'c' * 1024 * 1024) +
hd(b'd' * 1024 * 1024))) +
hd(b'e'))
file_item = self.env.container.file('manifest-abcde-submanifest')
self.assertEqual('"%s"' % expected_etag, file_item.info()['etag'])
self.assertEqual('"%s"' % expected_etag.decode('ascii'),
file_item.info()['etag'])
def test_slo_etag_mismatch(self):
file_item = self.env.container.file("manifest-a-bad-etag")
@ -499,7 +512,8 @@ class TestSlo(Base):
json.dumps([{
'size_bytes': 1024 * 1024,
'etag': 'not it',
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
except ResponseError as err:
self.assertEqual(400, err.status)
@ -512,8 +526,9 @@ class TestSlo(Base):
file_item.write(
json.dumps([{
'size_bytes': 1024 * 1024 - 1,
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'etag': hashlib.md5(b'a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
except ResponseError as err:
self.assertEqual(400, err.status)
@ -526,8 +541,9 @@ class TestSlo(Base):
file_item.write(
json.dumps([{
'size_bytes': 1024 * 1024,
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'etag': hashlib.md5(b'a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'},
hdrs={'Etag': 'NOTetagofthesegments'})
except ResponseError as err:
@ -535,8 +551,8 @@ class TestSlo(Base):
def test_slo_client_etag(self):
file_item = self.env.container.file("manifest-a-b-etag")
etag_a = hashlib.md5('a' * 1024 * 1024).hexdigest()
etag_b = hashlib.md5('b' * 1024 * 1024).hexdigest()
etag_a = hashlib.md5(b'a' * 1024 * 1024).hexdigest()
etag_b = hashlib.md5(b'b' * 1024 * 1024).hexdigest()
file_item.write(
json.dumps([{
'size_bytes': 1024 * 1024,
@ -544,9 +560,10 @@ class TestSlo(Base):
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}, {
'size_bytes': 1024 * 1024,
'etag': etag_b,
'path': '/%s/%s' % (self.env.container.name, 'seg_b')}]),
'path': '/%s/%s' % (self.env.container.name, 'seg_b'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'},
hdrs={'Etag': hashlib.md5(etag_a + etag_b).hexdigest()})
hdrs={'Etag': hashlib.md5((etag_a + etag_b).encode()).hexdigest()})
self.assert_status(201)
def test_slo_unspecified_etag(self):
@ -555,7 +572,8 @@ class TestSlo(Base):
json.dumps([{
'size_bytes': 1024 * 1024,
'etag': None,
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
self.assert_status(201)
@ -564,8 +582,9 @@ class TestSlo(Base):
file_item.write(
json.dumps([{
'size_bytes': None,
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'etag': hashlib.md5(b'a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
self.assert_status(201)
@ -574,18 +593,20 @@ class TestSlo(Base):
file_item.write(
json.dumps([{
'path': '/%s/%s' % (self.env.container.name,
'seg_with_%ff_funky_name')}]),
'seg_with_%ff_funky_name'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
self.assert_status(201)
self.assertEqual('z' * 10, file_item.read())
self.assertEqual(b'z' * 10, file_item.read())
def test_slo_missing_etag(self):
file_item = self.env.container.file("manifest-a-missing-etag")
file_item.write(
json.dumps([{
'size_bytes': 1024 * 1024,
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
self.assert_status(201)
@ -593,8 +614,9 @@ class TestSlo(Base):
file_item = self.env.container.file("manifest-a-missing-size")
file_item.write(
json.dumps([{
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'etag': hashlib.md5(b'a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
self.assert_status(201)
@ -602,7 +624,8 @@ class TestSlo(Base):
file_item = self.env.container.file("manifest-a-path-only")
file_item.write(
json.dumps([{
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
self.assert_status(201)
@ -611,9 +634,10 @@ class TestSlo(Base):
try:
file_item.write(
json.dumps([{
'teag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
'teag': hashlib.md5(b'a' * 1024 * 1024).hexdigest(),
'size_bytes': 1024 * 1024,
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
except ResponseError as err:
self.assertEqual(400, err.status)
@ -625,9 +649,10 @@ class TestSlo(Base):
try:
file_item.write(
json.dumps([{
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
'etag': hashlib.md5(b'a' * 1024 * 1024).hexdigest(),
'siz_bytes': 1024 * 1024,
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
'path': '/%s/%s' % (self.env.container.name, 'seg_a'),
}]).encode('ascii'),
parms={'multipart-manifest': 'put'})
except ResponseError as err:
self.assertEqual(400, err.status)
@ -640,14 +665,15 @@ class TestSlo(Base):
file_item.write(
json.dumps([
{'size_bytes': 1024 * 1024,
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
'etag': hashlib.md5(b'a' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_a')},
{'size_bytes': 1024 * 1024,
'etag': hashlib.md5('b' * 1024 * 1024).hexdigest(),
'etag': hashlib.md5(b'b' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_b')},
{'size_bytes': 1024 * 1024,
'etag': hashlib.md5('c' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_c')}]),
'etag': hashlib.md5(b'c' * 1024 * 1024).hexdigest(),
'path': '/%s/%s' % (self.env.container.name, 'seg_c')},
]).encode('ascii'),
parms={'multipart-manifest': 'put'})
self.assertEqual(400, catcher.exception.status)
@ -752,8 +778,8 @@ class TestSlo(Base):
except ValueError:
self.fail("COPY didn't copy the manifest (invalid json on GET)")
self.assertEqual(source_contents, copied_contents)
self.assertEqual(copied_json[0],
{'data': base64.b64encode('APRE' * 8)})
self.assertEqual(copied_json[0], {
'data': base64.b64encode(b'APRE' * 8).decode('ascii')})
def test_slo_copy_the_manifest_updating_metadata(self):
source = self.env.container.file("manifest-abcde")
@ -847,7 +873,7 @@ class TestSlo(Base):
parms={'multipart-manifest': 'get'})
self.assertEqual(400, file_item.conn.response.status)
resp_body = file_item.conn.response.read()
self.assertEqual(5, resp_body.count('403 Forbidden'),
self.assertEqual(5, resp_body.count(b'403 Forbidden'),
'Unexpected response body %r' % resp_body)
# create segments container in account2 with read access for account1
@ -863,7 +889,7 @@ class TestSlo(Base):
parms={'multipart-manifest': 'get'})
self.assertEqual(400, file_item.conn.response.status)
resp_body = file_item.conn.response.read()
self.assertEqual(5, resp_body.count('404 Not Found'),
self.assertEqual(5, resp_body.count(b'404 Not Found'),
'Unexpected response body %r' % resp_body)
# create segments in account2 container with same name as in account1,
@ -895,23 +921,24 @@ class TestSlo(Base):
if include_error:
manifest_data.append({'path': 'non-existent/segment'})
resp = file_item.write(
json.dumps(manifest_data),
json.dumps(manifest_data).encode('ascii'),
parms={'multipart-manifest': 'put', 'heartbeat': 'on'},
hdrs=headers, return_resp=True)
self.assertEqual(resp.status, 202)
self.assertTrue(resp.chunked)
body_lines = resp.body.split('\n', 2)
body_lines = resp.body.split(b'\n', 2)
self.assertFalse(body_lines[0].strip()) # all whitespace
self.assertEqual('\r', body_lines[1])
self.assertEqual(b'\r', body_lines[1])
return body_lines[2]
body_lines = do_put().split('\n')
body_lines = do_put().decode('utf8').split('\n')
self.assertIn('Response Status: 201 Created', body_lines)
self.assertIn('Etag', [line.split(':', 1)[0] for line in body_lines])
self.assertIn('Last Modified', [line.split(':', 1)[0]
for line in body_lines])
body_lines = do_put({'Accept': 'text/plain'}).split('\n')
body_lines = do_put(
{'Accept': 'text/plain'}).decode('utf8').split('\n')
self.assertIn('Response Status: 201 Created', body_lines)
self.assertIn('Etag', [line.split(':', 1)[0] for line in body_lines])
self.assertIn('Last Modified', [line.split(':', 1)[0]
@ -932,7 +959,7 @@ class TestSlo(Base):
'Errors': [],
})
body_lines = do_put(include_error=True).split('\n')
body_lines = do_put(include_error=True).decode('utf8').split('\n')
self.assertIn('Response Status: 400 Bad Request', body_lines)
self.assertIn('Response Body: Bad Request', body_lines)
self.assertNotIn('Etag', [line.split(':', 1)[0]
@ -982,7 +1009,7 @@ class TestSlo(Base):
file_item.write(
json.dumps([seg_info['seg_a'], seg_info['seg_b'],
seg_info['seg_c'], seg_info['seg_d'],
seg_info['seg_e']]),
seg_info['seg_e']]).encode('ascii'),
parms={'multipart-manifest': 'put'})
return file_item
@ -1053,15 +1080,19 @@ class TestSlo(Base):
self.assertEqual(len(value), 2)
self.assertEqual(value[0]['bytes'], 1024 * 1024)
self.assertEqual(value[0]['hash'],
hashlib.md5('d' * 1024 * 1024).hexdigest())
self.assertEqual(value[0]['name'],
'/%s/seg_d' % self.env.container.name.decode("utf-8"))
hashlib.md5(b'd' * 1024 * 1024).hexdigest())
expected_name = '/%s/seg_d' % self.env.container.name
if six.PY2:
expected_name = expected_name.decode("utf-8")
self.assertEqual(value[0]['name'], expected_name)
self.assertEqual(value[1]['bytes'], 1024 * 1024)
self.assertEqual(value[1]['hash'],
hashlib.md5('b' * 1024 * 1024).hexdigest())
self.assertEqual(value[1]['name'],
'/%s/seg_b' % self.env.container.name.decode("utf-8"))
hashlib.md5(b'b' * 1024 * 1024).hexdigest())
expected_name = '/%s/seg_b' % self.env.container.name
if six.PY2:
expected_name = expected_name.decode("utf-8")
self.assertEqual(value[1]['name'], expected_name)
def test_slo_get_raw_the_manifest_with_details_from_server(self):
manifest = self.env.container.file("manifest-db")
@ -1081,14 +1112,18 @@ class TestSlo(Base):
self.assertEqual(len(value), 2)
self.assertEqual(value[0]['size_bytes'], 1024 * 1024)
self.assertEqual(value[0]['etag'],
hashlib.md5('d' * 1024 * 1024).hexdigest())
self.assertEqual(value[0]['path'],
'/%s/seg_d' % self.env.container.name.decode("utf-8"))
hashlib.md5(b'd' * 1024 * 1024).hexdigest())
expected_name = '/%s/seg_d' % self.env.container.name
if six.PY2:
expected_name = expected_name.decode("utf-8")
self.assertEqual(value[0]['path'], expected_name)
self.assertEqual(value[1]['size_bytes'], 1024 * 1024)
self.assertEqual(value[1]['etag'],
hashlib.md5('b' * 1024 * 1024).hexdigest())
self.assertEqual(value[1]['path'],
'/%s/seg_b' % self.env.container.name.decode("utf-8"))
hashlib.md5(b'b' * 1024 * 1024).hexdigest())
expected_name = '/%s/seg_b' % self.env.container.name
if six.PY2:
expected_name = expected_name.decode("utf-8")
self.assertEqual(value[1]['path'], expected_name)
file_item = self.env.container.file("manifest-from-get-raw")
file_item.write(got_body, parms={'multipart-manifest': 'put'})
@ -1121,16 +1156,19 @@ class TestSlo(Base):
'etag': None,
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}])
self.assertRaises(ResponseError, file_item.write, manifest,
self.assertRaises(ResponseError, file_item.write,
manifest.encode('ascii'),
parms={'multipart-manifest': 'put'},
hdrs={'If-None-Match': '"not-star"'})
self.assert_status(400)
file_item.write(manifest, parms={'multipart-manifest': 'put'},
file_item.write(manifest.encode('ascii'),
parms={'multipart-manifest': 'put'},
hdrs={'If-None-Match': '*'})
self.assert_status(201)
self.assertRaises(ResponseError, file_item.write, manifest,
self.assertRaises(ResponseError, file_item.write,
manifest.encode('ascii'),
parms={'multipart-manifest': 'put'},
hdrs={'If-None-Match': '*'})
self.assert_status(412)
@ -1191,11 +1229,11 @@ class TestSlo(Base):
self.env.container.update_metadata(referer_metadata)
contents = slo_file.read(hdrs=headers)
self.assertEqual(4 * 1024 * 1024 + 1, len(contents))
self.assertEqual('a', contents[0])
self.assertEqual('a', contents[1024 * 1024 - 1])
self.assertEqual('b', contents[1024 * 1024])
self.assertEqual('d', contents[-2])
self.assertEqual('e', contents[-1])
self.assertEqual(b'a', contents[:1])
self.assertEqual(b'a', contents[1024 * 1024 - 1:1024 * 1024])
self.assertEqual(b'b', contents[1024 * 1024:1024 * 1024 + 1])
self.assertEqual(b'd', contents[-2:-1])
self.assertEqual(b'e', contents[-1:])
def test_slo_data_segments(self):
# len('APRE' * 8) == 32
@ -1212,40 +1250,37 @@ class TestSlo(Base):
file_item = self.env.container.file(file_name)
file_contents = file_item.read(size=3 * 1024 ** 2 + 456,
offset=28)
grouped_file_contents = [
(char, sum(1 for _char in grp))
for char, grp in itertools.groupby(file_contents)]
self.assertEqual([
('A', 1),
('P', 1),
('R', 1),
('E', 1),
('a', 1024 * 1024),
(b'A', 1),
(b'P', 1),
(b'R', 1),
(b'E', 1),
(b'a', 1024 * 1024),
] + [
('A', 1),
('P', 1),
('O', 1),
('S', 1),
(b'A', 1),
(b'P', 1),
(b'O', 1),
(b'S', 1),
] * 16 + [
('b', 1024 * 1024),
(b'b', 1024 * 1024),
] + [
('B', 1),
('P', 1),
('O', 1),
('S', 1),
(b'B', 1),
(b'P', 1),
(b'O', 1),
(b'S', 1),
] * 32 + [
('C', 1),
('P', 1),
('R', 1),
('E', 1),
(b'C', 1),
(b'P', 1),
(b'R', 1),
(b'E', 1),
] * 64 + [
('c', 1024 * 1024),
(b'c', 1024 * 1024),
] + [
('C', 1),
('P', 1),
('O', 1),
('S', 1),
], grouped_file_contents)
(b'C', 1),
(b'P', 1),
(b'O', 1),
(b'S', 1),
], group_file_contents(file_contents))
class TestSloUTF8(Base2, TestSlo):

673
test/functional/test_symlink.py
View File

@ -73,8 +73,10 @@ class TestSymlinkEnv(BaseEnv):
return (cls.link_cont, cls.tgt_cont)
@classmethod
def target_content_location(cls):
return '%s/%s' % (cls.tgt_cont, cls.tgt_obj)
def target_content_location(cls, override_obj=None, override_account=None):
account = override_account or tf.parsed[0].path.split('/', 2)[2]
return '/v1/%s/%s/%s' % (account, cls.tgt_cont,
override_obj or cls.tgt_obj)
@classmethod
def _make_request(cls, url, token, parsed, conn, method,
@ -102,20 +104,21 @@ class TestSymlinkEnv(BaseEnv):
return name
@classmethod
def _create_tgt_object(cls):
def _create_tgt_object(cls, body=TARGET_BODY):
resp = retry(cls._make_request, method='PUT',
headers={'Content-Type': 'application/target'},
container=cls.tgt_cont, obj=cls.tgt_obj,
body=TARGET_BODY)
body=body)
if resp.status != 201:
raise ResponseError(resp)
# sanity: successful put response has content-length 0
cls.tgt_length = str(len(TARGET_BODY))
cls.tgt_length = str(len(body))
cls.tgt_etag = resp.getheader('etag')
resp = retry(cls._make_request, method='GET',
container=cls.tgt_cont, obj=cls.tgt_obj)
if resp.status != 200 and resp.content != TARGET_BODY:
if resp.status != 200 and resp.content != body:
raise ResponseError(resp)
@classmethod
@ -176,10 +179,17 @@ class TestSymlink(Base):
yield uuid4().hex
self.obj_name_gen = object_name_generator()
self._account_name = None
def tearDown(self):
self.env.tearDown()
@property
def account_name(self):
if not self._account_name:
self._account_name = tf.parsed[0].path.split('/', 2)[2]
return self._account_name
def _make_request(self, url, token, parsed, conn, method,
container, obj='', headers=None, body=b'',
query_args=None, allow_redirects=True):
@ -210,22 +220,30 @@ class TestSymlink(Base):
headers=headers)
self.assertEqual(resp.status, 201)
def _test_put_symlink_with_etag(self, link_cont, link_obj, tgt_cont,
tgt_obj, etag, headers=None):
headers = headers or {}
headers.update({'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj),
'X-Symlink-Target-Etag': etag})
resp = retry(self._make_request, method='PUT',
container=link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 201, resp.content)
def _test_get_as_target_object(
self, link_cont, link_obj, expected_content_location,
use_account=1):
resp = retry(
self._make_request, method='GET',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.status, 200, resp.content)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'),
str(self.env.tgt_length))
self.assertEqual(resp.getheader('etag'), self.env.tgt_etag)
self.assertIn('Content-Location', resp.headers)
# TODO: content-location is a full path so it's better to assert
# with the value, instead of assertIn
self.assertIn(expected_content_location,
resp.getheader('content-location'))
self.assertEqual(expected_content_location,
resp.getheader('content-location'))
return resp
def _test_head_as_target_object(self, link_cont, link_obj, use_account=1):
@ -299,8 +317,8 @@ class TestSymlink(Base):
# and it's normalized
self._assertSymlink(
self.env.link_cont, link_obj,
expected_content_location='%s/%s' % (
self.env.tgt_cont, normalized_quoted_obj))
expected_content_location=self.env.target_content_location(
normalized_quoted_obj))
# create a symlink using the normalized target path
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
@ -309,8 +327,8 @@ class TestSymlink(Base):
# and it's ALSO normalized
self._assertSymlink(
self.env.link_cont, link_obj,
expected_content_location='%s/%s' % (
self.env.tgt_cont, normalized_quoted_obj))
expected_content_location=self.env.target_content_location(
normalized_quoted_obj))
def test_symlink_put_head_get(self):
link_obj = uuid4().hex
@ -322,6 +340,195 @@ class TestSymlink(Base):
self._assertSymlink(self.env.link_cont, link_obj)
def test_symlink_with_etag_put_head_get(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
self._assertSymlink(self.env.link_cont, link_obj)
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': self.env.tgt_etag})
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': 'not-the-etag'})
self.assertEqual(resp.status, 412)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_static_symlink_with_bad_etag_put_head_get(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# overwrite tgt object
self.env._create_tgt_object(body='updated target body')
resp = retry(
self._make_request, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
# but we still know where it points
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# uses a mechanism entirely divorced from if-match
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': self.env.tgt_etag})
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': 'not-the-etag'})
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='DELETE',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
# not-found-ness trumps if-match-ness
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 404)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_dynamic_link_to_static_link(self):
static_link_obj = uuid4().hex
# PUT static_link to tgt_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
symlink_obj = uuid4().hex
# PUT symlink to static_link
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.link_cont,
tgt_obj=static_link_obj)
self._test_get_as_target_object(
link_cont=self.env.link_cont, link_obj=symlink_obj,
expected_content_location=self.env.target_content_location())
def test_static_link_to_dynamic_link(self):
symlink_obj = uuid4().hex
# PUT symlink to tgt_obj
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
static_link_obj = uuid4().hex
# PUT a static_link to the symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
self._test_get_as_target_object(
link_cont=self.env.link_cont, link_obj=static_link_obj,
expected_content_location=self.env.target_content_location())
def test_static_link_to_nowhere(self):
missing_obj = uuid4().hex
static_link_obj = uuid4().hex
# PUT a static_link to the missing name
headers = {
'X-Symlink-Target': '%s/%s' % (self.env.link_cont, missing_obj),
'X-Symlink-Target-Etag': MD5_OF_EMPTY_STRING}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=static_link_obj,
headers=headers)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content, b'X-Symlink-Target does not exist')
def test_static_link_to_broken_symlink(self):
symlink_obj = uuid4().hex
# PUT symlink to tgt_obj
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
static_link_obj = uuid4().hex
# PUT a static_link to the symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
# break the symlink
resp = retry(
self._make_request, method='DELETE',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
self.assertEqual(resp.status // 100, 2)
# sanity
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=symlink_obj)
self.assertEqual(resp.status, 404)
# static_link is broken too!
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=static_link_obj)
self.assertEqual(resp.status, 404)
# interestingly you may create a static_link to a broken symlink
broken_static_link_obj = uuid4().hex
# PUT a static_link to the broken symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=broken_static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
def test_symlink_get_ranged(self):
link_obj = uuid4().hex
@ -353,9 +560,8 @@ class TestSymlink(Base):
container=self.env.link_cont, obj=link_obj, use_account=1)
self.assertEqual(resp.status, 404)
self.assertIn('Content-Location', resp.headers)
expected_location_hdr = "%s/%s" % (self.env.tgt_cont, target_obj)
self.assertIn(expected_location_hdr,
resp.getheader('content-location'))
self.assertEqual(self.env.target_content_location(target_obj),
resp.getheader('content-location'))
# HEAD on target object via symlink should return a 404 since target
# object has not yet been written
@ -396,8 +602,8 @@ class TestSymlink(Base):
self.assertEqual(resp.getheader('content-length'), str(target_length))
self.assertEqual(resp.getheader('etag'), target_etag)
self.assertIn('Content-Location', resp.headers)
self.assertIn(expected_location_hdr,
resp.getheader('content-location'))
self.assertEqual(self.env.target_content_location(target_obj),
resp.getheader('content-location'))
def test_symlink_chain(self):
# Testing to symlink chain like symlink -> symlink -> target.
@ -448,6 +654,66 @@ class TestSymlink(Base):
# However, HEAD/GET to the (just) link is still ok
self._assertLinkObject(container, too_many_chain_link)
def test_symlink_chain_with_etag(self):
# Testing to symlink chain like symlink -> symlink -> target.
symloop_max = cluster_info['symlink']['symloop_max']
# create symlink chain in a container. To simplify,
# use target container for all objects (symlinks and target) here
previous = self.env.tgt_obj
container = self.env.tgt_cont
for link_obj in itertools.islice(self.obj_name_gen, symloop_max):
# PUT link_obj point to tgt_obj
self._test_put_symlink_with_etag(link_cont=container,
link_obj=link_obj,
tgt_cont=container,
tgt_obj=previous,
etag=self.env.tgt_etag)
# set current link_obj to previous
previous = link_obj
# the last link is valid for symloop_max constraint
max_chain_link = link_obj
self._assertSymlink(link_cont=container, link_obj=max_chain_link)
# chained etag validation works as long as the target symlink works
headers = {'X-Symlink-Target': '%s/%s' % (container, max_chain_link),
'X-Symlink-Target-Etag': 'not-the-real-etag'}
resp = retry(self._make_request, method='PUT',
container=container, obj=uuid4().hex,
headers=headers)
self.assertEqual(resp.status, 409)
# PUT a new link_obj pointing to the max_chain_link can validate the
# ETag but will result in 409 error on the HEAD/GET.
too_many_chain_link = next(self.obj_name_gen)
self._test_put_symlink_with_etag(
link_cont=container, link_obj=too_many_chain_link,
tgt_cont=container, tgt_obj=max_chain_link,
etag=self.env.tgt_etag)
# try to HEAD to target object via too_many_chain_link
resp = retry(self._make_request, method='HEAD',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content, b'')
# try to GET to target object via too_many_chain_link
resp = retry(self._make_request, method='GET',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(
resp.content,
b'Too many levels of symbolic links, maximum allowed is %d' %
symloop_max)
# However, HEAD/GET to the (just) link is still ok
self._assertLinkObject(container, too_many_chain_link)
def test_symlink_and_slo_manifest_chain(self):
if 'slo' not in cluster_info:
raise SkipTest
@ -557,7 +823,7 @@ class TestSymlink(Base):
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj)}
resp = retry(
self._make_request, method='PUT', container=self.env.link_cont,
obj=link_obj, body='non-zero-length', headers=headers)
obj=link_obj, body=b'non-zero-length', headers=headers)
self.assertEqual(resp.status, 400)
self.assertEqual(resp.content,
@ -636,7 +902,6 @@ class TestSymlink(Base):
tgt_obj=self.env.tgt_obj)
copy_src = '%s/%s' % (self.env.link_cont, link_obj1)
account_one = tf.parsed[0].path.split('/', 2)[2]
perm_two = tf.swift_test_perm[1]
# add X-Content-Read to account 1 link_cont and tgt_cont
@ -659,7 +924,7 @@ class TestSymlink(Base):
# symlink to the account 2 container that points to the
# container/object in the account 2.
# (the container/object is not prepared)
headers = {'X-Copy-From-Account': account_one,
headers = {'X-Copy-From-Account': self.account_name,
'X-Copy-From': copy_src}
resp = retry(self._make_request_with_symlink_get, method='PUT',
container=self.env.link_cont, obj=link_obj2,
@ -669,6 +934,7 @@ class TestSymlink(Base):
# sanity: HEAD/GET on link_obj itself
self._assertLinkObject(self.env.link_cont, link_obj2, use_account=2)
account_two = tf.parsed[1].path.split('/', 2)[2]
# no target object in the account 2
for method in ('HEAD', 'GET'):
resp = retry(
@ -676,14 +942,15 @@ class TestSymlink(Base):
container=self.env.link_cont, obj=link_obj2, use_account=2)
self.assertEqual(resp.status, 404)
self.assertIn('content-location', resp.headers)
self.assertIn(self.env.target_content_location(),
resp.getheader('content-location'))
self.assertEqual(
self.env.target_content_location(override_account=account_two),
resp.getheader('content-location'))
# copy symlink itself to a different account with target account
# the target path will be in account 1
# the target path will have an object
headers = {'X-Symlink-target-Account': account_one,
'X-Copy-From-Account': account_one,
headers = {'X-Symlink-target-Account': self.account_name,
'X-Copy-From-Account': self.account_name,
'X-Copy-From': copy_src}
resp = retry(
self._make_request_with_symlink_get, method='PUT',
@ -780,7 +1047,8 @@ class TestSymlink(Base):
link_obj = uuid4().hex
value1 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
@ -821,6 +1089,73 @@ class TestSymlink(Base):
# sanity: no X-Object-Meta-Alpha exists in the response header
self.assertNotIn('X-Object-Meta-Alpha', resp.headers)
def test_post_to_broken_dynamic_symlink(self):
# create a symlink to nowhere
link_obj = '%s-the-link' % uuid4().hex
tgt_obj = '%s-no-where' % uuid4().hex
headers = {'X-Symlink-Target': '%s/%s' % (self.env.tgt_cont, tgt_obj)}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 201)
# it's a real link!
self._assertLinkObject(self.env.link_cont, link_obj)
# ... it's just broken
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 404)
target_path = '/v1/%s/%s/%s' % (
self.account_name, self.env.tgt_cont, tgt_obj)
self.assertEqual(target_path, resp.headers['Content-Location'])
# we'll redirect with the Location header to the (invalid) target
headers = {'X-Object-Meta-Alpha': 'apple'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
self.assertEqual(target_path, resp.headers['Location'])
# and of course metadata *is* applied to the link
resp = retry(
self._make_request_with_symlink_get, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertTrue(resp.getheader('X-Object-Meta-Alpha'), 'apple')
def test_post_to_broken_static_symlink(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# overwrite tgt object
old_tgt_etag = self.env.tgt_etag
self.env._create_tgt_object(body='updated target body')
# sanity
resp = retry(
self._make_request, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
# but POST will still 307
headers = {'X-Object-Meta-Alpha': 'apple'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
target_path = '/v1/%s/%s/%s' % (
self.account_name, self.env.tgt_cont, self.env.tgt_obj)
self.assertEqual(target_path, resp.headers['Location'])
# but we give you the Etag just like... FYI?
self.assertEqual(old_tgt_etag, resp.headers['X-Symlink-Target-Etag'])
def test_post_with_symlink_header(self):
# POSTing to a symlink is not allowed and should return a 307
# updating the symlink target with a POST should always fail
@ -878,11 +1213,9 @@ class TestSymlink(Base):
raise SkipTest
link_obj = uuid4().hex
account_one = tf.parsed[0].path.split('/', 2)[2]
# create symlink in account 2
# pointing to account 1
headers = {'X-Symlink-Target-Account': account_one,
headers = {'X-Symlink-Target-Account': self.account_name,
'X-Symlink-Target':
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj)}
resp = retry(self._make_request, method='PUT',
@ -900,6 +1233,9 @@ class TestSymlink(Base):
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 403)
# still know where it's pointing
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# add X-Content-Read to account 1 tgt_cont
# permit account 2 to read account 1 tgt_cont
@ -917,11 +1253,96 @@ class TestSymlink(Base):
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(),
use_account=2)
self.assertIn(account_one, resp.getheader('content-location'))
@requires_acls
def test_symlink_with_etag_put_target_account(self):
if tf.skip or tf.skip2:
raise SkipTest
link_obj = uuid4().hex
# try to create a symlink in account 2 pointing to account 1
symlink_headers = {
'X-Symlink-Target-Account': self.account_name,
'X-Symlink-Target':
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj),
'X-Symlink-Target-Etag': self.env.tgt_etag}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=symlink_headers, use_account=2)
# since we don't have read access to verify the object we get the
# permissions error
self.assertEqual(resp.status, 403)
perm_two = tf.swift_test_perm[1]
# add X-Content-Read to account 1 tgt_cont
# permit account 2 to read account 1 tgt_cont
# add acl to allow reading from source
acl_headers = {'X-Container-Read': perm_two}
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=acl_headers)
self.assertEqual(resp.status, 204)
# now we can create the symlink
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=symlink_headers, use_account=2)
self.assertEqual(resp.status, 201)
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
# GET to target object via symlink
resp = self._test_get_as_target_object(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(),
use_account=2)
# Overwrite target
resp = retry(self._make_request, method='PUT',
container=self.env.tgt_cont, obj=self.env.tgt_obj,
body='some other content')
self.assertEqual(resp.status, 201)
# link is now broken
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 409)
# but we still know where it points
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# sanity test, remove permissions
headers = {'X-Remove-Container-Read': 'remove'}
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=headers)
self.assertEqual(resp.status, 204)
# it should be ok to get the symlink itself, but not the target object
# because the read acl has been revoked
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 403)
# Still know where it is, though
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_symlink_invalid_etag(self):
link_obj = uuid4().hex
headers = {'X-Symlink-Target': '%s/%s' % (self.env.tgt_cont,
self.env.tgt_obj),
'X-Symlink-Target-Etag': 'not-the-real-etag'}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content,
b"Object Etag 'ab706c400731332bffa67ed4bc15dcac' "
b"does not match X-Symlink-Target-Etag header "
b"'not-the-real-etag'")
def test_symlink_object_listing(self):
link_obj = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
@ -933,9 +1354,53 @@ class TestSymlink(Base):
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
obj_info = object_list[0]
self.assertIn('symlink_path', obj_info)
self.assertEqual(self.env.target_content_location(),
obj_info['symlink_path'])
self.assertNotIn('symlink_etag', obj_info)
def test_static_link_object_listing(self):
link_obj = uuid4().hex
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# sanity
self._assertSymlink(self.env.link_cont, link_obj)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont,
query_args='format=json')
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
self.assertIn('symlink_path', object_list[0])
self.assertIn(self.env.target_content_location(),
object_list[0]['symlink_path'])
self.assertEqual(self.env.target_content_location(),
object_list[0]['symlink_path'])
obj_info = object_list[0]
self.assertIn('symlink_etag', obj_info)
self.assertEqual(self.env.tgt_etag,
obj_info['symlink_etag'])
self.assertEqual(int(self.env.tgt_length),
obj_info['symlink_bytes'])
self.assertEqual(obj_info['content_type'], 'application/target')
# POSTing to a static_link can change the listing Content-Type
headers = {'Content-Type': 'application/foo'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont,
query_args='format=json')
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
obj_info = object_list[0]
self.assertEqual(obj_info['content_type'], 'application/foo')
class TestCrossPolicySymlinkEnv(TestSymlinkEnv):
@ -1007,6 +1472,8 @@ class TestSymlinkSlo(Base):
"Expected slo_enabled to be True/False, got %r" %
(self.env.slo_enabled,))
self.file_symlink = self.env.container.file(uuid4().hex)
self.account_name = self.env.container.conn.storage_path.rsplit(
'/', 1)[-1]
def test_symlink_target_slo_manifest(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
@ -1020,6 +1487,142 @@ class TestSymlinkSlo(Base):
(b'e', 1),
], group_by_byte(self.file_symlink.read()))
manifest_body = self.file_symlink.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == self.file_symlink.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.assertEqual(obj_info, {
'name': self.file_symlink.name,
'content_type': 'application/octet-stream',
'hash': 'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'symlink_path': '/v1/%s/%s/manifest-abcde' % (
self.account_name, self.env.container.name),
})
def test_static_link_target_slo_manifest(self):
manifest_info = self.env.container2.file(
"manifest-abcde").info(parms={
'multipart-manifest': 'get'})
manifest_etag = manifest_info['etag']
self.file_symlink.write(hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': manifest_etag,
})
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(self.file_symlink.read()))
manifest_body = self.file_symlink.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
# check listing
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == self.file_symlink.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.maxDiff = None
slo_info = self.env.container2.file("manifest-abcde").info()
self.assertEqual(obj_info, {
'name': self.file_symlink.name,
'content_type': 'application/octet-stream',
'hash': u'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'slo_etag': slo_info['etag'],
'symlink_path': '/v1/%s/%s/manifest-abcde' % (
self.account_name, self.env.container2.name),
'symlink_bytes': 4 * 2 ** 20 + 1,
'symlink_etag': manifest_etag,
})
def test_static_link_target_slo_manifest_wrong_etag(self):
# try the slo "etag"
slo_etag = self.env.container2.file(
"manifest-abcde").info()['etag']
self.assertRaises(ResponseError, self.file_symlink.write, hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': slo_etag,
})
self.assert_status(400) # no quotes allowed!
# try the slo etag w/o the quotes
slo_etag = slo_etag.strip('"')
self.assertRaises(ResponseError, self.file_symlink.write, hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': slo_etag,
})
self.assert_status(409) # that just doesn't match
def test_static_link_target_symlink_to_slo_manifest(self):
# write symlink
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde')})
# write static_link
file_static_link = self.env.container.file(uuid4().hex)
file_static_link.write(hdrs={
'X-Symlink-Target': '%s/%s' % (
self.file_symlink.container, self.file_symlink.name),
'X-Symlink-Target-Etag': MD5_OF_EMPTY_STRING,
})
# validate reads
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(file_static_link.read()))
manifest_body = file_static_link.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
# check listing
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == file_static_link.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.maxDiff = None
self.assertEqual(obj_info, {
'name': file_static_link.name,
'content_type': 'application/octet-stream',
'hash': 'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'symlink_path': u'/v1/%s/%s/%s' % (
self.account_name, self.file_symlink.container,
self.file_symlink.name),
# the only time bytes/etag aren't the target object are when they
# validate through another static_link
'symlink_bytes': 0,
'symlink_etag': MD5_OF_EMPTY_STRING,
})
def test_symlink_target_slo_nested_manifest(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,

182
test/functional/test_versioned_writes.py
View File

@ -18,6 +18,7 @@ from copy import deepcopy
import json
import time
import unittest2
import six
from six.moves.urllib.parse import quote, unquote
import test.functional as tf
@ -54,9 +55,12 @@ class TestObjectVersioningEnv(BaseEnv):
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
if six.PY2:
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
else:
prefix = Utils.create_name()[:10]
cls.versions_container = cls.account.container(prefix + "-versions")
if not cls.versions_container.create():
@ -143,9 +147,12 @@ class TestCrossPolicyObjectVersioningEnv(BaseEnv):
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
if six.PY2:
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
else:
prefix = Utils.create_name()[:10]
cls.versions_container = cls.account.container(prefix + "-versions")
if not cls.versions_container.create(
@ -254,7 +261,7 @@ class TestObjectVersioning(Base):
put_headers = {'Content-Type': 'text/jibberish01',
'Content-Encoding': 'gzip',
'Content-Disposition': 'attachment; filename=myfile'}
versioned_obj.write("aaaaa", hdrs=put_headers)
versioned_obj.write(b"aaaaa", hdrs=put_headers)
obj_info = versioned_obj.info()
self.assertEqual('text/jibberish01', obj_info['content_type'])
expected_content_types.append('text/jibberish01')
@ -263,14 +270,15 @@ class TestObjectVersioning(Base):
# assert that content-encoding or content-disposition get *copied* to
# the object version unless they were set on the original PUT, so
# populate expected_headers by making a HEAD on the original object
resp_headers = dict(versioned_obj.conn.response.getheaders())
resp_headers = {
h.lower(): v for h, v in versioned_obj.conn.response.getheaders()}
expected_headers = {}
for k, v in put_headers.items():
if k.lower() in resp_headers:
expected_headers[k] = v
self.assertEqual(0, versions_container.info()['object_count'])
versioned_obj.write("bbbbb", hdrs={'Content-Type': 'text/jibberish02',
versioned_obj.write(b"bbbbb", hdrs={'Content-Type': 'text/jibberish02',
'X-Object-Meta-Foo': 'Bar'})
versioned_obj.initialize()
self.assertEqual(versioned_obj.content_type, 'text/jibberish02')
@ -282,10 +290,11 @@ class TestObjectVersioning(Base):
versioned_obj_name = versions_container.files()[0]
prev_version = versions_container.file(versioned_obj_name)
prev_version.initialize()
self.assertEqual("aaaaa", prev_version.read())
self.assertEqual(b"aaaaa", prev_version.read())
self.assertEqual(prev_version.content_type, 'text/jibberish01')
resp_headers = dict(prev_version.conn.response.getheaders())
resp_headers = {
h.lower(): v for h, v in prev_version.conn.response.getheaders()}
for k, v in expected_headers.items():
self.assertIn(k.lower(), resp_headers)
self.assertEqual(v, resp_headers[k.lower()])
@ -298,29 +307,29 @@ class TestObjectVersioning(Base):
self.assertEqual(1, versions_container.info()['object_count'])
# if we overwrite it again, there are two versions
versioned_obj.write("ccccc")
versioned_obj.write(b"ccccc")
self.assertEqual(2, versions_container.info()['object_count'])
expected_content_types.append('text/jibberish02')
versioned_obj_name = versions_container.files()[1]
prev_version = versions_container.file(versioned_obj_name)
prev_version.initialize()
self.assertEqual("bbbbb", prev_version.read())
self.assertEqual(b"bbbbb", prev_version.read())
self.assertEqual(prev_version.content_type, 'text/jibberish02')
self.assertNotIn('foo', prev_version.metadata)
self.assertIn('fu', prev_version.metadata)
# versioned_obj keeps the newest content
self.assertEqual("ccccc", versioned_obj.read())
self.assertEqual(b"ccccc", versioned_obj.read())
# test copy from a different container
src_container = self.env.account.container(Utils.create_name())
self.assertTrue(src_container.create())
src_name = Utils.create_name()
src_obj = src_container.file(src_name)
src_obj.write("ddddd", hdrs={'Content-Type': 'text/jibberish04'})
src_obj.write(b"ddddd", hdrs={'Content-Type': 'text/jibberish04'})
src_obj.copy(container.name, obj_name)
self.assertEqual("ddddd", versioned_obj.read())
self.assertEqual(b"ddddd", versioned_obj.read())
versioned_obj.initialize()
self.assertEqual(versioned_obj.content_type, 'text/jibberish04')
expected_content_types.append('text/jibberish04')
@ -330,7 +339,7 @@ class TestObjectVersioning(Base):
versioned_obj_name = versions_container.files()[2]
prev_version = versions_container.file(versioned_obj_name)
prev_version.initialize()
self.assertEqual("ccccc", prev_version.read())
self.assertEqual(b"ccccc", prev_version.read())
# for further use in the mode-specific tests
return (versioned_obj, expected_headers, expected_content_types)
@ -348,27 +357,28 @@ class TestObjectVersioning(Base):
# test delete
versioned_obj.delete()
self.assertEqual("ccccc", versioned_obj.read())
self.assertEqual(b"ccccc", versioned_obj.read())
expected_content_types.pop()
self.assertEqual(expected_content_types, [
o['content_type'] for o in versions_container.files(
parms={'format': 'json'})])
versioned_obj.delete()
self.assertEqual("bbbbb", versioned_obj.read())
self.assertEqual(b"bbbbb", versioned_obj.read())
expected_content_types.pop()
self.assertEqual(expected_content_types, [
o['content_type'] for o in versions_container.files(
parms={'format': 'json'})])
versioned_obj.delete()
self.assertEqual("aaaaa", versioned_obj.read())
self.assertEqual(b"aaaaa", versioned_obj.read())
self.assertEqual(0, versions_container.info()['object_count'])
# verify that all the original object headers have been copied back
obj_info = versioned_obj.info()
self.assertEqual('text/jibberish01', obj_info['content_type'])
resp_headers = dict(versioned_obj.conn.response.getheaders())
resp_headers = {
h.lower(): v for h, v in versioned_obj.conn.response.getheaders()}
for k, v in expected_headers.items():
self.assertIn(k.lower(), resp_headers)
self.assertEqual(v, resp_headers[k.lower()])
@ -390,27 +400,28 @@ class TestObjectVersioning(Base):
# test delete
versioned_obj.delete()
self.assertEqual("ccccc", versioned_obj.read())
self.assertEqual(b"ccccc", versioned_obj.read())
expected_content_types.pop()
self.assertEqual(expected_content_types, [
o['content_type'] for o in versions_container.files(
parms={'format': 'json'})])
versioned_obj.delete()
self.assertEqual("bbbbb", versioned_obj.read())
self.assertEqual(b"bbbbb", versioned_obj.read())
expected_content_types.pop()
self.assertEqual(expected_content_types, [
o['content_type'] for o in versions_container.files(
parms={'format': 'json'})])
versioned_obj.delete()
self.assertEqual("aaaaa", versioned_obj.read())
self.assertEqual(b"aaaaa", versioned_obj.read())
self.assertEqual(0, versions_container.info()['object_count'])
# verify that all the original object headers have been copied back
obj_info = versioned_obj.info()
self.assertEqual('text/jibberish01', obj_info['content_type'])
resp_headers = dict(versioned_obj.conn.response.getheaders())
resp_headers = {
h.lower(): v for h, v in versioned_obj.conn.response.getheaders()}
for k, v in expected_headers.items():
self.assertIn(k.lower(), resp_headers)
self.assertEqual(v, resp_headers[k.lower()])
@ -420,12 +431,14 @@ class TestObjectVersioning(Base):
def assert_most_recent_version(self, obj_name, content,
should_be_dlo=False):
name_len = len(obj_name if six.PY2 else obj_name.encode('utf8'))
archive_versions = self.env.versions_container.files(parms={
'prefix': '%03x%s/' % (len(obj_name), obj_name),
'prefix': '%03x%s/' % (name_len, obj_name),
'reverse': 'yes'})
archive_file = self.env.versions_container.file(archive_versions[0])
self.assertEqual(content, archive_file.read())
resp_headers = dict(archive_file.conn.response.getheaders())
resp_headers = {
h.lower(): v for h, v in archive_file.conn.response.getheaders()}
if should_be_dlo:
self.assertIn('x-object-manifest', resp_headers)
else:
@ -443,34 +456,35 @@ class TestObjectVersioning(Base):
time.sleep(.01) # guarantee that the timestamp changes
obj_name_seg = obj_name + '/' + i
versioned_obj = container.file(obj_name_seg)
versioned_obj.write(i)
versioned_obj.write(i.encode('ascii'))
# immediately overwrite
versioned_obj.write(i + i)
versioned_obj.write((i + i).encode('ascii'))
self.assertEqual(3, versions_container.info()['object_count'])
man_file = container.file(obj_name)
# write a normal file first
man_file.write('old content')
man_file.write(b'old content')
# guarantee that the timestamp changes
time.sleep(.01)
# overwrite with a dlo manifest
man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
man_file.write(b'', hdrs={"X-Object-Manifest": "%s/%s/" %
(self.env.container.name, obj_name)})
self.assertEqual(4, versions_container.info()['object_count'])
self.assertEqual("112233", man_file.read())
self.assert_most_recent_version(obj_name, 'old content')
self.assertEqual(b"112233", man_file.read())
self.assert_most_recent_version(obj_name, b'old content')
# overwrite the manifest with a normal file
man_file.write('new content')
man_file.write(b'new content')
self.assertEqual(5, versions_container.info()['object_count'])
# new most-recent archive is the dlo
self.assert_most_recent_version(obj_name, '112233', should_be_dlo=True)
self.assert_most_recent_version(
obj_name, b'112233', should_be_dlo=True)
return obj_name, man_file
@ -480,15 +494,16 @@ class TestObjectVersioning(Base):
# verify that restore works properly
man_file.delete()
self.assertEqual(4, self.env.versions_container.info()['object_count'])
self.assertEqual("112233", man_file.read())
resp_headers = dict(man_file.conn.response.getheaders())
self.assertEqual(b"112233", man_file.read())
resp_headers = {
h.lower(): v for h, v in man_file.conn.response.getheaders()}
self.assertIn('x-object-manifest', resp_headers)
self.assert_most_recent_version(obj_name, 'old content')
self.assert_most_recent_version(obj_name, b'old content')
man_file.delete()
self.assertEqual(3, self.env.versions_container.info()['object_count'])
self.assertEqual("old content", man_file.read())
self.assertEqual(b"old content", man_file.read())
def test_versioning_container_acl(self):
if tf.skip2:
@ -503,7 +518,7 @@ class TestObjectVersioning(Base):
# check account2 cannot write to versions container
fail_obj_name = Utils.create_name()
fail_obj = versions_container.file(fail_obj_name)
self.assertRaises(ResponseError, fail_obj.write, "should fail",
self.assertRaises(ResponseError, fail_obj.write, b"should fail",
cfg={'use_token': self.env.storage_token2})
# create container and give write access to account2
@ -528,22 +543,22 @@ class TestObjectVersioning(Base):
# write object twice to container and check version
obj_name = Utils.create_name()
versioned_obj = container.file(obj_name)
self.assertTrue(versioned_obj.write("never argue with the data",
self.assertTrue(versioned_obj.write(b"never argue with the data",
cfg={'use_token': self.env.storage_token2}))
self.assertEqual(versioned_obj.read(), "never argue with the data")
self.assertEqual(versioned_obj.read(), b"never argue with the data")
self.assertTrue(
versioned_obj.write("we don't have no beer, just tequila",
versioned_obj.write(b"we don't have no beer, just tequila",
cfg={'use_token': self.env.storage_token2}))
self.assertEqual(versioned_obj.read(),
"we don't have no beer, just tequila")
b"we don't have no beer, just tequila")
self.assertEqual(1, versions_container.info()['object_count'])
# read the original uploaded object
for filename in versions_container.files():
backup_file = versions_container.file(filename)
break
self.assertEqual(backup_file.read(), "never argue with the data")
self.assertEqual(backup_file.read(), b"never argue with the data")
# user3 (some random user with no access to any of account1)
# tries to read from versioned container
@ -556,13 +571,13 @@ class TestObjectVersioning(Base):
hdrs={'X-Container-Read': self.env.conn3.user_acl},
cfg={'use_token': self.env.storage_token2})
a2_obj = a2_container.file(Utils.create_name())
self.assertTrue(a2_obj.write("unused",
self.assertTrue(a2_obj.write(b"unused",
cfg={'use_token': self.env.storage_token2}))
# user3 cannot write, delete, or copy to/from source container either
number_of_versions = versions_container.info()['object_count']
self.assertRaises(ResponseError, versioned_obj.write,
"some random user trying to write data",
b"some random user trying to write data",
cfg={'use_token': self.env.storage_token3})
self.assertEqual(number_of_versions,
versions_container.info()['object_count'])
@ -610,11 +625,11 @@ class TestObjectVersioning(Base):
obj_name = Utils.create_name()
versioned_obj = container.file(obj_name)
versioned_obj.write("aaaaa")
self.assertEqual("aaaaa", versioned_obj.read())
versioned_obj.write(b"aaaaa")
self.assertEqual(b"aaaaa", versioned_obj.read())
versioned_obj.write("bbbbb")
self.assertEqual("bbbbb", versioned_obj.read())
versioned_obj.write(b"bbbbb")
self.assertEqual(b"bbbbb", versioned_obj.read())
# Use token from second account and try to delete the object
org_token = self.env.account.conn.storage_token
@ -627,7 +642,7 @@ class TestObjectVersioning(Base):
self.env.account.conn.storage_token = org_token
# Verify with token from first account
self.assertEqual("bbbbb", versioned_obj.read())
self.assertEqual(b"bbbbb", versioned_obj.read())
return versioned_obj
def test_versioning_check_acl(self):
@ -635,7 +650,7 @@ class TestObjectVersioning(Base):
raise SkipTest('Account2 not set')
versioned_obj = self._test_versioning_check_acl_setup()
versioned_obj.delete()
self.assertEqual("aaaaa", versioned_obj.read())
self.assertEqual(b"aaaaa", versioned_obj.read())
def _check_overwriting_symlink(self):
# assertions common to x-versions-location and x-history-location modes
@ -646,29 +661,29 @@ class TestObjectVersioning(Base):
tgt_b_name = Utils.create_name()
tgt_a = container.file(tgt_a_name)
tgt_a.write("aaaaa")
tgt_a.write(b"aaaaa")
tgt_b = container.file(tgt_b_name)
tgt_b.write("bbbbb")
tgt_b.write(b"bbbbb")
symlink_name = Utils.create_name()
sym_tgt_header = quote(unquote('%s/%s' % (container.name, tgt_a_name)))
sym_headers_a = {'X-Symlink-Target': sym_tgt_header}
symlink = container.file(symlink_name)
symlink.write("", hdrs=sym_headers_a)
self.assertEqual("aaaaa", symlink.read())
symlink.write(b"", hdrs=sym_headers_a)
self.assertEqual(b"aaaaa", symlink.read())
sym_headers_b = {'X-Symlink-Target': '%s/%s' % (container.name,
tgt_b_name)}
symlink.write("", hdrs=sym_headers_b)
self.assertEqual("bbbbb", symlink.read())
symlink.write(b"", hdrs=sym_headers_b)
self.assertEqual(b"bbbbb", symlink.read())
# the old version got saved off
self.assertEqual(1, versions_container.info()['object_count'])
versioned_obj_name = versions_container.files()[0]
prev_version = versions_container.file(versioned_obj_name)
prev_version_info = prev_version.info(parms={'symlink': 'get'})
self.assertEqual("aaaaa", prev_version.read())
self.assertEqual(b"aaaaa", prev_version.read())
self.assertEqual(MD5_OF_EMPTY_STRING, prev_version_info['etag'])
self.assertEqual(sym_tgt_header,
prev_version_info['x_symlink_target'])
@ -682,7 +697,7 @@ class TestObjectVersioning(Base):
# test delete
symlink.delete()
sym_info = symlink.info(parms={'symlink': 'get'})
self.assertEqual("aaaaa", symlink.read())
self.assertEqual(b"aaaaa", symlink.read())
self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag'])
self.assertEqual(
quote(unquote('%s/%s' % (self.env.container.name, target.name))),
@ -690,16 +705,16 @@ class TestObjectVersioning(Base):
def _setup_symlink(self):
target = self.env.container.file('target-object')
target.write('target object data')
target.write(b'target object data')
symlink = self.env.container.file('symlink')
symlink.write('', hdrs={
symlink.write(b'', hdrs={
'Content-Type': 'application/symlink',
'X-Symlink-Target': '%s/%s' % (
self.env.container.name, target.name)})
return symlink, target
def _assert_symlink(self, symlink, target):
self.assertEqual('target object data', symlink.read())
self.assertEqual(b'target object data', symlink.read())
self.assertEqual(target.info(), symlink.info())
self.assertEqual('application/symlink',
symlink.info(parms={
@ -708,7 +723,7 @@ class TestObjectVersioning(Base):
def _check_copy_destination_restore_symlink(self):
# assertions common to x-versions-location and x-history-location modes
symlink, target = self._setup_symlink()
symlink.write('this is not a symlink')
symlink.write(b'this is not a symlink')
# the symlink is versioned
version_container_files = self.env.versions_container.files(
parms={'format': 'json'})
@ -736,7 +751,7 @@ class TestObjectVersioning(Base):
# and versioned writes restore
symlink.delete()
self.assertEqual(1, self.env.versions_container.info()['object_count'])
self.assertEqual('this is not a symlink', symlink.read())
self.assertEqual(b'this is not a symlink', symlink.read())
symlink.delete()
self.assertEqual(0, self.env.versions_container.info()['object_count'])
self._assert_symlink(symlink, target)
@ -746,7 +761,7 @@ class TestObjectVersioning(Base):
raise SkipTest("Symlinks not enabled")
symlink, target = self._setup_symlink()
symlink.write('this is not a symlink')
symlink.write(b'this is not a symlink')
version_container_files = self.env.versions_container.files()
self.assertEqual(1, len(version_container_files))
versioned_obj = self.env.versions_container.file(
@ -812,7 +827,7 @@ class TestObjectVersioningHistoryMode(TestObjectVersioning):
parms={'format': 'json'})])
# update versioned_obj
versioned_obj.write("eeee", hdrs={'Content-Type': 'text/thanksgiving',
versioned_obj.write(b"eeee", hdrs={'Content-Type': 'text/thanksgiving',
'X-Object-Meta-Bar': 'foo'})
# verify the PUT object is kept successfully
obj_info = versioned_obj.info()
@ -822,7 +837,7 @@ class TestObjectVersioningHistoryMode(TestObjectVersioning):
self.assertEqual(8, versions_container.info()['object_count'])
# update versioned_obj
versioned_obj.write("ffff", hdrs={'Content-Type': 'text/teriyaki',
versioned_obj.write(b"ffff", hdrs={'Content-Type': 'text/teriyaki',
'X-Object-Meta-Food': 'chickin'})
# verify the PUT object is kept successfully
obj_info = versioned_obj.info()
@ -864,7 +879,7 @@ class TestObjectVersioningHistoryMode(TestObjectVersioning):
parms={'format': 'json'})])
# update versioned_obj
versioned_obj.write("eeee", hdrs={'Content-Type': 'text/thanksgiving',
versioned_obj.write(b"eeee", hdrs={'Content-Type': 'text/thanksgiving',
'X-Object-Meta-Bar': 'foo'})
# verify the PUT object is kept successfully
obj_info = versioned_obj.info()
@ -874,7 +889,7 @@ class TestObjectVersioningHistoryMode(TestObjectVersioning):
self.assertEqual(8, versions_container.info()['object_count'])
# update versioned_obj
versioned_obj.write("ffff", hdrs={'Content-Type': 'text/teriyaki',
versioned_obj.write(b"ffff", hdrs={'Content-Type': 'text/teriyaki',
'X-Object-Meta-Food': 'chickin'})
# verify the PUT object is kept successfully
obj_info = versioned_obj.info()
@ -898,7 +913,7 @@ class TestObjectVersioningHistoryMode(TestObjectVersioning):
self.assertEqual(404, cm.exception.status)
self.assertEqual(7, self.env.versions_container.info()['object_count'])
expected = ['old content', '112233', 'new content', '']
expected = [b'old content', b'112233', b'new content', b'']
bodies = [
self.env.versions_container.file(f).read()
@ -919,7 +934,7 @@ class TestObjectVersioningHistoryMode(TestObjectVersioning):
# and delete-marker with empty content
self.assertEqual(3, self.env.versions_container.info()['object_count'])
files = self.env.versions_container.files()
for actual, expected in zip(files, ['aaaaa', 'bbbbb', '']):
for actual, expected in zip(files, [b'aaaaa', b'bbbbb', b'']):
prev_version = self.env.versions_container.file(actual)
self.assertEqual(expected, prev_version.read())
@ -981,7 +996,7 @@ class TestSloWithVersioning(unittest2.TestCase):
('b', 1024 * 1024)):
seg_name = letter
file_item = self.segments_container.file(seg_name)
file_item.write(letter * size)
file_item.write((letter * size).encode('ascii'))
self.seg_info[seg_name] = {
'size_bytes': size,
'etag': file_item.md5,
@ -991,13 +1006,14 @@ class TestSloWithVersioning(unittest2.TestCase):
# create a manifest in the versioning container
file_item = self.container.file("my-slo-manifest")
file_item.write(
json.dumps([self.seg_info[seg_name]]),
json.dumps([self.seg_info[seg_name]]).encode('ascii'),
parms={'multipart-manifest': 'put'})
return file_item
def _assert_is_manifest(self, file_item, seg_name):
manifest_body = file_item.read(parms={'multipart-manifest': 'get'})
resp_headers = dict(file_item.conn.response.getheaders())
resp_headers = {
h.lower(): v for h, v in file_item.conn.response.getheaders()}
self.assertIn('x-static-large-object', resp_headers)
self.assertEqual('application/json; charset=utf-8',
file_item.content_type)
@ -1012,11 +1028,11 @@ class TestSloWithVersioning(unittest2.TestCase):
self.assertEqual(self.seg_info[seg_name][k_client],
manifest[0][k_slo])
def _assert_is_object(self, file_item, seg_name):
def _assert_is_object(self, file_item, seg_data):
file_contents = file_item.read()
self.assertEqual(1024 * 1024, len(file_contents))
self.assertEqual(seg_name, file_contents[0])
self.assertEqual(seg_name, file_contents[-1])
self.assertEqual(seg_data, file_contents[:1])
self.assertEqual(seg_data, file_contents[-1:])
def tearDown(self):
# remove versioning to allow simple container delete
@ -1027,24 +1043,24 @@ class TestSloWithVersioning(unittest2.TestCase):
file_item = self._create_manifest('a')
# sanity check: read the manifest, then the large object
self._assert_is_manifest(file_item, 'a')
self._assert_is_object(file_item, 'a')
self._assert_is_object(file_item, b'a')
# upload new manifest
file_item = self._create_manifest('b')
# sanity check: read the manifest, then the large object
self._assert_is_manifest(file_item, 'b')
self._assert_is_object(file_item, 'b')
self._assert_is_object(file_item, b'b')
versions_list = self.versions_container.files()
self.assertEqual(1, len(versions_list))
version_file = self.versions_container.file(versions_list[0])
# check the version is still a manifest
self._assert_is_manifest(version_file, 'a')
self._assert_is_object(version_file, 'a')
self._assert_is_object(version_file, b'a')
# delete the newest manifest
file_item.delete()
# expect the original manifest file to be restored
self._assert_is_manifest(file_item, 'a')
self._assert_is_object(file_item, 'a')
self._assert_is_object(file_item, b'a')

4
test/functional/tests.py
View File

@ -1290,6 +1290,10 @@ class TestFile(Base):
if not matches:
errors.append('Missing expected header %s' % k)
for (got_k, got_v) in matches:
# The Connection: header is parsed by cluster's LB and may
# be returned in either original lowercase or camel-cased.
if k == 'connection':
got_v = got_v.lower()
if got_v != v:
errors.append('Expected %s but got %s for %s' %
(v, got_v, k))

60
test/probe/test_object_failures.py
View File

@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import time
from os import listdir, unlink
from os.path import join as path_join
@ -26,7 +28,7 @@ from swift.common import direct_client
from swift.common.exceptions import ClientException
from swift.common.utils import hash_path, readconf
from swift.obj.diskfile import write_metadata, read_metadata, get_data_dir
from test.probe.common import ReplProbeTest
from test.probe.common import ReplProbeTest, ECProbeTest
RETRIES = 5
@ -198,5 +200,61 @@ class TestObjectFailures(ReplProbeTest):
self.run_quarantine_zero_byte_post()
class TestECObjectFailures(ECProbeTest):
def test_ec_missing_all_durable_fragments(self):
# This tests helps assert the behavior that when
# the proxy has enough fragments to reconstruct the object
# but none are marked as durable, the proxy should return a 404.
container_name = 'container-%s' % uuid4()
object_name = 'object-%s' % uuid4()
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url, self.token, container_name,
headers=headers)
# PUT object, should go to primary nodes
client.put_object(self.url, self.token, container_name,
object_name, contents='object contents')
# get our node lists
opart, onodes = self.object_ring.get_nodes(
self.account, container_name, object_name)
# sanity test
odata = client.get_object(self.url, self.token, container_name,
object_name)[-1]
self.assertEqual('object contents', odata)
# make all fragments non-durable
for node in onodes:
part_dir = self.storage_dir('object', node, part=opart)
for dirs, subdirs, files in os.walk(part_dir):
for fname in files:
if fname.endswith('.data'):
non_durable_fname = fname.replace('#d', '')
os.rename(os.path.join(dirs, fname),
os.path.join(dirs, non_durable_fname))
break
headers = direct_client.direct_head_object(
node, opart, self.account, container_name, object_name,
headers={
'X-Backend-Storage-Policy-Index': self.policy.idx,
'X-Backend-Fragment-Preferences': json.dumps([])})
self.assertNotIn('X-Backend-Durable-Timestamp', headers)
# Now a new GET should return *404* because all fragments
# are non-durable, even if they are reconstructable
try:
client.get_object(self.url, self.token, container_name,
object_name)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
if __name__ == '__main__':
main()

105
test/probe/test_object_handoff.py
View File

@ -309,6 +309,62 @@ class TestObjectHandoff(ReplProbeTest):
else:
self.fail("Expected ClientException but didn't get it")
def test_missing_primaries(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
# Create container/obj (goes to all three primaries)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries obj server
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
for onode in onodes:
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Indirectly (i.e., through proxy) try to GET object, it should return
# a 503, since all primaries will Timeout and handoffs return a 404.
try:
client.get_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 503)
else:
self.fail("Expected ClientException but didn't get it")
# Restart the first container/obj primary server again
onode = onodes[0]
start_server((onode['ip'], onode['port']), self.ipport2server)
# Send a delete that will reach first primary and handoff.
# Sure, the DELETE will return a 404 since the handoff doesn't
# have a .data file, but object server will still write a
# Tombstone in the handoff node!
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
# kill the first container/obj primary server again
kill_server((onode['ip'], onode['port']), self.ipport2server)
# a new GET should return a 404, since all primaries will Timeout
# and the handoff will return a 404 but this time with a tombstone
try:
client.get_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
class TestECObjectHandoff(ECProbeTest):
@ -523,5 +579,54 @@ class TestECObjectHandoff(ECProbeTest):
# ... all six unique
self.assertEqual(len(frag2count), 6)
def test_ec_primary_timeout(self):
container_name = 'container-%s' % uuid4()
object_name = 'object-%s' % uuid4()
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url, self.token, container_name,
headers=headers)
# PUT object, should go to primary nodes
old_contents = Body()
client.put_object(self.url, self.token, container_name,
object_name, contents=old_contents)
# get our node lists
opart, onodes = self.object_ring.get_nodes(
self.account, container_name, object_name)
# shutdown three of the primary data nodes
for i in range(3):
failed_primary = onodes[i]
failed_primary_device_path = self.device_dir('object',
failed_primary)
self.kill_drive(failed_primary_device_path)
# Indirectly (i.e., through proxy) try to GET object, it should return
# a 503, since all primaries will Timeout and handoffs return a 404.
try:
client.get_object(self.url, self.token, container_name,
object_name)
except client.ClientException as err:
self.assertEqual(err.http_status, 503)
else:
self.fail("Expected ClientException but didn't get it")
# Send a delete to write down tombstones in the handoff nodes
client.delete_object(self.url, self.token, container_name, object_name)
# Now a new GET should return 404 because the handoff nodes
# return a 404 with a Tombstone.
try:
client.get_object(self.url, self.token, container_name,
object_name)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
if __name__ == '__main__':
main()

3
test/unit/__init__.py
View File

@ -1367,7 +1367,8 @@ def xattr_supported_check():
fd, tmppath = mkstemp()
xattr.setxattr(fd, 'user.swift.testing_key', big_val)
except IOError as e:
if errno.errorcode.get(e.errno) in ('ENOSPC', 'ENOTSUP', 'EOPNOTSUPP'):
if errno.errorcode.get(e.errno) in ('ENOSPC', 'ENOTSUP', 'EOPNOTSUPP',
'ERANGE'):
# filesystem does not support xattr of this size
return False
raise

49
test/unit/account/test_server.py
View File

@ -32,6 +32,7 @@ import xml.dom.minidom
from swift import __version__ as swift_version
from swift.common.swob import (Request, WsgiBytesIO, HTTPNoContent)
from swift.common import constraints
from swift.account.backend import AccountBroker
from swift.account.server import AccountController
from swift.common.utils import (normalize_timestamp, replication, public,
mkdirs, storage_directory, Timestamp)
@ -49,7 +50,8 @@ class TestAccountController(unittest.TestCase):
self.testdir = os.path.join(self.testdir_base, 'account_server')
mkdirs(os.path.join(self.testdir, 'sda1'))
self.controller = AccountController(
{'devices': self.testdir, 'mount_check': 'false'})
{'devices': self.testdir, 'mount_check': 'false'},
logger=debug_logger())
def tearDown(self):
"""Tear down for testing swift.account.server.AccountController"""
@ -522,6 +524,51 @@ class TestAccountController(unittest.TestCase):
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 202)
def test_utf8_metadata(self):
ts_str = normalize_timestamp(1)
def get_test_meta(method, headers):
# Set metadata header
headers.setdefault('X-Timestamp', ts_str)
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': method},
headers=headers)
resp = req.get_response(self.controller)
self.assertIn(resp.status_int, (201, 202, 204))
db_path = os.path.join(*next(
(dir_name, file_name)
for dir_name, _, files in os.walk(self.testdir)
for file_name in files if file_name.endswith('.db')
))
broker = AccountBroker(db_path)
# Why not use broker.metadata, you ask? Because we want to get
# as close to the on-disk format as is reasonable.
result = json.loads(broker.get_raw_metadata())
# Clear it out for the next run
with broker.get() as conn:
conn.execute("UPDATE account_stat SET metadata=''")
conn.commit()
return result
wsgi_str = '\xf0\x9f\x91\x8d'
uni_str = u'\U0001f44d'
self.assertEqual(
get_test_meta('PUT', {'x-account-sysmeta-' + wsgi_str: wsgi_str}),
{u'X-Account-Sysmeta-' + uni_str: [uni_str, ts_str]})
self.assertEqual(
get_test_meta('PUT', {'x-account-meta-' + wsgi_str: wsgi_str}),
{u'X-Account-Meta-' + uni_str: [uni_str, ts_str]})
self.assertEqual(
get_test_meta('POST', {'x-account-sysmeta-' + wsgi_str: wsgi_str}),
{u'X-Account-Sysmeta-' + uni_str: [uni_str, ts_str]})
self.assertEqual(
get_test_meta('POST', {'x-account-meta-' + wsgi_str: wsgi_str}),
{u'X-Account-Meta-' + uni_str: [uni_str, ts_str]})
def test_PUT_GET_metadata(self):
# Set metadata header
req = Request.blank(

19
test/unit/common/middleware/s3api/test_bucket.py
View File

@ -84,6 +84,9 @@ class TestS3ApiBucket(S3ApiTestCase):
'HEAD', '/v1/AUTH_test/junk', swob.HTTPNoContent, {}, None)
self.swift.register(
'HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound, {}, None)
self.swift.register(
'HEAD', '/v1/AUTH_test/unavailable', swob.HTTPServiceUnavailable,
{}, None)
self.swift.register(
'GET', '/v1/AUTH_test/junk', swob.HTTPOk,
{'Content-Type': 'application/json'}, object_list)
@ -127,6 +130,15 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual(status.split()[0], '404')
self.assertEqual(body, b'') # sanity
def test_bucket_HEAD_503(self):
req = Request.blank('/unavailable',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '503')
self.assertEqual(body, b'') # sanity
def test_bucket_HEAD_slash(self):
req = Request.blank('/junk/',
environ={'REQUEST_METHOD': 'HEAD'},
@ -151,6 +163,9 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual(code, 'AccessDenied')
code = self._test_method_error('GET', '/bucket', swob.HTTPNotFound)
self.assertEqual(code, 'NoSuchBucket')
code = self._test_method_error('GET', '/bucket',
swob.HTTPServiceUnavailable)
self.assertEqual(code, 'ServiceUnavailable')
code = self._test_method_error('GET', '/bucket', swob.HTTPServerError)
self.assertEqual(code, 'InternalError')
@ -621,7 +636,7 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual(code, 'InternalError')
code = self._test_method_error(
'PUT', '/bucket', swob.HTTPServiceUnavailable)
self.assertEqual(code, 'InternalError')
self.assertEqual(code, 'ServiceUnavailable')
code = self._test_method_error(
'PUT', '/bucket+bucket', swob.HTTPCreated)
self.assertEqual(code, 'InvalidBucketName')
@ -673,7 +688,7 @@ class TestS3ApiBucket(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '500 Internal Server Error')
self.assertEqual(status, '503 Service Unavailable')
# The last call was PUT not POST for acl set
self.assertEqual(self.swift.calls, [
('PUT', '/v1/AUTH_test/bucket'),

22
test/unit/common/middleware/s3api/test_multi_delete.py
View File

@ -18,6 +18,7 @@ import json
import unittest
from datetime import datetime
from hashlib import md5
import mock
from swift.common import swob
from swift.common.swob import Request
@ -320,6 +321,27 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'MalformedXML')
@s3acl
def test_object_multi_DELETE_unhandled_exception(self):
exploding_resp = mock.MagicMock(
side_effect=Exception('kaboom'))
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
exploding_resp, {}, None)
elem = Element('Delete')
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = 'Key1'
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
def _test_object_multi_DELETE(self, account):
self.keys = ['Key1', 'Key2']
self.swift.register(

12
test/unit/common/middleware/s3api/test_obj.py
View File

@ -161,7 +161,7 @@ class TestS3ApiObj(S3ApiTestCase):
self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
swob.HTTPServiceUnavailable, {}, None)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '500')
self.assertEqual(status.split()[0], '503')
self.assertEqual(body, b'') # sanity
def test_object_HEAD(self):
@ -281,7 +281,7 @@ class TestS3ApiObj(S3ApiTestCase):
self.assertEqual(code, 'PreconditionFailed')
code = self._test_method_error('GET', '/bucket/object',
swob.HTTPServiceUnavailable)
self.assertEqual(code, 'InternalError')
self.assertEqual(code, 'ServiceUnavailable')
@s3acl
def test_object_GET(self):
@ -398,7 +398,7 @@ class TestS3ApiObj(S3ApiTestCase):
self.assertEqual(code, 'InternalError')
code = self._test_method_error('PUT', '/bucket/object',
swob.HTTPServiceUnavailable)
self.assertEqual(code, 'InternalError')
self.assertEqual(code, 'ServiceUnavailable')
code = self._test_method_error('PUT', '/bucket/object',
swob.HTTPCreated,
{'X-Amz-Copy-Source': ''})
@ -582,9 +582,9 @@ class TestS3ApiObj(S3ApiTestCase):
self.assertEqual('200 ', status[:4], body)
# Check that s3api does not return an etag header,
# specified copy source.
self.assertTrue(headers.get('etag') is None)
self.assertNotIn('etag', headers)
# Check that s3api does not return custom metadata in response
self.assertTrue(headers.get('x-amz-meta-something') is None)
self.assertNotIn('x-amz-meta-something', headers)
_, _, headers = self.swift.calls_with_headers[-1]
# Check that s3api converts a Content-MD5 header into an etag.
@ -964,7 +964,7 @@ class TestS3ApiObj(S3ApiTestCase):
self.assertEqual(code, 'InternalError')
code = self._test_method_error('DELETE', '/bucket/object',
swob.HTTPServiceUnavailable)
self.assertEqual(code, 'InternalError')
self.assertEqual(code, 'ServiceUnavailable')
with patch(
'swift.common.middleware.s3api.s3request.get_container_info',

6
test/unit/common/middleware/test_dlo.py
View File

@ -478,14 +478,14 @@ class TestDloGetManifest(DloTestCase):
def test_get_multi_range(self):
# DLO doesn't support multi-range GETs. The way that you express that
# in HTTP is to return a 200 response containing the whole entity.
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=5-9,15-19'})
with mock.patch(LIMIT, 3):
with mock.patch(LIMIT, 30):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertEqual(headers.get("Content-Length"), '25')
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')

221
test/unit/common/middleware/test_slo.py
View File

@ -54,7 +54,9 @@ def fake_start_response(*args, **kwargs):
def md5hex(s):
return hashlib.md5(s.encode('ascii')).hexdigest()
if not isinstance(s, bytes):
s = s.encode('ascii')
return hashlib.md5(s).hexdigest()
class SloTestCase(unittest.TestCase):
@ -1237,6 +1239,32 @@ class TestSloDeleteManifest(SloTestCase):
'DELETE', '/v1/AUTH_test/deltest-unauth/q_17',
swob.HTTPUnauthorized, {}, None)
self.app.register(
'GET', '/v1/AUTH_test/deltest/manifest-with-too-many-segs',
swob.HTTPOk, {'Content-Type': 'application/json',
'X-Static-Large-Object': 'true'},
json.dumps([{'name': '/deltest/a_1',
'hash': 'a', 'bytes': '1'},
{'name': '/deltest/multi-submanifest', 'sub_slo': True,
'hash': 'submanifest-etag',
'bytes': len(_submanifest_data)},
{'name': '/deltest/b_2',
'hash': 'b', 'bytes': '1'},
{'name': '/deltest/c_3',
'hash': 'c', 'bytes': '1'},
{'name': '/deltest/d_4',
'hash': 'b', 'bytes': '1'},
{'name': '/deltest/e_5',
'hash': 'c', 'bytes': '1'},
{'name': '/deltest/f_6',
'hash': 'b', 'bytes': '1'},
{'name': '/deltest/g_8',
'hash': 'c', 'bytes': '1'},
{'name': '/deltest/g_8',
'hash': 'c', 'bytes': '1'},
{'name': '/deltest/h_9',
'hash': 'd', 'bytes': '3'}]))
def test_handle_multipart_delete_man(self):
req = Request.blank(
'/v1/AUTH_test/deltest/man',
@ -1363,11 +1391,11 @@ class TestSloDeleteManifest(SloTestCase):
def test_handle_multipart_delete_nested_too_many_segments(self):
req = Request.blank(
'/v1/AUTH_test/deltest/manifest-with-submanifest?' +
'/v1/AUTH_test/deltest/manifest-with-too-many-segs?' +
'multipart-manifest=delete',
environ={'REQUEST_METHOD': 'DELETE',
'HTTP_ACCEPT': 'application/json'})
with patch.object(slo, 'MAX_BUFFERED_SLO_SEGMENTS', 1):
with patch.object(self.slo, 'max_manifest_segments', 1):
status, headers, body = self.call_slo(req)
self.assertEqual(status, '200 OK')
resp_data = json.loads(body)
@ -1532,12 +1560,13 @@ class TestSloHeadOldManifest(SloTestCase):
'hash': 'seg02-hash',
'content_type': 'text/plain',
'last_modified': '2013-11-19T11:33:45.137447'}])
self.manifest_json_etag = md5hex(manifest_json)
manifest_headers = {
'Content-Length': str(len(manifest_json)),
'Content-Type': 'test/data',
'X-Static-Large-Object': 'true',
'X-Object-Sysmeta-Artisanal-Etag': 'bespoke',
'Etag': md5hex(manifest_json)}
'Etag': self.manifest_json_etag}
manifest_headers.update(getattr(self, 'extra_manifest_headers', {}))
self.manifest_has_sysmeta = all(h in manifest_headers for h in (
'X-Object-Sysmeta-Slo-Etag', 'X-Object-Sysmeta-Slo-Size'))
@ -1553,6 +1582,7 @@ class TestSloHeadOldManifest(SloTestCase):
self.assertEqual(status, '200 OK')
self.assertIn(('Etag', '"%s"' % self.slo_etag), headers)
self.assertIn(('X-Manifest-Etag', self.manifest_json_etag), headers)
self.assertIn(('Content-Length', '300'), headers)
self.assertIn(('Content-Type', 'test/data'), headers)
self.assertEqual(body, b'') # it's a HEAD request, after all
@ -1829,11 +1859,12 @@ class TestSloGetManifest(SloTestCase):
'bytes': 25},
{'name': '/gettest/d_20', 'hash': md5hex("d" * 20),
'content_type': 'text/plain', 'bytes': '20'}])
self.abcd_manifest_json_etag = md5hex(_abcd_manifest_json)
self.app.register(
'GET', '/v1/AUTH_test/gettest/manifest-abcd',
swob.HTTPOk, {'Content-Type': 'application/json',
'X-Static-Large-Object': 'true',
'Etag': md5hex(_abcd_manifest_json)},
'Etag': self.abcd_manifest_json_etag},
_abcd_manifest_json)
# A submanifest segment is created using the response headers from a
@ -1995,9 +2026,8 @@ class TestSloGetManifest(SloTestCase):
status, headers, body = self.call_slo(req)
self.assertEqual(status, '200 OK')
self.assertTrue(
('Content-Type', 'application/json; charset=utf-8') in headers,
headers)
self.assertIn(
('Content-Type', 'application/json; charset=utf-8'), headers)
try:
resp_data = json.loads(body)
except ValueError:
@ -2010,6 +2040,7 @@ class TestSloGetManifest(SloTestCase):
{'hash': md5hex('c' * 15), 'bytes': '15', 'name': '/gettest/c_15',
'content_type': 'text/plain'}],
body)
self.assertIn(('Etag', md5hex(body)), headers)
def test_get_nonmanifest_passthrough(self):
req = Request.blank(
@ -2167,6 +2198,8 @@ class TestSloGetManifest(SloTestCase):
self.assertEqual(status, '200 OK')
self.assertEqual(headers['Content-Length'], '50')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.abcd_manifest_json_etag)
self.assertEqual(
body, b'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
@ -2402,6 +2435,164 @@ class TestSloGetManifest(SloTestCase):
('GET',
'/v1/AUTH_test/gettest/big_seg?multipart-manifest=get')])
def test_range_get_beyond_manifest_refetch_fails(self):
big = 'e' * 1024 * 1024
big_etag = md5hex(big)
big_manifest = json.dumps(
[{'name': '/gettest/big_seg', 'hash': big_etag,
'bytes': 1024 * 1024, 'content_type': 'application/foo'}])
self.app.register_responses(
'GET', '/v1/AUTH_test/gettest/big_manifest',
[(swob.HTTPOk, {'Content-Type': 'application/octet-stream',
'X-Static-Large-Object': 'true',
'X-Backend-Timestamp': '1234',
'Etag': md5hex(big_manifest)},
big_manifest),
(swob.HTTPNotFound, {}, None)])
req = Request.blank(
'/v1/AUTH_test/gettest/big_manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=100000-199999'})
status, headers, body = self.call_slo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '503 Service Unavailable')
self.assertNotIn('X-Static-Large-Object', headers)
self.assertEqual(self.app.calls, [
# has Range header, gets 416
('GET', '/v1/AUTH_test/gettest/big_manifest'),
# retry the first one
('GET', '/v1/AUTH_test/gettest/big_manifest'),
])
def test_range_get_beyond_manifest_refetch_finds_old(self):
big = 'e' * 1024 * 1024
big_etag = md5hex(big)
big_manifest = json.dumps(
[{'name': '/gettest/big_seg', 'hash': big_etag,
'bytes': 1024 * 1024, 'content_type': 'application/foo'}])
self.app.register_responses(
'GET', '/v1/AUTH_test/gettest/big_manifest',
[(swob.HTTPOk, {'Content-Type': 'application/octet-stream',
'X-Static-Large-Object': 'true',
'X-Backend-Timestamp': '1234',
'Etag': md5hex(big_manifest)},
big_manifest),
(swob.HTTPOk, {'X-Backend-Timestamp': '1233'}, [b'small body'])])
req = Request.blank(
'/v1/AUTH_test/gettest/big_manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=100000-199999'})
status, headers, body = self.call_slo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '503 Service Unavailable')
self.assertNotIn('X-Static-Large-Object', headers)
self.assertEqual(self.app.calls, [
# has Range header, gets 416
('GET', '/v1/AUTH_test/gettest/big_manifest'),
# retry the first one
('GET', '/v1/AUTH_test/gettest/big_manifest'),
])
def test_range_get_beyond_manifest_refetch_small_non_slo(self):
big = 'e' * 1024 * 1024
big_etag = md5hex(big)
big_manifest = json.dumps(
[{'name': '/gettest/big_seg', 'hash': big_etag,
'bytes': 1024 * 1024, 'content_type': 'application/foo'}])
self.app.register_responses(
'GET', '/v1/AUTH_test/gettest/big_manifest',
[(swob.HTTPOk, {'Content-Type': 'application/octet-stream',
'X-Static-Large-Object': 'true',
'X-Backend-Timestamp': '1234',
'Etag': md5hex(big_manifest)},
big_manifest),
(swob.HTTPOk, {'X-Backend-Timestamp': '1235'}, [b'small body'])])
req = Request.blank(
'/v1/AUTH_test/gettest/big_manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=100000-199999'})
status, headers, body = self.call_slo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '416 Requested Range Not Satisfiable')
self.assertNotIn('X-Static-Large-Object', headers)
self.assertEqual(self.app.calls, [
# has Range header, gets 416
('GET', '/v1/AUTH_test/gettest/big_manifest'),
# retry the first one
('GET', '/v1/AUTH_test/gettest/big_manifest'),
])
def test_range_get_beyond_manifest_refetch_big_non_slo(self):
big = 'e' * 1024 * 1024
big_etag = md5hex(big)
big_manifest = json.dumps(
[{'name': '/gettest/big_seg', 'hash': big_etag,
'bytes': 1024 * 1024, 'content_type': 'application/foo'}])
self.app.register_responses(
'GET', '/v1/AUTH_test/gettest/big_manifest',
[(swob.HTTPOk, {'Content-Type': 'application/octet-stream',
'X-Static-Large-Object': 'true',
'X-Backend-Timestamp': '1234',
'Etag': md5hex(big_manifest)},
big_manifest),
(swob.HTTPOk, {'X-Backend-Timestamp': '1235'},
[b'x' * 1024 * 1024])])
req = Request.blank(
'/v1/AUTH_test/gettest/big_manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=100000-199999'})
status, headers, body = self.call_slo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK') # NOT 416 or 206!
self.assertNotIn('X-Static-Large-Object', headers)
self.assertEqual(len(body), 1024 * 1024)
self.assertEqual(body, b'x' * 1024 * 1024)
self.assertEqual(self.app.calls, [
# has Range header, gets 416
('GET', '/v1/AUTH_test/gettest/big_manifest'),
# retry the first one
('GET', '/v1/AUTH_test/gettest/big_manifest'),
])
def test_range_get_beyond_manifest_refetch_tombstone(self):
big = 'e' * 1024 * 1024
big_etag = md5hex(big)
big_manifest = json.dumps(
[{'name': '/gettest/big_seg', 'hash': big_etag,
'bytes': 1024 * 1024, 'content_type': 'application/foo'}])
self.app.register_responses(
'GET', '/v1/AUTH_test/gettest/big_manifest',
[(swob.HTTPOk, {'Content-Type': 'application/octet-stream',
'X-Static-Large-Object': 'true',
'X-Backend-Timestamp': '1234',
'Etag': md5hex(big_manifest)},
big_manifest),
(swob.HTTPNotFound, {'X-Backend-Timestamp': '1345'}, None)])
req = Request.blank(
'/v1/AUTH_test/gettest/big_manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=100000-199999'})
status, headers, body = self.call_slo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '404 Not Found')
self.assertNotIn('X-Static-Large-Object', headers)
self.assertEqual(self.app.calls, [
# has Range header, gets 416
('GET', '/v1/AUTH_test/gettest/big_manifest'),
# retry the first one
('GET', '/v1/AUTH_test/gettest/big_manifest'),
])
def test_range_get_bogus_content_range(self):
# Just a little paranoia; Swift currently sends back valid
# Content-Range headers, but if somehow someone sneaks an invalid one
@ -2857,6 +3048,8 @@ class TestSloGetManifest(SloTestCase):
self.assertEqual(status, '200 OK')
self.assertEqual(headers['Content-Length'], '50')
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag)
self.assertEqual(headers['X-Manifest-Etag'],
self.abcd_manifest_json_etag)
self.assertEqual(body, b'')
# Note the lack of recursive descent into manifest-bc. We know the
# content-length from the outer manifest, so there's no need for any
@ -3614,11 +3807,12 @@ class TestSloConditionalGetOldManifest(SloTestCase):
_bc_manifest_json)
_abcd_manifest_json = json.dumps(self.slo_data)
self.abcd_manifest_json_etag = md5hex(_abcd_manifest_json)
manifest_headers = {
'Content-Length': str(len(_abcd_manifest_json)),
'Content-Type': 'application/json',
'X-Static-Large-Object': 'true',
'Etag': md5hex(_abcd_manifest_json),
'Etag': self.abcd_manifest_json_etag,
'X-Object-Sysmeta-Custom-Etag': 'a custom etag'}
manifest_headers.update(getattr(self, 'extra_manifest_headers', {}))
self.manifest_has_sysmeta = all(h in manifest_headers for h in (
@ -3927,7 +4121,7 @@ class TestSloConditionalGetNewManifest(TestSloConditionalGetOldManifest):
super(TestSloConditionalGetNewManifest, self).setUp()
class TestSloBulkLogger(unittest.TestCase):
class TestSloBulkDeleter(unittest.TestCase):
def test_reused_logger(self):
slo_mware = slo.filter_factory({})('fake app')
self.assertTrue(slo_mware.logger is slo_mware.bulk_deleter.logger)
@ -3936,6 +4130,13 @@ class TestSloBulkLogger(unittest.TestCase):
slo_mware = slo.filter_factory({'delete_concurrency': 5})('fake app')
self.assertEqual(5, slo_mware.bulk_deleter.delete_concurrency)
def test_uses_big_max_deletes(self):
slo_mware = slo.filter_factory(
{'max_manifest_segments': 123456789})('fake app')
self.assertGreaterEqual(
slo_mware.bulk_deleter.max_deletes_per_request,
123456789)
class TestSwiftInfo(unittest.TestCase):
def setUp(self):

433
test/unit/common/middleware/test_symlink.py
View File

@ -15,6 +15,7 @@
# limitations under the License.
import unittest
import io
import json
import mock
@ -23,7 +24,7 @@ from swift.common import swob
from swift.common.middleware import symlink, copy, versioned_writes, \
listing_formats
from swift.common.swob import Request
from swift.common.utils import MD5_OF_EMPTY_STRING
from swift.common.utils import MD5_OF_EMPTY_STRING, get_swift_info
from test.unit.common.middleware.helpers import FakeSwift
from test.unit.common.middleware.test_versioned_writes import FakeCache
@ -77,6 +78,14 @@ class TestSymlinkMiddlewareBase(unittest.TestCase):
class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
def test_symlink_info(self):
swift_info = get_swift_info()
self.assertEqual(swift_info['symlink'], {
'symloop_max': 2,
'static_links': True,
})
def test_symlink_simple_put(self):
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
@ -90,6 +99,196 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
self.assertNotIn('X-Object-Sysmeta-Symlink-Target-Account', hdrs)
val = hdrs.get('X-Object-Sysmeta-Container-Update-Override-Etag')
self.assertEqual(val, '%s; symlink_target=c1/o' % MD5_OF_EMPTY_STRING)
self.assertEqual('application/symlink', hdrs.get('Content-Type'))
def test_symlink_simple_put_with_content_type(self):
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={'X-Symlink-Target': 'c1/o',
'Content-Type': 'application/linkyfoo'},
body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[0]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertNotIn('X-Object-Sysmeta-Symlink-Target-Account', hdrs)
val = hdrs.get('X-Object-Sysmeta-Container-Update-Override-Etag')
self.assertEqual(val, '%s; symlink_target=c1/o' % MD5_OF_EMPTY_STRING)
self.assertEqual('application/linkyfoo', hdrs.get('Content-Type'))
def test_symlink_simple_put_with_etag(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, {
'Etag': 'tgt-etag', 'Content-Length': 42,
'Content-Type': 'application/foo'})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'tgt-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[1]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertNotIn('X-Object-Sysmeta-Symlink-Target-Account', hdrs)
val = hdrs.get('X-Object-Sysmeta-Container-Update-Override-Etag')
self.assertEqual(val, '%s; symlink_target=c1/o; '
'symlink_target_etag=tgt-etag; '
'symlink_target_bytes=42' % MD5_OF_EMPTY_STRING)
self.assertEqual([
('HEAD', '/v1/a/c1/o'),
('PUT', '/v1/a/c/symlink'),
], self.app.calls)
self.assertEqual('application/foo',
self.app._calls[-1].headers['Content-Type'])
def test_symlink_simple_put_with_etag_target_missing_content_type(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, {
'Etag': 'tgt-etag', 'Content-Length': 42})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'tgt-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[1]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertNotIn('X-Object-Sysmeta-Symlink-Target-Account', hdrs)
val = hdrs.get('X-Object-Sysmeta-Container-Update-Override-Etag')
self.assertEqual(val, '%s; symlink_target=c1/o; '
'symlink_target_etag=tgt-etag; '
'symlink_target_bytes=42' % MD5_OF_EMPTY_STRING)
self.assertEqual([
('HEAD', '/v1/a/c1/o'),
('PUT', '/v1/a/c/symlink'),
], self.app.calls)
# N.B. the ObjectController would call _update_content_type on PUT
# regardless, but you actually can't get a HEAD response without swob
# setting a Content-Type
self.assertEqual('text/html; charset=UTF-8',
self.app._calls[-1].headers['Content-Type'])
def test_symlink_simple_put_with_etag_explicit_content_type(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, {
'Etag': 'tgt-etag', 'Content-Length': 42,
'Content-Type': 'application/foo'})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'tgt-etag',
'Content-Type': 'application/bar',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[1]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertNotIn('X-Object-Sysmeta-Symlink-Target-Account', hdrs)
val = hdrs.get('X-Object-Sysmeta-Container-Update-Override-Etag')
self.assertEqual(val, '%s; symlink_target=c1/o; '
'symlink_target_etag=tgt-etag; '
'symlink_target_bytes=42' % MD5_OF_EMPTY_STRING)
self.assertEqual([
('HEAD', '/v1/a/c1/o'),
('PUT', '/v1/a/c/symlink'),
], self.app.calls)
self.assertEqual('application/bar',
self.app._calls[-1].headers['Content-Type'])
def test_symlink_simple_put_with_unmatched_etag(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, {
'Etag': 'tgt-etag', 'Content-Length': 42})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'not-tgt-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '409 Conflict')
self.assertIn(('Content-Location', '/v1/a/c1/o'), headers)
self.assertEqual(body, b"Object Etag 'tgt-etag' does not match "
b"X-Symlink-Target-Etag header 'not-tgt-etag'")
def test_symlink_simple_put_to_non_existing_object(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPNotFound, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'not-tgt-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '409 Conflict')
self.assertIn(('Content-Location', '/v1/a/c1/o'), headers)
self.assertIn(b'does not exist', body)
def test_symlink_put_with_prevalidated_etag(self):
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT', headers={
'X-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Etag': 'tgt-etag',
'X-Object-Sysmeta-Symlink-Target-Bytes': '13',
'Content-Type': 'application/foo',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
self.assertEqual([
# N.B. no HEAD!
('PUT', '/v1/a/c/symlink'),
], self.app.calls)
self.assertEqual('application/foo',
self.app._calls[-1].headers['Content-Type'])
method, path, hdrs = self.app.calls_with_headers[0]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertNotIn('X-Object-Sysmeta-Symlink-Target-Account', hdrs)
val = hdrs.get('X-Object-Sysmeta-Container-Update-Override-Etag')
self.assertEqual(val, '%s; symlink_target=c1/o; '
'symlink_target_etag=tgt-etag; '
'symlink_target_bytes=13' % MD5_OF_EMPTY_STRING)
def test_symlink_put_with_prevalidated_etag_sysmeta_incomplete(self):
req = Request.blank('/v1/a/c/symlink', method='PUT', headers={
'X-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Etag': 'tgt-etag',
}, body='')
with self.assertRaises(KeyError) as cm:
self.call_sym(req)
self.assertEqual(cm.exception.args[0], swob.header_to_environ_key(
'X-Object-Sysmeta-Symlink-Target-Bytes'))
def test_symlink_chunked_put(self):
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={'X-Symlink-Target': 'c1/o'},
environ={'wsgi.input': io.BytesIO(b'')})
self.assertIsNone(req.content_length) # sanity
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
method, path, hdrs = self.app.calls_with_headers[0]
val = hdrs.get('X-Object-Sysmeta-Symlink-Target')
self.assertEqual(val, 'c1/o')
self.assertNotIn('X-Object-Sysmeta-Symlink-Target-Account', hdrs)
val = hdrs.get('X-Object-Sysmeta-Container-Update-Override-Etag')
self.assertEqual(val, '%s; symlink_target=c1/o' % MD5_OF_EMPTY_STRING)
def test_symlink_chunked_put_error(self):
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={'X-Symlink-Target': 'c1/o'},
environ={'wsgi.input':
io.BytesIO(b'this has a body')})
self.assertIsNone(req.content_length) # sanity
status, headers, body = self.call_sym(req)
self.assertEqual(status, '400 Bad Request')
def test_symlink_put_different_account(self):
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
@ -248,6 +447,64 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
self.assertNotIn('X-Symlink-Target-Account', dict(headers))
self.assertNotIn('Content-Location', dict(headers))
def test_get_static_link_mismatched_etag(self):
self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
'X-Object-Sysmeta-Symlink-Target-Etag': 'the-etag'})
# apparently target object was overwritten
self.app.register('GET', '/v1/a/c1/o', swob.HTTPOk,
{'ETag': 'not-the-etag'}, 'resp_body')
req = Request.blank('/v1/a/c/symlink', method='GET')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '409 Conflict')
self.assertEqual(body, b"Object Etag 'not-the-etag' does not "
b"match X-Symlink-Target-Etag header 'the-etag'")
def test_get_static_link_to_symlink(self):
self.app.register('GET', '/v1/a/c/static_link', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c/symlink',
'X-Object-Sysmeta-Symlink-Target-Etag': 'the-etag'})
self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk,
{'ETag': 'the-etag',
'X-Object-Sysmeta-Symlink-Target': 'c1/o'})
self.app.register('GET', '/v1/a/c1/o', swob.HTTPOk,
{'ETag': 'not-the-etag'}, 'resp_body')
req = Request.blank('/v1/a/c/static_link', method='GET')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '200 OK')
def test_get_static_link_to_symlink_fails(self):
self.app.register('GET', '/v1/a/c/static_link', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c/symlink',
'X-Object-Sysmeta-Symlink-Target-Etag': 'the-etag'})
self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk,
{'ETag': 'not-the-etag',
'X-Object-Sysmeta-Symlink-Target': 'c1/o'})
req = Request.blank('/v1/a/c/static_link', method='GET')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '409 Conflict')
self.assertEqual(body, b"X-Symlink-Target-Etag headers do not match")
def put_static_link_to_symlink(self):
self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk,
{'ETag': 'symlink-etag',
'X-Object-Sysmeta-Symlink-Target': 'c/o',
'Content-Type': 'application/symlink'})
self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk,
{'ETag': 'tgt-etag',
'Content-Type': 'application/data'}, 'resp_body')
self.app.register('PUT', '/v1/a/c/static_link', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/static_link', method='PUT',
headers={
'X-Symlink-Target': 'c/symlink',
'X-Symlink-Target-Etag': 'symlink-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
self.assertEqual([], self.app.calls)
self.assertEqual('application/data',
self.app._calls[-1].headers['Content-Type'])
def test_head_symlink(self):
self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c1/o',
@ -298,15 +555,21 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
self.assertFalse(calls[2:])
def test_symlink_too_deep(self):
self.app.register('HEAD', '/v1/a/c/symlink', swob.HTTPOk,
self.app.register('GET', '/v1/a/c/symlink', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c/sym1'})
self.app.register('HEAD', '/v1/a/c/sym1', swob.HTTPOk,
self.app.register('GET', '/v1/a/c/sym1', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c/sym2'})
self.app.register('HEAD', '/v1/a/c/sym2', swob.HTTPOk,
self.app.register('GET', '/v1/a/c/sym2', swob.HTTPOk,
{'X-Object-Sysmeta-Symlink-Target': 'c/o'})
req = Request.blank('/v1/a/c/symlink', method='HEAD')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '409 Conflict')
self.assertEqual(body, b'')
req = Request.blank('/v1/a/c/symlink')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '409 Conflict')
self.assertEqual(body, b'Too many levels of symbolic links, '
b'maximum allowed is 2')
def test_symlink_change_symloopmax(self):
# similar test to test_symlink_too_deep, but now changed the limit to 3
@ -390,11 +653,11 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
status, headers, body = self.call_sym(req)
self.assertEqual(status, '404 Not Found')
def test_check_symlink_header(self):
def test_validate_and_prep_request_headers(self):
def do_test(headers):
req = Request.blank('/v1/a/c/o', method='PUT',
headers=headers)
symlink._check_symlink_header(req)
symlink._validate_and_prep_request_headers(req)
# normal cases
do_test({'X-Symlink-Target': 'c1/o1'})
@ -419,12 +682,12 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
{'X-Symlink-Target': 'cont/obj',
'X-Symlink-Target-Account': swob.wsgi_quote(target)})
def test_check_symlink_header_invalid_format(self):
def test_validate_and_prep_request_headers_invalid_format(self):
def do_test(headers, status, err_msg):
req = Request.blank('/v1/a/c/o', method='PUT',
headers=headers)
with self.assertRaises(swob.HTTPException) as cm:
symlink._check_symlink_header(req)
symlink._validate_and_prep_request_headers(req)
self.assertEqual(cm.exception.status, status)
self.assertEqual(cm.exception.body, err_msg)
@ -484,11 +747,11 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
'412 Precondition Failed',
b'Account name cannot contain slashes')
def test_check_symlink_header_points_to_itself(self):
def test_validate_and_prep_request_headers_points_to_itself(self):
req = Request.blank('/v1/a/c/o', method='PUT',
headers={'X-Symlink-Target': 'c/o'})
with self.assertRaises(swob.HTTPException) as cm:
symlink._check_symlink_header(req)
symlink._validate_and_prep_request_headers(req)
self.assertEqual(cm.exception.status, '400 Bad Request')
self.assertEqual(cm.exception.body, b'Symlink cannot target itself')
@ -497,7 +760,7 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
headers={'X-Symlink-Target': 'c/o',
'X-Symlink-Target-Account': 'a'})
with self.assertRaises(swob.HTTPException) as cm:
symlink._check_symlink_header(req)
symlink._validate_and_prep_request_headers(req)
self.assertEqual(cm.exception.status, '400 Bad Request')
self.assertEqual(cm.exception.body, b'Symlink cannot target itself')
@ -505,7 +768,7 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
req = Request.blank('/v1/a/c/o', method='PUT',
headers={'X-Symlink-Target': 'c/o',
'X-Symlink-Target-Account': 'a1'})
symlink._check_symlink_header(req)
symlink._validate_and_prep_request_headers(req)
def test_symloop_max_config(self):
self.app = FakeSwift()
@ -665,6 +928,145 @@ class SymlinkCopyingTestCase(TestSymlinkMiddlewareBase):
self.assertEqual(
hdrs.get('X-Object-Sysmeta-Symlink-Target-Account'), 'a2')
def test_static_link_to_new_slo_manifest(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, {
'X-Static-Large-Object': 'True',
'Etag': 'manifest-etag',
'X-Object-Sysmeta-Slo-Size': '1048576',
'X-Object-Sysmeta-Slo-Etag': 'this-is-not-used',
'Content-Length': 42,
'Content-Type': 'application/big-data',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'956859738870e5ca6aa17eeda58e4df0; '
'slo_etag=71e938d37c1d06dc634dd24660255a88',
})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'manifest-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
self.assertEqual([
('HEAD', '/v1/a/c1/o'),
('PUT', '/v1/a/c/symlink'),
], self.app.calls)
method, path, hdrs = self.app.calls_with_headers[-1]
self.assertEqual('application/big-data', hdrs['Content-Type'])
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target'], 'c1/o')
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target-Etag'],
'manifest-etag')
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target-Bytes'],
'1048576')
self.assertEqual(
hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'],
'd41d8cd98f00b204e9800998ecf8427e; '
'slo_etag=71e938d37c1d06dc634dd24660255a88; '
'symlink_target=c1/o; '
'symlink_target_etag=manifest-etag; '
'symlink_target_bytes=1048576')
def test_static_link_to_old_slo_manifest(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, {
'X-Static-Large-Object': 'True',
'Etag': 'manifest-etag',
'X-Object-Sysmeta-Slo-Size': '1048576',
'X-Object-Sysmeta-Slo-Etag': '71e938d37c1d06dc634dd24660255a88',
'Content-Length': 42,
'Content-Type': 'application/big-data',
})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'manifest-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
self.assertEqual([
('HEAD', '/v1/a/c1/o'),
('PUT', '/v1/a/c/symlink'),
], self.app.calls)
method, path, hdrs = self.app.calls_with_headers[-1]
self.assertEqual('application/big-data', hdrs['Content-Type'])
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target'], 'c1/o')
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target-Etag'],
'manifest-etag')
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target-Bytes'],
'1048576')
self.assertEqual(
hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'],
'd41d8cd98f00b204e9800998ecf8427e; '
'slo_etag=71e938d37c1d06dc634dd24660255a88; '
'symlink_target=c1/o; '
'symlink_target_etag=manifest-etag; '
'symlink_target_bytes=1048576')
def test_static_link_to_really_old_slo_manifest(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, {
'X-Static-Large-Object': 'True',
'Etag': 'manifest-etag',
'Content-Length': 42,
'Content-Type': 'application/big-data',
})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'manifest-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
self.assertEqual([
('HEAD', '/v1/a/c1/o'),
('PUT', '/v1/a/c/symlink'),
], self.app.calls)
method, path, hdrs = self.app.calls_with_headers[-1]
self.assertEqual('application/big-data', hdrs['Content-Type'])
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target'], 'c1/o')
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target-Etag'],
'manifest-etag')
# symlink m/w is doing a HEAD, it's not going to going to read the
# manifest body and sum up the bytes - so we just use manifest size
self.assertEqual(hdrs['X-Object-Sysmeta-Symlink-Target-Bytes'],
'42')
# no slo_etag, and target_bytes is manifest
self.assertEqual(
hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'],
'd41d8cd98f00b204e9800998ecf8427e; '
'symlink_target=c1/o; '
'symlink_target_etag=manifest-etag; '
'symlink_target_bytes=42')
def test_static_link_to_slo_manifest_slo_etag(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPOk, {
'Etag': 'manifest-etag',
'X-Object-Sysmeta-Slo-Etag': 'slo-etag',
'Content-Length': 42,
})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
# unquoted slo-etag doesn't match
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'slo-etag',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '409 Conflict')
# the quoted slo-etag is just straight up invalid
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': '"slo-etag"',
}, body='')
status, headers, body = self.call_sym(req)
self.assertEqual(status, '400 Bad Request')
self.assertEqual(b'Bad X-Symlink-Target-Etag format', body)
class SymlinkVersioningTestCase(TestSymlinkMiddlewareBase):
# verify interaction of versioned_writes and symlink middlewares
@ -793,13 +1195,16 @@ class TestSymlinkContainerContext(TestSymlinkMiddlewareBase):
def test_extract_symlink_path_json_symlink_path(self):
obj_dict = {"bytes": 6,
"last_modified": "1",
"hash": "etag; symlink_target=c/o",
"hash": "etag; symlink_target=c/o; something_else=foo; "
"symlink_target_etag=tgt_etag; symlink_target_bytes=8",
"name": "obj",
"content_type": "application/octet-stream"}
obj_dict = self.context._extract_symlink_path_json(
obj_dict, 'v1', 'AUTH_a')
self.assertEqual(obj_dict['hash'], 'etag')
self.assertEqual(obj_dict['hash'], 'etag; something_else=foo')
self.assertEqual(obj_dict['symlink_path'], '/v1/AUTH_a/c/o')
self.assertEqual(obj_dict['symlink_etag'], 'tgt_etag')
self.assertEqual(obj_dict['symlink_bytes'], 8)
def test_extract_symlink_path_json_symlink_path_and_account(self):
obj_dict = {

34
test/unit/common/middleware/test_versioned_writes.py
View File

@ -417,7 +417,7 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
self.assertRequestEqual(req, self.authorized[1])
self.assertEqual(3, self.app.call_count)
self.assertEqual([
('GET', '/v1/a/c/o'),
('GET', '/v1/a/c/o?symlink=get'),
('PUT', '/v1/a/ver_cont/001o/0000000060.00000'),
('PUT', '/v1/a/c/o'),
], self.app.calls)
@ -449,7 +449,7 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
self.assertRequestEqual(req, self.authorized[1])
self.assertEqual(3, self.app.call_count)
self.assertEqual([
('GET', '/v1/a/c/o'),
('GET', '/v1/a/c/o?symlink=get'),
('PUT', '/v1/a/ver_cont/001o/0000003600.00000'),
('PUT', '/v1/a/c/o'),
], self.app.calls)
@ -682,7 +682,7 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
prefix_listing_prefix = '/v1/a/ver_cont?prefix=001o/&'
self.assertEqual(self.app.calls, [
('GET', prefix_listing_prefix + 'marker=&reverse=on'),
('GET', '/v1/a/ver_cont/001o/2'),
('GET', '/v1/a/ver_cont/001o/2?symlink=get'),
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/2'),
])
@ -777,7 +777,7 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
self.assertEqual(self.app.calls, [
('GET', prefix_listing_prefix + 'marker=&reverse=on'),
('HEAD', '/v1/a/c/o'),
('GET', '/v1/a/ver_cont/001o/1'),
('GET', '/v1/a/ver_cont/001o/1?symlink=get'),
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/1'),
('DELETE', '/v1/a/ver_cont/001o/2'),
@ -941,7 +941,7 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
prefix_listing_prefix = '/v1/a/ver_cont?prefix=001o/&'
self.assertEqual(self.app.calls, [
('GET', prefix_listing_prefix + 'marker=&reverse=on'),
('GET', '/v1/a/ver_cont/001o/1'),
('GET', '/v1/a/ver_cont/001o/1?symlink=get'),
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/1'),
])
@ -989,8 +989,8 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
prefix_listing_prefix = '/v1/a/ver_cont?prefix=001o/&'
self.assertEqual(self.app.calls, [
('GET', prefix_listing_prefix + 'marker=&reverse=on'),
('GET', '/v1/a/ver_cont/001o/2'),
('GET', '/v1/a/ver_cont/001o/1'),
('GET', '/v1/a/ver_cont/001o/2?symlink=get'),
('GET', '/v1/a/ver_cont/001o/1?symlink=get'),
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/1'),
])
@ -1114,7 +1114,7 @@ class VersionedWritesOldContainersTestCase(VersionedWritesBaseTestCase):
self.assertEqual(self.app.calls, [
('GET', prefix_listing_prefix + 'marker=&reverse=on'),
('GET', prefix_listing_prefix + 'marker=001o/2'),
('GET', '/v1/a/ver_cont/001o/2'),
('GET', '/v1/a/ver_cont/001o/2?symlink=get'),
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/2'),
])
@ -1167,8 +1167,8 @@ class VersionedWritesOldContainersTestCase(VersionedWritesBaseTestCase):
self.assertEqual(self.app.calls, [
('GET', prefix_listing_prefix + 'marker=&reverse=on'),
('GET', prefix_listing_prefix + 'marker=001o/2'),
('GET', '/v1/a/ver_cont/001o/2'),
('GET', '/v1/a/ver_cont/001o/1'),
('GET', '/v1/a/ver_cont/001o/2?symlink=get'),
('GET', '/v1/a/ver_cont/001o/1?symlink=get'),
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/1'),
])
@ -1282,14 +1282,14 @@ class VersionedWritesOldContainersTestCase(VersionedWritesBaseTestCase):
prefix_listing_prefix = '/v1/a/ver_cont?prefix=001o/&'
self.assertEqual(self.app.calls, [
('GET', prefix_listing_prefix + 'marker=&reverse=on'),
('GET', '/v1/a/ver_cont/001o/4'),
('GET', '/v1/a/ver_cont/001o/3'),
('GET', '/v1/a/ver_cont/001o/2'),
('GET', '/v1/a/ver_cont/001o/4?symlink=get'),
('GET', '/v1/a/ver_cont/001o/3?symlink=get'),
('GET', '/v1/a/ver_cont/001o/2?symlink=get'),
('GET', prefix_listing_prefix + 'marker=001o/2&reverse=on'),
('GET', prefix_listing_prefix + 'marker=&end_marker=001o/2'),
('GET', prefix_listing_prefix + 'marker=001o/0&end_marker=001o/2'),
('GET', prefix_listing_prefix + 'marker=001o/1&end_marker=001o/2'),
('GET', '/v1/a/ver_cont/001o/1'),
('GET', '/v1/a/ver_cont/001o/1?symlink=get'),
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/1'),
])
@ -1354,13 +1354,13 @@ class VersionedWritesOldContainersTestCase(VersionedWritesBaseTestCase):
prefix_listing_prefix = '/v1/a/ver_cont?prefix=001o/&'
self.assertEqual(self.app.calls, [
('GET', prefix_listing_prefix + 'marker=&reverse=on'),
('GET', '/v1/a/ver_cont/001o/4'),
('GET', '/v1/a/ver_cont/001o/3'),
('GET', '/v1/a/ver_cont/001o/4?symlink=get'),
('GET', '/v1/a/ver_cont/001o/3?symlink=get'),
('GET', prefix_listing_prefix + 'marker=001o/3&reverse=on'),
('GET', prefix_listing_prefix + 'marker=&end_marker=001o/3'),
('GET', prefix_listing_prefix + 'marker=001o/1&end_marker=001o/3'),
('GET', prefix_listing_prefix + 'marker=001o/2&end_marker=001o/3'),
('GET', '/v1/a/ver_cont/001o/2'),
('GET', '/v1/a/ver_cont/001o/2?symlink=get'),
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/2'),
])

15
test/unit/common/test_header_key_dict.py
View File

@ -15,6 +15,7 @@
import unittest
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.swob import bytes_to_wsgi
class TestHeaderKeyDict(unittest.TestCase):
@ -27,6 +28,20 @@ class TestHeaderKeyDict(unittest.TestCase):
self.assertEqual(headers['content-length'], '20')
self.assertEqual(headers['CONTENT-LENGTH'], '20')
def test_unicode(self):
def mkstr(prefix):
return bytes_to_wsgi((prefix + u'\U0001f44d').encode('utf8'))
headers = HeaderKeyDict()
headers[mkstr('x-object-meta-')] = 'ok'
self.assertIn(mkstr('x-object-meta-'), headers)
self.assertIn(mkstr('X-Object-Meta-'), headers)
self.assertIn(mkstr('X-OBJECT-META-'), headers)
keys = list(headers)
self.assertNotIn(mkstr('x-object-meta-'), keys)
self.assertIn(mkstr('X-Object-Meta-'), keys)
self.assertNotIn(mkstr('X-OBJECT-META-'), keys)
def test_setdefault(self):
headers = HeaderKeyDict()

4
test/unit/common/test_internal_client.py
View File

@ -107,7 +107,7 @@ class GetMetadataInternalClient(internal_client.InternalClient):
self.metadata = 'some_metadata'
def _get_metadata(self, path, metadata_prefix, acceptable_statuses=None,
headers=None):
headers=None, params=None):
self.get_metadata_called += 1
self.test.assertEqual(self.path, path)
self.test.assertEqual(self.metadata_prefix, metadata_prefix)
@ -649,7 +649,7 @@ class TestInternalClient(unittest.TestCase):
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
body_file=None, params=None):
self.make_request_called += 1
self.test.assertEqual('HEAD', method)
self.test.assertEqual(self.path, path)

59
test/unit/common/test_utils.py
View File

@ -5939,6 +5939,13 @@ class TestAuditLocationGenerator(unittest.TestCase):
class TestGreenAsyncPile(unittest.TestCase):
def setUp(self):
self.timeout = Timeout(5.0)
def tearDown(self):
self.timeout.cancel()
def test_runs_everything(self):
def run_test():
tests_ran[0] += 1
@ -6045,6 +6052,58 @@ class TestGreenAsyncPile(unittest.TestCase):
# pending remains 0
self.assertEqual(0, pile._pending)
def _exploder(self, arg):
if isinstance(arg, Exception):
raise arg
else:
return arg
def test_blocking_last_next_explodes(self):
pile = utils.GreenAsyncPile(2)
pile.spawn(self._exploder, 1)
pile.spawn(self._exploder, 2)
pile.spawn(self._exploder, Exception('kaboom'))
self.assertEqual(1, next(pile))
self.assertEqual(2, next(pile))
with self.assertRaises(StopIteration):
next(pile)
self.assertEqual(pile.inflight, 0)
self.assertEqual(pile._pending, 0)
def test_no_blocking_last_next_explodes(self):
pile = utils.GreenAsyncPile(10)
pile.spawn(self._exploder, 1)
self.assertEqual(1, next(pile))
pile.spawn(self._exploder, 2)
self.assertEqual(2, next(pile))
pile.spawn(self._exploder, Exception('kaboom'))
with self.assertRaises(StopIteration):
next(pile)
self.assertEqual(pile.inflight, 0)
self.assertEqual(pile._pending, 0)
def test_exceptions_in_streaming_pile(self):
with utils.StreamingPile(2) as pile:
results = list(pile.asyncstarmap(self._exploder, [
(1,),
(Exception('kaboom'),),
(3,),
]))
self.assertEqual(results, [1, 3])
self.assertEqual(pile.inflight, 0)
self.assertEqual(pile._pending, 0)
def test_exceptions_at_end_of_streaming_pile(self):
with utils.StreamingPile(2) as pile:
results = list(pile.asyncstarmap(self._exploder, [
(1,),
(2,),
(Exception('kaboom'),),
]))
self.assertEqual(results, [1, 2])
self.assertEqual(pile.inflight, 0)
self.assertEqual(pile._pending, 0)
class TestLRUCache(unittest.TestCase):

2
test/unit/helpers.py
View File

@ -92,7 +92,7 @@ def setup_servers(the_object_server=object_server, extra_conf=None):
conf = {'devices': _testdir, 'swift_dir': _testdir,
'mount_check': 'false', 'allowed_headers':
'content-encoding, x-object-manifest, content-disposition, foo',
'allow_versions': 't'}
'allow_versions': 't', 'node_timeout': 20}
if extra_conf:
conf.update(extra_conf)
prolis = listen_zero()

22
test/unit/obj/test_reconstructor.py
View File

@ -5031,16 +5031,18 @@ class TestReconstructFragmentArchive(BaseTestObjectReconstructor):
archive_bodies = encode_frag_archive_bodies(self.policy, body)
# pop the index to the destination node
archive_bodies.pop(1)
ec_archive_dict[
(md5(body).hexdigest(), next(ts).internal)] = archive_bodies
key = (md5(body).hexdigest(), next(ts).internal, bool(i % 2))
ec_archive_dict[key] = archive_bodies
responses = list()
# fill out response list by 3 different etag bodies
for etag, ts in itertools.cycle(ec_archive_dict):
body = ec_archive_dict[(etag, ts)].pop(0)
for etag, ts, durable in itertools.cycle(ec_archive_dict):
body = ec_archive_dict[(etag, ts, durable)].pop(0)
headers = get_header_frag_index(self, body)
headers.update({'X-Object-Sysmeta-Ec-Etag': etag,
'X-Backend-Timestamp': ts})
if durable:
headers['X-Backend-Durable-Timestamp'] = ts
responses.append((200, body, headers))
if len(responses) >= (self.policy.object_ring.replicas - 1):
break
@ -5063,7 +5065,7 @@ class TestReconstructFragmentArchive(BaseTestObjectReconstructor):
# 1 error log per etag to report not enough responses
self.assertEqual(3, len(error_lines))
for error_line in error_lines:
for expected_etag, ts in ec_archive_dict:
for expected_etag, ts, durable in ec_archive_dict:
if expected_etag in error_line:
break
else:
@ -5072,13 +5074,15 @@ class TestReconstructFragmentArchive(BaseTestObjectReconstructor):
(list(ec_archive_dict), error_line))
# remove the found etag which should not be found in the
# following error lines
del ec_archive_dict[(expected_etag, ts)]
del ec_archive_dict[(expected_etag, ts, durable)]
expected = 'Unable to get enough responses (%s/10) to ' \
'reconstruct 10.0.0.1:1001/sdb/0%s policy#0 ' \
'frag#1 with ETag' % \
'reconstruct %s 10.0.0.1:1001/sdb/0%s policy#0 ' \
'frag#1 with ETag %s and timestamp %s' % \
(etag_count[expected_etag],
self.obj_path.decode('utf8'))
'durable' if durable else 'non-durable',
self.obj_path.decode('utf8'),
expected_etag, ts)
self.assertIn(
expected, error_line,
"Unexpected error line found: Expected: %s Got: %s"

4
test/unit/obj/test_replicator.py
View File

@ -19,6 +19,7 @@ import os
import mock
from gzip import GzipFile
from shutil import rmtree
import six
import six.moves.cPickle as pickle
import time
import tempfile
@ -124,7 +125,8 @@ def _mock_process(ret):
MockProcess.captured_log = captured_log
orig_process = subprocess.Popen
MockProcess.ret_code = (i[0] for i in ret)
MockProcess.ret_log = (i[1] for i in ret)
MockProcess.ret_log = (i[1] if six.PY2 else i[1].encode('utf8')
for i in ret)
MockProcess.check_args = (i[2] for i in ret)
object_replicator.subprocess.Popen = MockProcess
yield captured_log

15
test/unit/obj/test_updater.py
View File

@ -38,6 +38,7 @@ from swift.obj.diskfile import (
from swift.common.ring import RingData
from swift.common import utils
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.swob import bytes_to_wsgi
from swift.common.utils import (
hash_path, normalize_timestamp, mkdirs, write_pickle)
from swift.common.storage_policy import StoragePolicy, POLICIES
@ -504,13 +505,13 @@ class TestObjectUpdater(unittest.TestCase):
self.assertEqual(inc.readline(),
b'PUT /sda1/0/a/c/o HTTP/1.1\r\n')
headers = HeaderKeyDict()
line = inc.readline()
while line and line != b'\r\n':
headers[line.split(b':')[0]] = \
line.split(b':')[1].strip()
line = inc.readline()
self.assertIn(b'x-container-timestamp', headers)
self.assertIn(b'X-Backend-Storage-Policy-Index',
line = bytes_to_wsgi(inc.readline())
while line and line != '\r\n':
headers[line.split(':')[0]] = \
line.split(':')[1].strip()
line = bytes_to_wsgi(inc.readline())
self.assertIn('x-container-timestamp', headers)
self.assertIn('X-Backend-Storage-Policy-Index',
headers)
except BaseException as err:
return err

129
test/unit/proxy/controllers/test_obj.py
View File

@ -1570,6 +1570,45 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
def test_GET_primaries_explode(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
codes = [Exception('kaboom!')] * self.obj_ring.replicas + (
[404] * self.obj_ring.max_more_nodes)
with set_http_connect(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
def test_GET_primaries_timeout(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
codes = [Timeout()] * self.obj_ring.replicas + (
[404] * self.obj_ring.max_more_nodes)
with set_http_connect(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
def test_GET_primaries_mixed_explode_and_timeout(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
primaries = []
for i in range(self.obj_ring.replicas):
if i % 2:
primaries.append(Timeout())
else:
primaries.append(Exception('kaboom!'))
codes = primaries + [404] * self.obj_ring.max_more_nodes
with set_http_connect(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
def test_primary_returns_some_nonsense_timestamp(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
# an un-handled ValueError in _make_node_request should just continue
# to the next node rather than hang the request
headers = [{'X-Backend-Timestamp': 'not-a-timestamp'}, {}]
codes = [200, 200]
with set_http_connect(*codes, headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
def test_GET_not_found_when_404_newer(self):
# if proxy receives a 404, it keeps waiting for other connections until
# max number of nodes in hopes of finding an object, but if 404 is
@ -2136,7 +2175,7 @@ class ECObjectControllerMixin(CommonObjectControllerMixin):
{'obj': obj1, 'frag': 13},
]
# ... and the rests are 404s which is limited by request_count
# ... and the rest are 404s which is limited by request_count
# (2 * replicas in default) rather than max_extra_requests limitation
# because the retries will be in ResumingGetter if the responses
# are 404s
@ -2147,7 +2186,7 @@ class ECObjectControllerMixin(CommonObjectControllerMixin):
with capture_http_requests(fake_response) as log:
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
self.assertEqual(resp.status_int, 503)
# expect a request to all nodes
self.assertEqual(2 * self.replicas(), len(log))
@ -2693,7 +2732,7 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
with capture_http_requests(fake_response) as log:
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
self.assertEqual(resp.status_int, 503)
collected_responses = defaultdict(set)
for conn in log:
@ -2792,7 +2831,7 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
collected_indexes[fi].append(conn)
self.assertEqual(len(collected_indexes), 7)
def test_GET_with_mixed_frags_and_no_quorum_will_503(self):
def test_GET_with_mixed_nondurable_frags_and_no_quorum_will_503(self):
# all nodes have a frag but there is no one set that reaches quorum,
# which means there is no backend 404 response, but proxy should still
# return 404 rather than 503
@ -2801,6 +2840,68 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
obj3 = self._make_ec_object_stub(pattern='obj3')
obj4 = self._make_ec_object_stub(pattern='obj4')
node_frags = [
{'obj': obj1, 'frag': 0, 'durable': False},
{'obj': obj2, 'frag': 0, 'durable': False},
{'obj': obj3, 'frag': 0, 'durable': False},
{'obj': obj1, 'frag': 1, 'durable': False},
{'obj': obj2, 'frag': 1, 'durable': False},
{'obj': obj3, 'frag': 1, 'durable': False},
{'obj': obj1, 'frag': 2, 'durable': False},
{'obj': obj2, 'frag': 2, 'durable': False},
{'obj': obj3, 'frag': 2, 'durable': False},
{'obj': obj1, 'frag': 3, 'durable': False},
{'obj': obj2, 'frag': 3, 'durable': False},
{'obj': obj3, 'frag': 3, 'durable': False},
{'obj': obj1, 'frag': 4, 'durable': False},
{'obj': obj2, 'frag': 4, 'durable': False},
{'obj': obj3, 'frag': 4, 'durable': False},
{'obj': obj1, 'frag': 5, 'durable': False},
{'obj': obj2, 'frag': 5, 'durable': False},
{'obj': obj3, 'frag': 5, 'durable': False},
{'obj': obj1, 'frag': 6, 'durable': False},
{'obj': obj2, 'frag': 6, 'durable': False},
{'obj': obj3, 'frag': 6, 'durable': False},
{'obj': obj1, 'frag': 7, 'durable': False},
{'obj': obj2, 'frag': 7, 'durable': False},
{'obj': obj3, 'frag': 7, 'durable': False},
{'obj': obj1, 'frag': 8, 'durable': False},
{'obj': obj2, 'frag': 8, 'durable': False},
{'obj': obj3, 'frag': 8, 'durable': False},
{'obj': obj4, 'frag': 8, 'durable': False},
]
fake_response = self._fake_ec_node_response(node_frags)
req = swob.Request.blank('/v1/a/c/o')
with capture_http_requests(fake_response) as log:
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
collected_etags = set()
collected_status = set()
for conn in log:
etag = conn.resp.headers['X-Object-Sysmeta-Ec-Etag']
collected_etags.add(etag)
collected_status.add(conn.resp.status)
# default node_iter will exhaust at 2 * replicas
self.assertEqual(len(log), 2 * self.replicas())
self.assertEqual(
{obj1['etag'], obj2['etag'], obj3['etag'], obj4['etag']},
collected_etags)
self.assertEqual({200}, collected_status)
def test_GET_with_mixed_frags_and_no_quorum_will_503(self):
# all nodes have a frag but there is no one set that reaches quorum,
# but since they're all marked durable (so we *should* be able to
# reconstruct), proxy will 503
obj1 = self._make_ec_object_stub(pattern='obj1')
obj2 = self._make_ec_object_stub(pattern='obj2')
obj3 = self._make_ec_object_stub(pattern='obj3')
obj4 = self._make_ec_object_stub(pattern='obj4')
node_frags = [
{'obj': obj1, 'frag': 0},
{'obj': obj2, 'frag': 0},
@ -3229,9 +3330,9 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
with capture_http_requests(fake_response) as log:
resp = req.get_response(self.app)
# read body to provoke any EC decode errors
self.assertFalse(resp.body)
self.assertTrue(resp.body)
self.assertEqual(resp.status_int, 404)
self.assertEqual(resp.status_int, 503)
self.assertEqual(len(log), self.replicas() * 2)
collected_etags = set()
for conn in log:
@ -3240,7 +3341,10 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
self.assertEqual({obj1['etag'], obj2['etag'], None}, collected_etags)
log_lines = self.app.logger.get_lines_for_level('error')
self.assertEqual(log_lines,
['Problem with fragment response: ETag mismatch'] * 7)
['Problem with fragment response: ETag mismatch'] * 7
+ ['Object returning 503 for []'])
# Note the empty list above -- that log line comes out of
# best_response but we've already thrown out the "good" responses :-/
def test_GET_mixed_success_with_range(self):
fragment_size = self.policy.fragment_size
@ -3926,7 +4030,7 @@ class TestECDuplicationObjController(
{'obj': obj1, 'frag': 8},
{'obj': obj2, 'frag': 8},
]
# ... and the rests are 404s which is limited by request_count
# ... and the rest are 404s which is limited by request_count
# (2 * replicas in default) rather than max_extra_requests limitation
# because the retries will be in ResumingGetter if the responses
# are 404s
@ -3937,7 +4041,7 @@ class TestECDuplicationObjController(
with capture_http_requests(fake_response) as log:
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
self.assertEqual(resp.status_int, 503)
collected_responses = defaultdict(set)
for conn in log:
@ -4267,9 +4371,9 @@ class TestECDuplicationObjController(
with capture_http_requests(fake_response) as log:
resp = req.get_response(self.app)
# read body to provoke any EC decode errors
self.assertFalse(resp.body)
self.assertTrue(resp.body)
self.assertEqual(resp.status_int, 404)
self.assertEqual(resp.status_int, 503)
self.assertEqual(len(log), self.replicas() * 2)
collected_etags = set()
for conn in log:
@ -4278,7 +4382,8 @@ class TestECDuplicationObjController(
self.assertEqual({obj1['etag'], obj2['etag'], None}, collected_etags)
log_lines = self.app.logger.get_lines_for_level('error')
self.assertEqual(log_lines,
['Problem with fragment response: ETag mismatch'] * 7)
['Problem with fragment response: ETag mismatch'] * 7
+ ['Object returning 503 for []'])
def _test_determine_chunk_destinations_prioritize(
self, missing_two, missing_one):

24
test/unit/proxy/test_server.py
View File

@ -3975,6 +3975,13 @@ class TestReplicatedObjectController(
test_status_map((200, 200, 404, 404, 500), 404)
test_status_map((200, 200, 500, 500, 500), 503)
POLICIES.default.object_ring.max_more_nodes = 3
test_status_map(
(200, 200,
Timeout(), Timeout(), Timeout(), # Can't reach primaries
404, 404, 404), # No surprise: handoffs know nothing
503)
def test_HEAD_newest(self):
with save_globals():
def test_status_map(statuses, expected, timestamps,
@ -7527,7 +7534,7 @@ class TestECGets(unittest.TestCase):
}
resp = self._setup_nodes_and_do_GET(objs, node_state)
self.assertEqual(resp.status_int, 503)
self.assertEqual(resp.status_int, 404)
def test_GET_with_multiple_frags_per_node(self):
# verify object GET behavior when multiple fragments are on same node
@ -7632,13 +7639,26 @@ class TestECGets(unittest.TestCase):
# will be sent frag prefs that exclude frag_index 1)
node_state = {
0: [dict(ref='obj1a', frag_index=1, durable=False)],
1: [dict(ref='obj1b', frag_index=1, durable=True)],
1: [dict(ref='obj1b', frag_index=1, durable=False)],
2: [dict(ref='obj1c', frag_index=1, durable=False)]
}
resp = self._setup_nodes_and_do_GET(objs, node_state)
self.assertEqual(resp.status_int, 404)
# if we know it should be durable, we can be more specific.
# note that we need to set *both* of those first ones durable
# to avoid a flaky test -- in the future we can be smarter and
# let the durability bubble up, even from a duplicate frag
node_state = {
0: [dict(ref='obj1a', frag_index=1, durable=True)],
1: [dict(ref='obj1b', frag_index=1, durable=True)],
2: [dict(ref='obj1c', frag_index=1, durable=False)]
}
resp = self._setup_nodes_and_do_GET(objs, node_state)
self.assertEqual(resp.status_int, 503)
class TestObjectDisconnectCleanup(unittest.TestCase):

33
tox.ini
View File

@ -48,18 +48,49 @@ commands = ./.functests {posargs}
basepython = python3
commands =
nosetests {posargs: \
test/functional/s3api/test_acl.py \
test/functional/s3api/test_multi_delete.py \
test/functional/s3api/test_multi_upload.py \
test/functional/s3api/test_object.py \
test/functional/s3api/test_presigned.py \
test/functional/s3api/test_service.py \
test/functional/test_access_control.py \
test/functional/test_account.py \
test/functional/test_container.py \
test/functional/test_dlo.py \
test/functional/test_domain_remap.py \
test/functional/test_object.py \
test/functional/test_slo.py \
test/functional/test_staticweb.py \
test/functional/test_symlink.py \
test/functional/test_tempurl.py \
test/functional/test_versioned_writes.py \
test/functional/tests.py}
[testenv:func-ec-py3]
basepython = python3
commands = {[testenv:func-py3]commands}
setenv = SWIFT_TEST_IN_PROCESS=1
SWIFT_TEST_IN_PROCESS_CONF_LOADER=ec
[testenv:func-s3api-py3]
basepython = python3
commands = {[testenv:func-py3]commands}
setenv = SWIFT_TEST_IN_PROCESS=1
SWIFT_TEST_IN_PROCESS_CONF_LOADER=s3api
[testenv:func-encryption-py3]
basepython = python3
commands = {[testenv:func-py3]commands}
setenv = SWIFT_TEST_IN_PROCESS=1
SWIFT_TEST_IN_PROCESS_CONF_LOADER=encryption
[testenv:func-domain-remap-staticweb-py3]
basepython = python3
commands = {[testenv:func-py3]commands}
setenv = SWIFT_TEST_IN_PROCESS=1
SWIFT_TEST_IN_PROCESS_CONF_LOADER=domain_remap_staticweb
[testenv:func-encryption]
commands = ./.functests {posargs}
setenv = SWIFT_TEST_IN_PROCESS=1
@ -95,7 +126,7 @@ commands = sphinx-build -W -b html doc/source doc/build/html
[testenv:api-ref]
# This environment is called from CI scripts to test and publish
# the API Ref to developer.openstack.org.
# the API Ref to docs.openstack.org.
basepython = python3
deps = -r{toxinidir}/doc/requirements.txt
commands =