Merge remote-tracking branch 'remotes/origin/master' into merge-master
Conflicts resolved: .zuul.yaml requirements.txt swift/obj/replicator.py Change-Id: I8c988890173f8e038f3285ac63537f5bf8698937
This commit is contained in:
Kota Tsuyuzaki
commit
d391957fd7
|
|
@ -331,6 +331,7 @@
|
||||||
irrelevant-files:
|
irrelevant-files:
|
||||||
- ^(api-ref|doc|releasenotes)/.*$
|
- ^(api-ref|doc|releasenotes)/.*$
|
||||||
- ^test/(functional|probe)/.*$
|
- ^test/(functional|probe)/.*$
|
||||||
|
voting: false
|
||||||
- swift-tox-py36:
|
- swift-tox-py36:
|
||||||
irrelevant-files:
|
irrelevant-files:
|
||||||
- ^(api-ref|doc|releasenotes)/.*$
|
- ^(api-ref|doc|releasenotes)/.*$
|
||||||
|
|
@ -340,7 +341,6 @@
|
||||||
irrelevant-files:
|
irrelevant-files:
|
||||||
- ^(api-ref|doc|releasenotes)/.*$
|
- ^(api-ref|doc|releasenotes)/.*$
|
||||||
- ^test/(functional|probe)/.*$
|
- ^test/(functional|probe)/.*$
|
||||||
voting: false
|
|
||||||
- swift-tox-func:
|
- swift-tox-func:
|
||||||
irrelevant-files:
|
irrelevant-files:
|
||||||
- ^(api-ref|doc|releasenotes)/.*$
|
- ^(api-ref|doc|releasenotes)/.*$
|
||||||
|
|
@ -412,7 +412,7 @@
|
||||||
# long-running jobs, like probetests or (once they move to
|
# long-running jobs, like probetests or (once they move to
|
||||||
# in-tree definitions) dsvm jobs.
|
# in-tree definitions) dsvm jobs.
|
||||||
- swift-tox-py27
|
- swift-tox-py27
|
||||||
- swift-tox-py35
|
- swift-tox-py37
|
||||||
- swift-tox-func
|
- swift-tox-func
|
||||||
- swift-tox-func-encryption
|
- swift-tox-func-encryption
|
||||||
- swift-tox-func-domain-remap-staticweb
|
- swift-tox-func-domain-remap-staticweb
|
||||||
|
|
|
||||||
|
|
@ -1888,7 +1888,7 @@ swift (1.13.1, OpenStack Icehouse)
|
||||||
A new proxy config variable (strict_cors_mode, default to True)
|
A new proxy config variable (strict_cors_mode, default to True)
|
||||||
has been added. Setting it to False keeps the old behavior. For
|
has been added. Setting it to False keeps the old behavior. For
|
||||||
an overview of old versus new behavior, please see
|
an overview of old versus new behavior, please see
|
||||||
https://review.openstack.org/#/c/69419/
|
https://review.opendev.org/#/c/69419/
|
||||||
|
|
||||||
* Invert the responsibility of the two instances of proxy-logging in
|
* Invert the responsibility of the two instances of proxy-logging in
|
||||||
the proxy pipeline
|
the proxy pipeline
|
||||||
|
|
|
||||||
|
|
@ -52,7 +52,7 @@ Reviewing Someone Else's Code
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
All code reviews in OpenStack projects are done on
|
All code reviews in OpenStack projects are done on
|
||||||
https://review.openstack.org/. Reviewing patches is one of the most effective
|
https://review.opendev.org/. Reviewing patches is one of the most effective
|
||||||
ways you can contribute to the community.
|
ways you can contribute to the community.
|
||||||
|
|
||||||
We've written REVIEW_GUIDELINES.rst (found in this source tree) to help you
|
We've written REVIEW_GUIDELINES.rst (found in this source tree) to help you
|
||||||
|
|
|
||||||
|
|
@ -162,6 +162,11 @@ try:
|
||||||
except OSError:
|
except OSError:
|
||||||
warnings.warn('Cannot get last updated time from git repository. '
|
warnings.warn('Cannot get last updated time from git repository. '
|
||||||
'Not setting "html_last_updated_fmt".')
|
'Not setting "html_last_updated_fmt".')
|
||||||
|
else:
|
||||||
|
if not isinstance(html_last_updated_fmt, str):
|
||||||
|
# for py3
|
||||||
|
html_last_updated_fmt = html_last_updated_fmt.decode('ascii')
|
||||||
|
|
||||||
|
|
||||||
# If true, SmartyPants will be used to convert quotes and dashes to
|
# If true, SmartyPants will be used to convert quotes and dashes to
|
||||||
# typographically correct entities.
|
# typographically correct entities.
|
||||||
|
|
|
||||||
|
|
@ -1109,6 +1109,8 @@ multipart-manifest_put:
|
||||||
description: |
|
description: |
|
||||||
If you include the ``multipart-manifest=put`` query parameter, the object
|
If you include the ``multipart-manifest=put`` query parameter, the object
|
||||||
is a static large object manifest and the body contains the manifest.
|
is a static large object manifest and the body contains the manifest.
|
||||||
|
See `Static large objects <https://docs.openstack.org/swift/latest
|
||||||
|
/api/large_objects.html#static-large-objects>`_ for more information.
|
||||||
in: query
|
in: query
|
||||||
required: false
|
required: false
|
||||||
type: string
|
type: string
|
||||||
|
|
|
||||||
|
|
@ -2252,7 +2252,7 @@ For distros with more recent kernels (for example Ubuntu 12.04 Precise),
|
||||||
we recommend using the default settings (including the default inode size
|
we recommend using the default settings (including the default inode size
|
||||||
of 256 bytes) when creating the file system::
|
of 256 bytes) when creating the file system::
|
||||||
|
|
||||||
mkfs.xfs /dev/sda1
|
mkfs.xfs -L D1 /dev/sda1
|
||||||
|
|
||||||
In the last couple of years, XFS has made great improvements in how inodes
|
In the last couple of years, XFS has made great improvements in how inodes
|
||||||
are allocated and used. Using the default inode size no longer has an
|
are allocated and used. Using the default inode size no longer has an
|
||||||
|
|
@ -2262,7 +2262,7 @@ For distros with older kernels (for example Ubuntu 10.04 Lucid),
|
||||||
some settings can dramatically impact performance. We recommend the
|
some settings can dramatically impact performance. We recommend the
|
||||||
following when creating the file system::
|
following when creating the file system::
|
||||||
|
|
||||||
mkfs.xfs -i size=1024 /dev/sda1
|
mkfs.xfs -i size=1024 -L D1 /dev/sda1
|
||||||
|
|
||||||
Setting the inode size is important, as XFS stores xattr data in the inode.
|
Setting the inode size is important, as XFS stores xattr data in the inode.
|
||||||
If the metadata is too large to fit in the inode, a new extent is created,
|
If the metadata is too large to fit in the inode, a new extent is created,
|
||||||
|
|
@ -2272,15 +2272,15 @@ headroom.
|
||||||
|
|
||||||
The following example mount options are recommended when using XFS::
|
The following example mount options are recommended when using XFS::
|
||||||
|
|
||||||
mount -t xfs -o noatime,nodiratime,nobarrier,logbufs=8 /dev/sda1 /srv/node/sda
|
mount -t xfs -o noatime,nodiratime,nobarrier,logbufs=8 -L D1 /srv/node/d1
|
||||||
|
|
||||||
We do not recommend running Swift on RAID, but if you are using
|
We do not recommend running Swift on RAID, but if you are using
|
||||||
RAID it is also important to make sure that the proper sunit and swidth
|
RAID it is also important to make sure that the proper sunit and swidth
|
||||||
settings get set so that XFS can make most efficient use of the RAID array.
|
settings get set so that XFS can make most efficient use of the RAID array.
|
||||||
|
|
||||||
For a standard Swift install, all data drives are mounted directly under
|
For a standard Swift install, all data drives are mounted directly under
|
||||||
``/srv/node`` (as can be seen in the above example of mounting ``/dev/sda1`` as
|
``/srv/node`` (as can be seen in the above example of mounting label ``D1``
|
||||||
``/srv/node/sda``). If you choose to mount the drives in another directory,
|
as ``/srv/node/d1``). If you choose to mount the drives in another directory,
|
||||||
be sure to set the `devices` config option in all of the server configs to
|
be sure to set the `devices` config option in all of the server configs to
|
||||||
point to the correct directory.
|
point to the correct directory.
|
||||||
|
|
||||||
|
|
@ -2322,7 +2322,7 @@ The following settings should be in `/etc/sysctl.conf`::
|
||||||
# double amount of allowed conntrack
|
# double amount of allowed conntrack
|
||||||
net.ipv4.netfilter.ip_conntrack_max = 262144
|
net.ipv4.netfilter.ip_conntrack_max = 262144
|
||||||
|
|
||||||
To load the updated sysctl settings, run ``sudo sysctl -p``
|
To load the updated sysctl settings, run ``sudo sysctl -p``.
|
||||||
|
|
||||||
A note about changing the TIME_WAIT values. By default the OS will hold
|
A note about changing the TIME_WAIT values. By default the OS will hold
|
||||||
a port open for 60 seconds to ensure that any remaining packets can be
|
a port open for 60 seconds to ensure that any remaining packets can be
|
||||||
|
|
|
||||||
|
|
@ -117,7 +117,7 @@ Tracking your changes
|
||||||
---------------------
|
---------------------
|
||||||
|
|
||||||
After proposing changes to Swift, you can track them at
|
After proposing changes to Swift, you can track them at
|
||||||
https://review.openstack.org. After logging in, you will see a dashboard of
|
https://review.opendev.org. After logging in, you will see a dashboard of
|
||||||
"Outgoing reviews" for changes you have proposed, "Incoming reviews" for
|
"Outgoing reviews" for changes you have proposed, "Incoming reviews" for
|
||||||
changes you are reviewing, and "Recently closed" changes for which you were
|
changes you are reviewing, and "Recently closed" changes for which you were
|
||||||
either a reviewer or owner.
|
either a reviewer or owner.
|
||||||
|
|
|
||||||
|
|
@ -47,4 +47,4 @@ Install and configure components
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
# curl -o /etc/swift/proxy-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/proxy-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/proxy-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/proxy-server.conf-sample
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,6 @@ Install and configure components
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
# curl -o /etc/swift/proxy-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/proxy-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/proxy-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/proxy-server.conf-sample
|
||||||
|
|
||||||
3. .. include:: controller-include.txt
|
3. .. include:: controller-include.txt
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,6 @@ Install and configure components
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
# curl -o /etc/swift/proxy-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/proxy-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/proxy-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/proxy-server.conf-sample
|
||||||
|
|
||||||
4. .. include:: controller-include.txt
|
4. .. include:: controller-include.txt
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ This section applies to Red Hat Enterprise Linux 7 and CentOS 7.
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
# curl -o /etc/swift/swift.conf \
|
# curl -o /etc/swift/swift.conf \
|
||||||
https://git.openstack.org/cgit/openstack/swift/plain/etc/swift.conf-sample?h=stable/queens
|
https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/swift.conf-sample
|
||||||
|
|
||||||
#. Edit the ``/etc/swift/swift.conf`` file and complete the following
|
#. Edit the ``/etc/swift/swift.conf`` file and complete the following
|
||||||
actions:
|
actions:
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ This section applies to Ubuntu 14.04 (LTS) and Debian.
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
# curl -o /etc/swift/swift.conf \
|
# curl -o /etc/swift/swift.conf \
|
||||||
https://git.openstack.org/cgit/openstack/swift/plain/etc/swift.conf-sample?h=stable/queens
|
https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/swift.conf-sample
|
||||||
|
|
||||||
#. Edit the ``/etc/swift/swift.conf`` file and complete the following
|
#. Edit the ``/etc/swift/swift.conf`` file and complete the following
|
||||||
actions:
|
actions:
|
||||||
|
|
|
||||||
|
|
@ -133,9 +133,9 @@ Install and configure components
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
# curl -o /etc/swift/account-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/account-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/account-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/account-server.conf-sample
|
||||||
# curl -o /etc/swift/container-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/container-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/container-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/container-server.conf-sample
|
||||||
# curl -o /etc/swift/object-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/object-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/object-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/object-server.conf-sample
|
||||||
|
|
||||||
3. .. include:: storage-include1.txt
|
3. .. include:: storage-include1.txt
|
||||||
4. .. include:: storage-include2.txt
|
4. .. include:: storage-include2.txt
|
||||||
|
|
|
||||||
|
|
@ -137,9 +137,9 @@ Install and configure components
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
# curl -o /etc/swift/account-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/account-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/account-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/account-server.conf-sample
|
||||||
# curl -o /etc/swift/container-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/container-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/container-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/container-server.conf-sample
|
||||||
# curl -o /etc/swift/object-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/object-server.conf-sample?h=stable/queens
|
# curl -o /etc/swift/object-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/object-server.conf-sample
|
||||||
|
|
||||||
3. .. include:: storage-include1.txt
|
3. .. include:: storage-include1.txt
|
||||||
4. .. include:: storage-include2.txt
|
4. .. include:: storage-include2.txt
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ chardet==3.0.4
|
||||||
cliff==2.11.0
|
cliff==2.11.0
|
||||||
cmd2==0.8.1
|
cmd2==0.8.1
|
||||||
coverage==3.6
|
coverage==3.6
|
||||||
cryptography==1.6
|
cryptography==1.8.2
|
||||||
debtcollector==1.19.0
|
debtcollector==1.19.0
|
||||||
dnspython==1.14.0
|
dnspython==1.14.0
|
||||||
docutils==0.11
|
docutils==0.11
|
||||||
|
|
@ -40,7 +40,7 @@ mock==2.0
|
||||||
monotonic==1.4
|
monotonic==1.4
|
||||||
msgpack==0.5.6
|
msgpack==0.5.6
|
||||||
netaddr==0.7.19
|
netaddr==0.7.19
|
||||||
netifaces==0.5
|
netifaces==0.8
|
||||||
nose==1.3.7
|
nose==1.3.7
|
||||||
nosehtmloutput==0.0.3
|
nosehtmloutput==0.0.3
|
||||||
nosexcover==1.0.10
|
nosexcover==1.0.10
|
||||||
|
|
@ -49,6 +49,7 @@ os-api-ref==1.0.0
|
||||||
os-testr==0.8.0
|
os-testr==0.8.0
|
||||||
oslo.config==4.0.0
|
oslo.config==4.0.0
|
||||||
oslo.i18n==3.20.0
|
oslo.i18n==3.20.0
|
||||||
|
oslo.log==3.22.0
|
||||||
oslo.serialization==2.25.0
|
oslo.serialization==2.25.0
|
||||||
oslo.utils==3.36.0
|
oslo.utils==3.36.0
|
||||||
PasteDeploy==1.3.3
|
PasteDeploy==1.3.3
|
||||||
|
|
|
||||||
|
|
@ -2,18 +2,18 @@
|
||||||
# of appearance. Changing the order has an impact on the overall integration
|
# of appearance. Changing the order has an impact on the overall integration
|
||||||
# process, which may cause wedges in the gate later.
|
# process, which may cause wedges in the gate later.
|
||||||
|
|
||||||
dnspython>=1.14.0;python_version=='2.7' # http://www.dnspython.org/LICENSE
|
dnspython>=1.14.0;python_version=='2.7' # http://www.dnspython.org/LICENSE
|
||||||
eventlet>=0.17.4,!=0.23.0 # MIT
|
eventlet>=0.17.4,!=0.23.0 # MIT
|
||||||
greenlet>=0.3.1
|
greenlet>=0.3.1
|
||||||
netifaces>=0.5,!=0.10.0,!=0.10.1
|
netifaces>=0.8,!=0.10.0,!=0.10.1
|
||||||
PasteDeploy>=1.3.3
|
PasteDeploy>=1.3.3
|
||||||
lxml>=3.4.1
|
lxml>=3.4.1
|
||||||
requests>=2.14.2 # Apache-2.0
|
requests>=2.14.2 # Apache-2.0
|
||||||
six>=1.9.0
|
six>=1.9.0
|
||||||
xattr>=0.4
|
xattr>=0.4;sys_platform!='win32' # MIT
|
||||||
PyECLib>=1.3.1 # BSD
|
PyECLib>=1.3.1 # BSD
|
||||||
cryptography!=2.0,>=1.6 # BSD/Apache-2.0
|
cryptography!=2.0,>=1.8.2 # BSD/Apache-2.0
|
||||||
ipaddress>=1.0.16;python_version<'3.3' # PSF
|
ipaddress>=1.0.16;python_version<'3.3' # PSF
|
||||||
# grpcio will fail to work with eventlet starting with 1.3.5.
|
# grpcio will fail to work with eventlet starting with 1.3.5.
|
||||||
# see this for a similar issue with gevent: https://github.com/grpc/grpc/issues/4629 and https://github.com/gevent/gevent/issues/786
|
# see this for a similar issue with gevent: https://github.com/grpc/grpc/issues/4629 and https://github.com/gevent/gevent/issues/786
|
||||||
# don't use eventlet for the object-server ?
|
# don't use eventlet for the object-server ?
|
||||||
|
|
|
||||||
|
|
@ -15,9 +15,7 @@
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import six
|
import six
|
||||||
from six.moves.urllib.parse import unquote
|
from six.moves.urllib.parse import unquote, urlparse
|
||||||
|
|
||||||
from swift.common.utils import urlparse
|
|
||||||
|
|
||||||
|
|
||||||
def clean_acl(name, value):
|
def clean_acl(name, value):
|
||||||
|
|
|
||||||
|
|
@ -121,14 +121,14 @@ Here's an example using ``curl`` with tiny 1-byte segments::
|
||||||
import json
|
import json
|
||||||
|
|
||||||
import six
|
import six
|
||||||
from six.moves.urllib.parse import unquote
|
|
||||||
|
|
||||||
from hashlib import md5
|
from hashlib import md5
|
||||||
from swift.common import constraints
|
from swift.common import constraints
|
||||||
from swift.common.exceptions import ListingIterError, SegmentError
|
from swift.common.exceptions import ListingIterError, SegmentError
|
||||||
from swift.common.http import is_success
|
from swift.common.http import is_success
|
||||||
from swift.common.swob import Request, Response, \
|
from swift.common.swob import Request, Response, \
|
||||||
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict
|
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \
|
||||||
|
str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote
|
||||||
from swift.common.utils import get_logger, \
|
from swift.common.utils import get_logger, \
|
||||||
RateLimitedIterator, quote, close_if_possible, closing_if_possible
|
RateLimitedIterator, quote, close_if_possible, closing_if_possible
|
||||||
from swift.common.request_helpers import SegmentedIterable
|
from swift.common.request_helpers import SegmentedIterable
|
||||||
|
|
@ -143,9 +143,18 @@ class GetContext(WSGIContext):
|
||||||
|
|
||||||
def _get_container_listing(self, req, version, account, container,
|
def _get_container_listing(self, req, version, account, container,
|
||||||
prefix, marker=''):
|
prefix, marker=''):
|
||||||
|
'''
|
||||||
|
:param version: whatever
|
||||||
|
:param account: native
|
||||||
|
:param container: native
|
||||||
|
:param prefix: native
|
||||||
|
:param marker: native
|
||||||
|
'''
|
||||||
con_req = make_subrequest(
|
con_req = make_subrequest(
|
||||||
req.environ,
|
req.environ,
|
||||||
path=quote('/'.join(['', version, account, container])),
|
path=wsgi_quote('/'.join([
|
||||||
|
'', str_to_wsgi(version),
|
||||||
|
str_to_wsgi(account), str_to_wsgi(container)])),
|
||||||
method='GET',
|
method='GET',
|
||||||
headers={'x-auth-token': req.headers.get('x-auth-token')},
|
headers={'x-auth-token': req.headers.get('x-auth-token')},
|
||||||
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
|
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
|
||||||
|
|
@ -156,14 +165,24 @@ class GetContext(WSGIContext):
|
||||||
con_resp = con_req.get_response(self.dlo.app)
|
con_resp = con_req.get_response(self.dlo.app)
|
||||||
if not is_success(con_resp.status_int):
|
if not is_success(con_resp.status_int):
|
||||||
if req.method == 'HEAD':
|
if req.method == 'HEAD':
|
||||||
con_resp.body = ''
|
con_resp.body = b''
|
||||||
return con_resp, None
|
return con_resp, None
|
||||||
with closing_if_possible(con_resp.app_iter):
|
with closing_if_possible(con_resp.app_iter):
|
||||||
return None, json.loads(''.join(con_resp.app_iter))
|
return None, json.loads(b''.join(con_resp.app_iter))
|
||||||
|
|
||||||
def _segment_listing_iterator(self, req, version, account, container,
|
def _segment_listing_iterator(self, req, version, account, container,
|
||||||
prefix, segments, first_byte=None,
|
prefix, segments, first_byte=None,
|
||||||
last_byte=None):
|
last_byte=None):
|
||||||
|
'''
|
||||||
|
:param req: upstream request
|
||||||
|
:param version: native
|
||||||
|
:param account: native
|
||||||
|
:param container: native
|
||||||
|
:param prefix: native
|
||||||
|
:param segments: array of dicts, with native strings
|
||||||
|
:param first_byte: number
|
||||||
|
:param last_byte: number
|
||||||
|
'''
|
||||||
# It's sort of hokey that this thing takes in the first page of
|
# It's sort of hokey that this thing takes in the first page of
|
||||||
# segments as an argument, but we need to compute the etag and content
|
# segments as an argument, but we need to compute the etag and content
|
||||||
# length from the first page, and it's better to have a hokey
|
# length from the first page, and it's better to have a hokey
|
||||||
|
|
@ -173,7 +192,6 @@ class GetContext(WSGIContext):
|
||||||
if last_byte is None:
|
if last_byte is None:
|
||||||
last_byte = float("inf")
|
last_byte = float("inf")
|
||||||
|
|
||||||
marker = ''
|
|
||||||
while True:
|
while True:
|
||||||
for segment in segments:
|
for segment in segments:
|
||||||
seg_length = int(segment['bytes'])
|
seg_length = int(segment['bytes'])
|
||||||
|
|
@ -188,7 +206,7 @@ class GetContext(WSGIContext):
|
||||||
break
|
break
|
||||||
|
|
||||||
seg_name = segment['name']
|
seg_name = segment['name']
|
||||||
if isinstance(seg_name, six.text_type):
|
if six.PY2:
|
||||||
seg_name = seg_name.encode("utf-8")
|
seg_name = seg_name.encode("utf-8")
|
||||||
|
|
||||||
# We deliberately omit the etag and size here;
|
# We deliberately omit the etag and size here;
|
||||||
|
|
@ -227,16 +245,18 @@ class GetContext(WSGIContext):
|
||||||
"Got status %d listing container /%s/%s" %
|
"Got status %d listing container /%s/%s" %
|
||||||
(error_response.status_int, account, container))
|
(error_response.status_int, account, container))
|
||||||
|
|
||||||
def get_or_head_response(self, req, x_object_manifest,
|
def get_or_head_response(self, req, x_object_manifest):
|
||||||
response_headers=None):
|
'''
|
||||||
if response_headers is None:
|
:param req: user's request
|
||||||
response_headers = self._response_headers
|
:param x_object_manifest: as unquoted, native string
|
||||||
|
'''
|
||||||
|
response_headers = self._response_headers
|
||||||
|
|
||||||
container, obj_prefix = x_object_manifest.split('/', 1)
|
container, obj_prefix = x_object_manifest.split('/', 1)
|
||||||
container = unquote(container)
|
|
||||||
obj_prefix = unquote(obj_prefix)
|
|
||||||
|
|
||||||
version, account, _junk = req.split_path(2, 3, True)
|
version, account, _junk = req.split_path(2, 3, True)
|
||||||
|
version = wsgi_to_str(version)
|
||||||
|
account = wsgi_to_str(account)
|
||||||
error_response, segments = self._get_container_listing(
|
error_response, segments = self._get_container_listing(
|
||||||
req, version, account, container, obj_prefix)
|
req, version, account, container, obj_prefix)
|
||||||
if error_response:
|
if error_response:
|
||||||
|
|
@ -311,7 +331,7 @@ class GetContext(WSGIContext):
|
||||||
if h.lower() != "etag"]
|
if h.lower() != "etag"]
|
||||||
etag = md5()
|
etag = md5()
|
||||||
for seg_dict in segments:
|
for seg_dict in segments:
|
||||||
etag.update(seg_dict['hash'].strip('"'))
|
etag.update(seg_dict['hash'].strip('"').encode('utf8'))
|
||||||
response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
|
response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
|
||||||
|
|
||||||
app_iter = None
|
app_iter = None
|
||||||
|
|
@ -353,7 +373,8 @@ class GetContext(WSGIContext):
|
||||||
for header, value in self._response_headers:
|
for header, value in self._response_headers:
|
||||||
if (header.lower() == 'x-object-manifest'):
|
if (header.lower() == 'x-object-manifest'):
|
||||||
close_if_possible(resp_iter)
|
close_if_possible(resp_iter)
|
||||||
response = self.get_or_head_response(req, value)
|
response = self.get_or_head_response(
|
||||||
|
req, wsgi_to_str(wsgi_unquote(value)))
|
||||||
return response(req.environ, start_response)
|
return response(req.environ, start_response)
|
||||||
# Not a dynamic large object manifest; just pass it through.
|
# Not a dynamic large object manifest; just pass it through.
|
||||||
start_response(self._response_status,
|
start_response(self._response_status,
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,8 @@ def _header_strip(value):
|
||||||
# behave as though it wasn't provided
|
# behave as though it wasn't provided
|
||||||
return None
|
return None
|
||||||
return stripped
|
return stripped
|
||||||
|
|
||||||
|
|
||||||
_header_strip.re = re.compile('^[\x00-\x20]*|[\x00-\x20]*$')
|
_header_strip.re = re.compile('^[\x00-\x20]*|[\x00-\x20]*$')
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1427,8 +1429,10 @@ class S3Request(swob.Request):
|
||||||
else:
|
else:
|
||||||
# otherwise we do naive HEAD request with the authentication
|
# otherwise we do naive HEAD request with the authentication
|
||||||
resp = self.get_response(app, 'HEAD', self.container_name, '')
|
resp = self.get_response(app, 'HEAD', self.container_name, '')
|
||||||
|
headers = resp.sw_headers.copy()
|
||||||
|
headers.update(resp.sysmeta_headers)
|
||||||
return headers_to_container_info(
|
return headers_to_container_info(
|
||||||
resp.sw_headers, resp.status_int) # pylint: disable-msg=E1101
|
headers, resp.status_int) # pylint: disable-msg=E1101
|
||||||
|
|
||||||
def gen_multipart_manifest_delete_query(self, app, obj=None):
|
def gen_multipart_manifest_delete_query(self, app, obj=None):
|
||||||
if not self.allow_multipart_uploads:
|
if not self.allow_multipart_uploads:
|
||||||
|
|
|
||||||
|
|
@ -80,7 +80,7 @@ class S3Response(S3ResponseBase, swob.Response):
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
swob.Response.__init__(self, *args, **kwargs)
|
swob.Response.__init__(self, *args, **kwargs)
|
||||||
|
|
||||||
sw_sysmeta_headers = swob.HeaderKeyDict()
|
s3_sysmeta_headers = swob.HeaderKeyDict()
|
||||||
sw_headers = swob.HeaderKeyDict()
|
sw_headers = swob.HeaderKeyDict()
|
||||||
headers = HeaderKeyDict()
|
headers = HeaderKeyDict()
|
||||||
self.is_slo = False
|
self.is_slo = False
|
||||||
|
|
@ -103,12 +103,14 @@ class S3Response(S3ResponseBase, swob.Response):
|
||||||
key = sysmeta_prefix(_server_type) + \
|
key = sysmeta_prefix(_server_type) + \
|
||||||
key[len('x-%s-sysmeta-swift3-' % _server_type):]
|
key[len('x-%s-sysmeta-swift3-' % _server_type):]
|
||||||
|
|
||||||
if key not in sw_sysmeta_headers:
|
if key not in s3_sysmeta_headers:
|
||||||
# To avoid overwrite s3api sysmeta by older swift3
|
# To avoid overwrite s3api sysmeta by older swift3
|
||||||
# sysmeta set the key only when the key does not exist
|
# sysmeta set the key only when the key does not exist
|
||||||
sw_sysmeta_headers[key] = val
|
s3_sysmeta_headers[key] = val
|
||||||
elif is_s3api_sysmeta(key, _server_type):
|
elif is_s3api_sysmeta(key, _server_type):
|
||||||
sw_sysmeta_headers[key] = val
|
s3_sysmeta_headers[key] = val
|
||||||
|
else:
|
||||||
|
sw_headers[key] = val
|
||||||
else:
|
else:
|
||||||
sw_headers[key] = val
|
sw_headers[key] = val
|
||||||
|
|
||||||
|
|
@ -132,7 +134,7 @@ class S3Response(S3ResponseBase, swob.Response):
|
||||||
self.is_slo = config_true_value(val)
|
self.is_slo = config_true_value(val)
|
||||||
|
|
||||||
# Check whether we stored the AWS-style etag on upload
|
# Check whether we stored the AWS-style etag on upload
|
||||||
override_etag = sw_sysmeta_headers.get(
|
override_etag = s3_sysmeta_headers.get(
|
||||||
sysmeta_header('object', 'etag'))
|
sysmeta_header('object', 'etag'))
|
||||||
if override_etag is not None:
|
if override_etag is not None:
|
||||||
# Multipart uploads in AWS have ETags like
|
# Multipart uploads in AWS have ETags like
|
||||||
|
|
@ -153,7 +155,7 @@ class S3Response(S3ResponseBase, swob.Response):
|
||||||
|
|
||||||
# Used for pure swift header handling at the request layer
|
# Used for pure swift header handling at the request layer
|
||||||
self.sw_headers = sw_headers
|
self.sw_headers = sw_headers
|
||||||
self.sysmeta_headers = sw_sysmeta_headers
|
self.sysmeta_headers = s3_sysmeta_headers
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_swift_resp(cls, sw_resp):
|
def from_swift_resp(cls, sw_resp):
|
||||||
|
|
|
||||||
|
|
@ -67,7 +67,7 @@ from six.moves import urllib
|
||||||
from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \
|
from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \
|
||||||
HTTPException
|
HTTPException
|
||||||
from swift.common.utils import config_true_value, split_path, get_logger, \
|
from swift.common.utils import config_true_value, split_path, get_logger, \
|
||||||
cache_from_env
|
cache_from_env, append_underscore
|
||||||
from swift.common.wsgi import ConfigFileError
|
from swift.common.wsgi import ConfigFileError
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -149,7 +149,8 @@ class S3Token(object):
|
||||||
self._timeout = float(conf.get('http_timeout', '10.0'))
|
self._timeout = float(conf.get('http_timeout', '10.0'))
|
||||||
if not (0 < self._timeout <= 60):
|
if not (0 < self._timeout <= 60):
|
||||||
raise ValueError('http_timeout must be between 0 and 60 seconds')
|
raise ValueError('http_timeout must be between 0 and 60 seconds')
|
||||||
self._reseller_prefix = conf.get('reseller_prefix', 'AUTH_')
|
self._reseller_prefix = append_underscore(
|
||||||
|
conf.get('reseller_prefix', 'AUTH'))
|
||||||
self._delay_auth_decision = config_true_value(
|
self._delay_auth_decision = config_true_value(
|
||||||
conf.get('delay_auth_decision'))
|
conf.get('delay_auth_decision'))
|
||||||
|
|
||||||
|
|
@ -274,7 +275,9 @@ class S3Token(object):
|
||||||
string_to_sign = s3_auth_details['string_to_sign']
|
string_to_sign = s3_auth_details['string_to_sign']
|
||||||
if isinstance(string_to_sign, six.text_type):
|
if isinstance(string_to_sign, six.text_type):
|
||||||
string_to_sign = string_to_sign.encode('utf-8')
|
string_to_sign = string_to_sign.encode('utf-8')
|
||||||
token = base64.urlsafe_b64encode(string_to_sign).encode('ascii')
|
token = base64.urlsafe_b64encode(string_to_sign)
|
||||||
|
if isinstance(token, six.binary_type):
|
||||||
|
token = token.decode('ascii')
|
||||||
|
|
||||||
# NOTE(chmou): This is to handle the special case with nova
|
# NOTE(chmou): This is to handle the special case with nova
|
||||||
# when we have the option s3_affix_tenant. We will force it to
|
# when we have the option s3_affix_tenant. We will force it to
|
||||||
|
|
|
||||||
|
|
@ -125,14 +125,17 @@ Example usage of this middleware via ``swift``:
|
||||||
|
|
||||||
import cgi
|
import cgi
|
||||||
import json
|
import json
|
||||||
|
import six
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
from six.moves.urllib.parse import urlparse
|
||||||
|
|
||||||
from swift.common.utils import human_readable, split_path, config_true_value, \
|
from swift.common.utils import human_readable, split_path, config_true_value, \
|
||||||
quote, register_swift_info, get_logger, urlparse
|
quote, register_swift_info, get_logger
|
||||||
from swift.common.wsgi import make_env, WSGIContext
|
from swift.common.wsgi import make_env, WSGIContext
|
||||||
from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND
|
from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND
|
||||||
from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound, \
|
from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound, \
|
||||||
Request
|
Request, wsgi_quote, wsgi_to_str
|
||||||
from swift.proxy.controllers.base import get_container_info
|
from swift.proxy.controllers.base import get_container_info
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -145,6 +148,12 @@ class _StaticWebContext(WSGIContext):
|
||||||
that might need to be handled to make keeping contextual
|
that might need to be handled to make keeping contextual
|
||||||
information about the request a bit simpler than storing it in
|
information about the request a bit simpler than storing it in
|
||||||
the WSGI env.
|
the WSGI env.
|
||||||
|
|
||||||
|
:param staticweb: The staticweb middleware object in use.
|
||||||
|
:param version: A WSGI string representation of the swift api version.
|
||||||
|
:param account: A WSGI string representation of the account name.
|
||||||
|
:param container: A WSGI string representation of the container name.
|
||||||
|
:param obj: A WSGI string representation of the object name.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, staticweb, version, account, container, obj):
|
def __init__(self, staticweb, version, account, container, obj):
|
||||||
|
|
@ -223,9 +232,9 @@ class _StaticWebContext(WSGIContext):
|
||||||
:param start_response: The original WSGI start_response hook.
|
:param start_response: The original WSGI start_response hook.
|
||||||
:param prefix: Any prefix desired for the container listing.
|
:param prefix: Any prefix desired for the container listing.
|
||||||
"""
|
"""
|
||||||
label = env['PATH_INFO']
|
label = wsgi_to_str(env['PATH_INFO'])
|
||||||
if self._listings_label:
|
if self._listings_label:
|
||||||
groups = env['PATH_INFO'].split('/')
|
groups = wsgi_to_str(env['PATH_INFO']).split('/')
|
||||||
label = '{0}/{1}'.format(self._listings_label,
|
label = '{0}/{1}'.format(self._listings_label,
|
||||||
'/'.join(groups[4:]))
|
'/'.join(groups[4:]))
|
||||||
|
|
||||||
|
|
@ -262,14 +271,14 @@ class _StaticWebContext(WSGIContext):
|
||||||
self.agent, swift_source='SW')
|
self.agent, swift_source='SW')
|
||||||
tmp_env['QUERY_STRING'] = 'delimiter=/'
|
tmp_env['QUERY_STRING'] = 'delimiter=/'
|
||||||
if prefix:
|
if prefix:
|
||||||
tmp_env['QUERY_STRING'] += '&prefix=%s' % quote(prefix)
|
tmp_env['QUERY_STRING'] += '&prefix=%s' % wsgi_quote(prefix)
|
||||||
else:
|
else:
|
||||||
prefix = ''
|
prefix = ''
|
||||||
resp = self._app_call(tmp_env)
|
resp = self._app_call(tmp_env)
|
||||||
if not is_success(self._get_status_int()):
|
if not is_success(self._get_status_int()):
|
||||||
return self._error_response(resp, env, start_response)
|
return self._error_response(resp, env, start_response)
|
||||||
listing = None
|
listing = None
|
||||||
body = ''.join(resp)
|
body = b''.join(resp)
|
||||||
if body:
|
if body:
|
||||||
listing = json.loads(body)
|
listing = json.loads(body)
|
||||||
if not listing:
|
if not listing:
|
||||||
|
|
@ -280,7 +289,8 @@ class _StaticWebContext(WSGIContext):
|
||||||
'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \
|
'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \
|
||||||
'<html>\n' \
|
'<html>\n' \
|
||||||
' <head>\n' \
|
' <head>\n' \
|
||||||
' <title>Listing of %s</title>\n' % cgi.escape(label)
|
' <title>Listing of %s</title>\n' % \
|
||||||
|
cgi.escape(label)
|
||||||
if self._listings_css:
|
if self._listings_css:
|
||||||
body += ' <link rel="stylesheet" type="text/css" ' \
|
body += ' <link rel="stylesheet" type="text/css" ' \
|
||||||
'href="%s" />\n' % (self._build_css_path(prefix))
|
'href="%s" />\n' % (self._build_css_path(prefix))
|
||||||
|
|
@ -308,7 +318,8 @@ class _StaticWebContext(WSGIContext):
|
||||||
' </tr>\n'
|
' </tr>\n'
|
||||||
for item in listing:
|
for item in listing:
|
||||||
if 'subdir' in item:
|
if 'subdir' in item:
|
||||||
subdir = item['subdir'].encode("utf-8")
|
subdir = item['subdir'] if six.PY3 else \
|
||||||
|
item['subdir'].encode('utf-8')
|
||||||
if prefix:
|
if prefix:
|
||||||
subdir = subdir[len(prefix):]
|
subdir = subdir[len(prefix):]
|
||||||
body += ' <tr class="item subdir">\n' \
|
body += ' <tr class="item subdir">\n' \
|
||||||
|
|
@ -319,13 +330,16 @@ class _StaticWebContext(WSGIContext):
|
||||||
(quote(subdir), cgi.escape(subdir))
|
(quote(subdir), cgi.escape(subdir))
|
||||||
for item in listing:
|
for item in listing:
|
||||||
if 'name' in item:
|
if 'name' in item:
|
||||||
name = item['name'].encode("utf-8")
|
name = item['name'] if six.PY3 else \
|
||||||
|
item['name'].encode('utf-8')
|
||||||
if prefix:
|
if prefix:
|
||||||
name = name[len(prefix):]
|
name = name[len(prefix):]
|
||||||
content_type = item['content_type'].encode("utf-8")
|
content_type = item['content_type'] if six.PY3 else \
|
||||||
|
item['content_type'].encode('utf-8')
|
||||||
bytes = human_readable(item['bytes'])
|
bytes = human_readable(item['bytes'])
|
||||||
last_modified = (
|
last_modified = (
|
||||||
cgi.escape(item['last_modified'].encode("utf-8")).
|
cgi.escape(item['last_modified'] if six.PY3 else
|
||||||
|
item['last_modified'].encode('utf-8')).
|
||||||
split('.')[0].replace('T', ' '))
|
split('.')[0].replace('T', ' '))
|
||||||
body += ' <tr class="item %s">\n' \
|
body += ' <tr class="item %s">\n' \
|
||||||
' <td class="colname"><a href="%s">%s</a></td>\n' \
|
' <td class="colname"><a href="%s">%s</a></td>\n' \
|
||||||
|
|
@ -362,7 +376,8 @@ class _StaticWebContext(WSGIContext):
|
||||||
env['wsgi.url_scheme'] = self.url_scheme
|
env['wsgi.url_scheme'] = self.url_scheme
|
||||||
if self.url_host:
|
if self.url_host:
|
||||||
env['HTTP_HOST'] = self.url_host
|
env['HTTP_HOST'] = self.url_host
|
||||||
resp = HTTPMovedPermanently(location=(env['PATH_INFO'] + '/'))
|
resp = HTTPMovedPermanently(
|
||||||
|
location=wsgi_quote(env['PATH_INFO'] + '/'))
|
||||||
return resp(env, start_response)
|
return resp(env, start_response)
|
||||||
|
|
||||||
def handle_container(self, env, start_response):
|
def handle_container(self, env, start_response):
|
||||||
|
|
@ -466,9 +481,9 @@ class _StaticWebContext(WSGIContext):
|
||||||
self.version, self.account, self.container),
|
self.version, self.account, self.container),
|
||||||
self.agent, swift_source='SW')
|
self.agent, swift_source='SW')
|
||||||
tmp_env['QUERY_STRING'] = 'limit=1&delimiter=/&prefix=%s' % (
|
tmp_env['QUERY_STRING'] = 'limit=1&delimiter=/&prefix=%s' % (
|
||||||
quote(self.obj + '/'), )
|
quote(wsgi_to_str(self.obj) + '/'), )
|
||||||
resp = self._app_call(tmp_env)
|
resp = self._app_call(tmp_env)
|
||||||
body = ''.join(resp)
|
body = b''.join(resp)
|
||||||
if not is_success(self._get_status_int()) or not body or \
|
if not is_success(self._get_status_int()) or not body or \
|
||||||
not json.loads(body):
|
not json.loads(body):
|
||||||
resp = HTTPNotFound()(env, self._start_response)
|
resp = HTTPNotFound()(env, self._start_response)
|
||||||
|
|
|
||||||
|
|
@ -161,7 +161,7 @@ from cgi import parse_header
|
||||||
from six.moves.urllib.parse import unquote
|
from six.moves.urllib.parse import unquote
|
||||||
|
|
||||||
from swift.common.utils import get_logger, register_swift_info, split_path, \
|
from swift.common.utils import get_logger, register_swift_info, split_path, \
|
||||||
MD5_OF_EMPTY_STRING, closing_if_possible
|
MD5_OF_EMPTY_STRING, closing_if_possible, quote
|
||||||
from swift.common.constraints import check_account_format
|
from swift.common.constraints import check_account_format
|
||||||
from swift.common.wsgi import WSGIContext, make_subrequest
|
from swift.common.wsgi import WSGIContext, make_subrequest
|
||||||
from swift.common.request_helpers import get_sys_meta_prefix, \
|
from swift.common.request_helpers import get_sys_meta_prefix, \
|
||||||
|
|
@ -208,6 +208,7 @@ def _check_symlink_header(req):
|
||||||
req, TGT_OBJ_SYMLINK_HDR, 2,
|
req, TGT_OBJ_SYMLINK_HDR, 2,
|
||||||
'X-Symlink-Target header must be of the '
|
'X-Symlink-Target header must be of the '
|
||||||
'form <container name>/<object name>')
|
'form <container name>/<object name>')
|
||||||
|
req.headers[TGT_OBJ_SYMLINK_HDR] = quote('%s/%s' % (container, obj))
|
||||||
|
|
||||||
# Check account format if it exists
|
# Check account format if it exists
|
||||||
account = check_account_format(
|
account = check_account_format(
|
||||||
|
|
@ -217,7 +218,9 @@ def _check_symlink_header(req):
|
||||||
# Extract request path
|
# Extract request path
|
||||||
_junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True)
|
_junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True)
|
||||||
|
|
||||||
if not account:
|
if account:
|
||||||
|
req.headers[TGT_ACCT_SYMLINK_HDR] = quote(account)
|
||||||
|
else:
|
||||||
account = req_acc
|
account = req_acc
|
||||||
|
|
||||||
# Check if symlink targets the symlink itself or not
|
# Check if symlink targets the symlink itself or not
|
||||||
|
|
@ -378,9 +381,9 @@ class SymlinkObjectContext(WSGIContext):
|
||||||
:returns: new request for target path if it's symlink otherwise
|
:returns: new request for target path if it's symlink otherwise
|
||||||
None
|
None
|
||||||
"""
|
"""
|
||||||
version, account, _junk = split_path(req.path, 2, 3, True)
|
version, account, _junk = req.split_path(2, 3, True)
|
||||||
account = self._response_header_value(
|
account = self._response_header_value(
|
||||||
TGT_ACCT_SYSMETA_SYMLINK_HDR) or account
|
TGT_ACCT_SYSMETA_SYMLINK_HDR) or quote(account)
|
||||||
target_path = os.path.join(
|
target_path = os.path.join(
|
||||||
'/', version, account,
|
'/', version, account,
|
||||||
symlink_target.lstrip('/'))
|
symlink_target.lstrip('/'))
|
||||||
|
|
@ -485,7 +488,7 @@ class SymlinkObjectContext(WSGIContext):
|
||||||
if tgt_co:
|
if tgt_co:
|
||||||
version, account, _junk = req.split_path(2, 3, True)
|
version, account, _junk = req.split_path(2, 3, True)
|
||||||
target_acc = self._response_header_value(
|
target_acc = self._response_header_value(
|
||||||
TGT_ACCT_SYSMETA_SYMLINK_HDR) or account
|
TGT_ACCT_SYSMETA_SYMLINK_HDR) or quote(account)
|
||||||
location_hdr = os.path.join(
|
location_hdr = os.path.join(
|
||||||
'/', version, target_acc, tgt_co)
|
'/', version, target_acc, tgt_co)
|
||||||
req.environ['swift.leave_relative_location'] = True
|
req.environ['swift.leave_relative_location'] = True
|
||||||
|
|
|
||||||
|
|
@ -75,9 +75,8 @@ from six.moves import cPickle as pickle
|
||||||
from six.moves.configparser import (ConfigParser, NoSectionError,
|
from six.moves.configparser import (ConfigParser, NoSectionError,
|
||||||
NoOptionError, RawConfigParser)
|
NoOptionError, RawConfigParser)
|
||||||
from six.moves import range, http_client
|
from six.moves import range, http_client
|
||||||
from six.moves.urllib.parse import ParseResult
|
|
||||||
from six.moves.urllib.parse import quote as _quote
|
from six.moves.urllib.parse import quote as _quote
|
||||||
from six.moves.urllib.parse import urlparse as stdlib_urlparse
|
from six.moves.urllib.parse import urlparse
|
||||||
|
|
||||||
from swift import gettext_ as _
|
from swift import gettext_ as _
|
||||||
import swift.common.exceptions
|
import swift.common.exceptions
|
||||||
|
|
@ -2265,7 +2264,7 @@ def get_hub():
|
||||||
|
|
||||||
Note about epoll:
|
Note about epoll:
|
||||||
|
|
||||||
Review: https://review.openstack.org/#/c/18806/
|
Review: https://review.opendev.org/#/c/18806/
|
||||||
|
|
||||||
There was a problem where once out of every 30 quadrillion
|
There was a problem where once out of every 30 quadrillion
|
||||||
connections, a coroutine wouldn't wake up when the client
|
connections, a coroutine wouldn't wake up when the client
|
||||||
|
|
@ -3244,38 +3243,6 @@ class StreamingPile(GreenAsyncPile):
|
||||||
self.pool.__exit__(type, value, traceback)
|
self.pool.__exit__(type, value, traceback)
|
||||||
|
|
||||||
|
|
||||||
class ModifiedParseResult(ParseResult):
|
|
||||||
"""Parse results class for urlparse."""
|
|
||||||
|
|
||||||
@property
|
|
||||||
def hostname(self):
|
|
||||||
netloc = self.netloc.split('@', 1)[-1]
|
|
||||||
if netloc.startswith('['):
|
|
||||||
return netloc[1:].split(']')[0]
|
|
||||||
elif ':' in netloc:
|
|
||||||
return netloc.rsplit(':')[0]
|
|
||||||
return netloc
|
|
||||||
|
|
||||||
@property
|
|
||||||
def port(self):
|
|
||||||
netloc = self.netloc.split('@', 1)[-1]
|
|
||||||
if netloc.startswith('['):
|
|
||||||
netloc = netloc.rsplit(']')[1]
|
|
||||||
if ':' in netloc:
|
|
||||||
return int(netloc.rsplit(':')[1])
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def urlparse(url):
|
|
||||||
"""
|
|
||||||
urlparse augmentation.
|
|
||||||
This is necessary because urlparse can't handle RFC 2732 URLs.
|
|
||||||
|
|
||||||
:param url: URL to parse.
|
|
||||||
"""
|
|
||||||
return ModifiedParseResult(*stdlib_urlparse(url))
|
|
||||||
|
|
||||||
|
|
||||||
def validate_sync_to(value, allowed_sync_hosts, realms_conf):
|
def validate_sync_to(value, allowed_sync_hosts, realms_conf):
|
||||||
"""
|
"""
|
||||||
Validates an X-Container-Sync-To header value, returning the
|
Validates an X-Container-Sync-To header value, returning the
|
||||||
|
|
|
||||||
|
|
@ -430,6 +430,7 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
|
||||||
|
|
||||||
# for py3:
|
# for py3:
|
||||||
def get_default_type(self):
|
def get_default_type(self):
|
||||||
|
'''If the client didn't provide a content type, leave it blank.'''
|
||||||
return ''
|
return ''
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -365,8 +365,9 @@ class ContainerSharder(ContainerReplicator):
|
||||||
'Swift Container Sharder',
|
'Swift Container Sharder',
|
||||||
request_tries,
|
request_tries,
|
||||||
allow_modify_pipeline=False)
|
allow_modify_pipeline=False)
|
||||||
except IOError as err:
|
except (OSError, IOError) as err:
|
||||||
if err.errno != errno.ENOENT:
|
if err.errno != errno.ENOENT and \
|
||||||
|
not str(err).endswith(' not found'):
|
||||||
raise
|
raise
|
||||||
raise SystemExit(
|
raise SystemExit(
|
||||||
'Unable to load internal client from config: %r (%s)' %
|
'Unable to load internal client from config: %r (%s)' %
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ from random import choice, random
|
||||||
from struct import unpack_from
|
from struct import unpack_from
|
||||||
|
|
||||||
from eventlet import sleep, Timeout
|
from eventlet import sleep, Timeout
|
||||||
|
from six.moves.urllib.parse import urlparse
|
||||||
|
|
||||||
import swift.common.db
|
import swift.common.db
|
||||||
from swift.common.db import DatabaseConnectionError
|
from swift.common.db import DatabaseConnectionError
|
||||||
|
|
@ -37,7 +38,7 @@ from swift.common.ring import Ring
|
||||||
from swift.common.ring.utils import is_local_device
|
from swift.common.ring.utils import is_local_device
|
||||||
from swift.common.utils import (
|
from swift.common.utils import (
|
||||||
clean_content_type, config_true_value,
|
clean_content_type, config_true_value,
|
||||||
FileLikeIter, get_logger, hash_path, quote, urlparse, validate_sync_to,
|
FileLikeIter, get_logger, hash_path, quote, validate_sync_to,
|
||||||
whataremyips, Timestamp, decode_timestamps)
|
whataremyips, Timestamp, decode_timestamps)
|
||||||
from swift.common.daemon import Daemon
|
from swift.common.daemon import Daemon
|
||||||
from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND
|
from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND
|
||||||
|
|
@ -238,8 +239,9 @@ class ContainerSync(Daemon):
|
||||||
try:
|
try:
|
||||||
self.swift = InternalClient(
|
self.swift = InternalClient(
|
||||||
internal_client_conf, 'Swift Container Sync', request_tries)
|
internal_client_conf, 'Swift Container Sync', request_tries)
|
||||||
except IOError as err:
|
except (OSError, IOError) as err:
|
||||||
if err.errno != errno.ENOENT:
|
if err.errno != errno.ENOENT and \
|
||||||
|
not str(err).endswith(' not found'):
|
||||||
raise
|
raise
|
||||||
raise SystemExit(
|
raise SystemExit(
|
||||||
_('Unable to load internal client from config: '
|
_('Unable to load internal client from config: '
|
||||||
|
|
|
||||||
|
|
@ -818,9 +818,10 @@ class ObjectReplicator(Daemon):
|
||||||
except Exception:
|
except Exception:
|
||||||
self.logger.exception('ERROR creating %s' % obj_path)
|
self.logger.exception('ERROR creating %s' % obj_path)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for partition in df_mgr.listdir(obj_path):
|
for partition in df_mgr.listdir(obj_path):
|
||||||
if (override_partitions is not None
|
if (override_partitions is not None and partition.isdigit()
|
||||||
and partition not in override_partitions):
|
and int(partition) not in override_partitions):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if (partition.startswith('auditor_status_') and
|
if (partition.startswith('auditor_status_') and
|
||||||
|
|
|
||||||
|
|
@ -16,8 +16,10 @@
|
||||||
import os
|
import os
|
||||||
import test.functional as tf
|
import test.functional as tf
|
||||||
from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \
|
from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \
|
||||||
BotoClientError, S3ResponseError
|
S3ResponseError
|
||||||
import six
|
import six
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
|
||||||
RETRY_COUNT = 3
|
RETRY_COUNT = 3
|
||||||
|
|
@ -92,11 +94,12 @@ class Connection(object):
|
||||||
# 404 means NoSuchBucket, NoSuchKey, or NoSuchUpload
|
# 404 means NoSuchBucket, NoSuchKey, or NoSuchUpload
|
||||||
if e.status != 404:
|
if e.status != 404:
|
||||||
raise
|
raise
|
||||||
except (BotoClientError, S3ResponseError) as e:
|
except Exception as e:
|
||||||
exceptions.append(e)
|
exceptions.append(''.join(
|
||||||
|
traceback.format_exception(*sys.exc_info())))
|
||||||
if exceptions:
|
if exceptions:
|
||||||
# raise the first exception
|
exceptions.insert(0, 'Too many errors to continue:')
|
||||||
raise exceptions.pop(0)
|
raise Exception('\n========\n'.join(exceptions))
|
||||||
|
|
||||||
def make_request(self, method, bucket='', obj='', headers=None, body='',
|
def make_request(self, method, bucket='', obj='', headers=None, body='',
|
||||||
query=None):
|
query=None):
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,8 @@
|
||||||
|
|
||||||
import functools
|
import functools
|
||||||
from unittest2 import SkipTest
|
from unittest2 import SkipTest
|
||||||
|
from six.moves.urllib.parse import unquote
|
||||||
|
from swift.common.utils import quote
|
||||||
import test.functional as tf
|
import test.functional as tf
|
||||||
from test.functional import cluster_info
|
from test.functional import cluster_info
|
||||||
from test.functional.tests import Utils, Base, BaseEnv
|
from test.functional.tests import Utils, Base, BaseEnv
|
||||||
|
|
@ -74,8 +76,8 @@ class TestStaticWebEnv(BaseEnv):
|
||||||
'listings_css',
|
'listings_css',
|
||||||
'dir/',
|
'dir/',
|
||||||
'dir/obj',
|
'dir/obj',
|
||||||
'dir/subdir/',
|
'dir/some sub%dir/',
|
||||||
'dir/subdir/obj']
|
'dir/some sub%dir/obj']
|
||||||
|
|
||||||
cls.objects = {}
|
cls.objects = {}
|
||||||
for item in sorted(objects):
|
for item in sorted(objects):
|
||||||
|
|
@ -168,12 +170,12 @@ class TestStaticWeb(Base):
|
||||||
def _test_redirect_slash_direct(self, anonymous):
|
def _test_redirect_slash_direct(self, anonymous):
|
||||||
host = self.env.account.conn.storage_netloc
|
host = self.env.account.conn.storage_netloc
|
||||||
path = '%s/%s' % (self.env.account.conn.storage_path,
|
path = '%s/%s' % (self.env.account.conn.storage_path,
|
||||||
self.env.container.name)
|
quote(self.env.container.name))
|
||||||
self._test_redirect_with_slash(host, path, anonymous=anonymous)
|
self._test_redirect_with_slash(host, path, anonymous=anonymous)
|
||||||
|
|
||||||
path = '%s/%s/%s' % (self.env.account.conn.storage_path,
|
path = '%s/%s/%s' % (self.env.account.conn.storage_path,
|
||||||
self.env.container.name,
|
quote(self.env.container.name),
|
||||||
self.env.objects['dir/'].name)
|
quote(self.env.objects['dir/'].name))
|
||||||
self._test_redirect_with_slash(host, path, anonymous=anonymous)
|
self._test_redirect_with_slash(host, path, anonymous=anonymous)
|
||||||
|
|
||||||
def test_redirect_slash_auth_direct(self):
|
def test_redirect_slash_auth_direct(self):
|
||||||
|
|
@ -185,10 +187,10 @@ class TestStaticWeb(Base):
|
||||||
@requires_domain_remap
|
@requires_domain_remap
|
||||||
def _test_redirect_slash_remap_acct(self, anonymous):
|
def _test_redirect_slash_remap_acct(self, anonymous):
|
||||||
host = self.domain_remap_acct
|
host = self.domain_remap_acct
|
||||||
path = '/%s' % self.env.container.name
|
path = '/%s' % quote(self.env.container.name)
|
||||||
self._test_redirect_with_slash(host, path, anonymous=anonymous)
|
self._test_redirect_with_slash(host, path, anonymous=anonymous)
|
||||||
|
|
||||||
path = '/%s/%s' % (self.env.container.name,
|
path = '/%s/%s' % (quote(self.env.container.name),
|
||||||
self.env.objects['dir/'].name)
|
self.env.objects['dir/'].name)
|
||||||
self._test_redirect_with_slash(host, path, anonymous=anonymous)
|
self._test_redirect_with_slash(host, path, anonymous=anonymous)
|
||||||
|
|
||||||
|
|
@ -229,13 +231,14 @@ class TestStaticWeb(Base):
|
||||||
self._set_staticweb_headers(listings=True,
|
self._set_staticweb_headers(listings=True,
|
||||||
listings_css=(css is not None))
|
listings_css=(css is not None))
|
||||||
if title is None:
|
if title is None:
|
||||||
title = path
|
title = unquote(path)
|
||||||
expected_in = ['Listing of %s' % title] + [
|
expected_in = ['Listing of %s' % title] + [
|
||||||
'<a href="{0}">{0}</a>'.format(link) for link in links]
|
'<a href="{0}">{1}</a>'.format(quote(link), link)
|
||||||
|
for link in links]
|
||||||
expected_not_in = notins
|
expected_not_in = notins
|
||||||
if css:
|
if css:
|
||||||
expected_in.append('<link rel="stylesheet" type="text/css" '
|
expected_in.append('<link rel="stylesheet" type="text/css" '
|
||||||
'href="%s" />' % css)
|
'href="%s" />' % quote(css))
|
||||||
self._test_get_path(host, path, anonymous=anonymous,
|
self._test_get_path(host, path, anonymous=anonymous,
|
||||||
expected_in=expected_in,
|
expected_in=expected_in,
|
||||||
expected_not_in=expected_not_in)
|
expected_not_in=expected_not_in)
|
||||||
|
|
@ -244,7 +247,7 @@ class TestStaticWeb(Base):
|
||||||
objects = self.env.objects
|
objects = self.env.objects
|
||||||
host = self.env.account.conn.storage_netloc
|
host = self.env.account.conn.storage_netloc
|
||||||
path = '%s/%s/' % (self.env.account.conn.storage_path,
|
path = '%s/%s/' % (self.env.account.conn.storage_path,
|
||||||
self.env.container.name)
|
quote(self.env.container.name))
|
||||||
css = objects['listings_css'].name if listings_css else None
|
css = objects['listings_css'].name if listings_css else None
|
||||||
self._test_listing(host, path, anonymous=True, css=css,
|
self._test_listing(host, path, anonymous=True, css=css,
|
||||||
links=[objects['index'].name,
|
links=[objects['index'].name,
|
||||||
|
|
@ -252,15 +255,15 @@ class TestStaticWeb(Base):
|
||||||
notins=[objects['dir/obj'].name])
|
notins=[objects['dir/obj'].name])
|
||||||
|
|
||||||
path = '%s/%s/%s/' % (self.env.account.conn.storage_path,
|
path = '%s/%s/%s/' % (self.env.account.conn.storage_path,
|
||||||
self.env.container.name,
|
quote(self.env.container.name),
|
||||||
objects['dir/'].name)
|
quote(objects['dir/'].name))
|
||||||
css = '../%s' % objects['listings_css'].name if listings_css else None
|
css = '../%s' % objects['listings_css'].name if listings_css else None
|
||||||
self._test_listing(host, path, anonymous=anonymous, css=css,
|
self._test_listing(
|
||||||
links=[objects['dir/obj'].name.split('/')[-1],
|
host, path, anonymous=anonymous, css=css,
|
||||||
objects['dir/subdir/'].name.split('/')[-1]
|
links=[objects['dir/obj'].name.split('/')[-1],
|
||||||
+ '/'],
|
objects['dir/some sub%dir/'].name.split('/')[-1] + '/'],
|
||||||
notins=[objects['index'].name,
|
notins=[objects['index'].name,
|
||||||
objects['dir/subdir/obj'].name])
|
objects['dir/some sub%dir/obj'].name])
|
||||||
|
|
||||||
def test_listing_auth_direct_without_css(self):
|
def test_listing_auth_direct_without_css(self):
|
||||||
self._test_listing_direct(False, False)
|
self._test_listing_direct(False, False)
|
||||||
|
|
@ -293,13 +296,12 @@ class TestStaticWeb(Base):
|
||||||
title = '%s/%s/%s/' % (self.env.account.conn.storage_path,
|
title = '%s/%s/%s/' % (self.env.account.conn.storage_path,
|
||||||
self.env.container.name,
|
self.env.container.name,
|
||||||
objects['dir/'])
|
objects['dir/'])
|
||||||
self._test_listing(host, path, title=title, anonymous=anonymous,
|
self._test_listing(
|
||||||
css=css,
|
host, path, title=title, anonymous=anonymous, css=css,
|
||||||
links=[objects['dir/obj'].name.split('/')[-1],
|
links=[objects['dir/obj'].name.split('/')[-1],
|
||||||
objects['dir/subdir/'].name.split('/')[-1]
|
objects['dir/some sub%dir/'].name.split('/')[-1] + '/'],
|
||||||
+ '/'],
|
notins=[objects['index'].name,
|
||||||
notins=[objects['index'].name,
|
objects['dir/some sub%dir/obj'].name])
|
||||||
objects['dir/subdir/obj'].name])
|
|
||||||
|
|
||||||
def test_listing_auth_remap_acct_without_css(self):
|
def test_listing_auth_remap_acct_without_css(self):
|
||||||
self._test_listing_remap_acct(False, False)
|
self._test_listing_remap_acct(False, False)
|
||||||
|
|
@ -332,13 +334,12 @@ class TestStaticWeb(Base):
|
||||||
title = '%s/%s/%s/' % (self.env.account.conn.storage_path,
|
title = '%s/%s/%s/' % (self.env.account.conn.storage_path,
|
||||||
self.env.container.name,
|
self.env.container.name,
|
||||||
objects['dir/'])
|
objects['dir/'])
|
||||||
self._test_listing(host, path, title=title, anonymous=anonymous,
|
self._test_listing(
|
||||||
css=css,
|
host, path, title=title, anonymous=anonymous, css=css,
|
||||||
links=[objects['dir/obj'].name.split('/')[-1],
|
links=[objects['dir/obj'].name.split('/')[-1],
|
||||||
objects['dir/subdir/'].name.split('/')[-1]
|
objects['dir/some sub%dir/'].name.split('/')[-1] + '/'],
|
||||||
+ '/'],
|
notins=[objects['index'].name,
|
||||||
notins=[objects['index'].name,
|
objects['dir/some sub%dir/obj'].name])
|
||||||
objects['dir/subdir/obj'].name])
|
|
||||||
|
|
||||||
def test_listing_auth_remap_cont_without_css(self):
|
def test_listing_auth_remap_cont_without_css(self):
|
||||||
self._test_listing_remap_cont(False, False)
|
self._test_listing_remap_cont(False, False)
|
||||||
|
|
@ -369,12 +370,12 @@ class TestStaticWeb(Base):
|
||||||
objects = self.env.objects
|
objects = self.env.objects
|
||||||
host = self.env.account.conn.storage_netloc
|
host = self.env.account.conn.storage_netloc
|
||||||
path = '%s/%s/' % (self.env.account.conn.storage_path,
|
path = '%s/%s/' % (self.env.account.conn.storage_path,
|
||||||
self.env.container.name)
|
quote(self.env.container.name))
|
||||||
self._test_index(host, path, anonymous=anonymous)
|
self._test_index(host, path, anonymous=anonymous)
|
||||||
|
|
||||||
path = '%s/%s/%s/' % (self.env.account.conn.storage_path,
|
path = '%s/%s/%s/' % (self.env.account.conn.storage_path,
|
||||||
self.env.container.name,
|
quote(self.env.container.name),
|
||||||
objects['dir/'].name)
|
quote(objects['dir/'].name))
|
||||||
self._test_index(host, path, anonymous=anonymous, expected_status=404)
|
self._test_index(host, path, anonymous=anonymous, expected_status=404)
|
||||||
|
|
||||||
def test_index_auth_direct(self):
|
def test_index_auth_direct(self):
|
||||||
|
|
|
||||||
|
|
@ -270,23 +270,45 @@ class TestSymlink(Base):
|
||||||
target_obj = 'dealde%2Fl04 011e%204c8df/flash.png'
|
target_obj = 'dealde%2Fl04 011e%204c8df/flash.png'
|
||||||
link_obj = uuid4().hex
|
link_obj = uuid4().hex
|
||||||
|
|
||||||
# Now let's write a new target object and symlink will be able to
|
# create target using unnormalized path
|
||||||
# return object
|
|
||||||
resp = retry(
|
resp = retry(
|
||||||
self._make_request, method='PUT', container=self.env.tgt_cont,
|
self._make_request, method='PUT', container=self.env.tgt_cont,
|
||||||
obj=target_obj, body=TARGET_BODY)
|
obj=target_obj, body=TARGET_BODY)
|
||||||
|
|
||||||
self.assertEqual(resp.status, 201)
|
self.assertEqual(resp.status, 201)
|
||||||
|
# you can get it using either name
|
||||||
|
resp = retry(
|
||||||
|
self._make_request, method='GET', container=self.env.tgt_cont,
|
||||||
|
obj=target_obj)
|
||||||
|
self.assertEqual(resp.status, 200)
|
||||||
|
self.assertEqual(resp.content, TARGET_BODY)
|
||||||
|
normalized_quoted_obj = 'dealde/l04%20011e%204c8df/flash.png'
|
||||||
|
self.assertEqual(normalized_quoted_obj, urllib.parse.quote(
|
||||||
|
urllib.parse.unquote(target_obj)))
|
||||||
|
resp = retry(
|
||||||
|
self._make_request, method='GET', container=self.env.tgt_cont,
|
||||||
|
obj=normalized_quoted_obj)
|
||||||
|
self.assertEqual(resp.status, 200)
|
||||||
|
self.assertEqual(resp.content, TARGET_BODY)
|
||||||
|
|
||||||
# PUT symlink
|
# create a symlink using the un-normalized target path
|
||||||
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
|
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
|
||||||
tgt_cont=self.env.tgt_cont,
|
tgt_cont=self.env.tgt_cont,
|
||||||
tgt_obj=target_obj)
|
tgt_obj=target_obj)
|
||||||
|
# and it's normalized
|
||||||
self._assertSymlink(
|
self._assertSymlink(
|
||||||
self.env.link_cont, link_obj,
|
self.env.link_cont, link_obj,
|
||||||
expected_content_location="%s/%s" % (self.env.tgt_cont,
|
expected_content_location='%s/%s' % (
|
||||||
target_obj))
|
self.env.tgt_cont, normalized_quoted_obj))
|
||||||
|
|
||||||
|
# create a symlink using the normalized target path
|
||||||
|
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
|
||||||
|
tgt_cont=self.env.tgt_cont,
|
||||||
|
tgt_obj=normalized_quoted_obj)
|
||||||
|
# and it's ALSO normalized
|
||||||
|
self._assertSymlink(
|
||||||
|
self.env.link_cont, link_obj,
|
||||||
|
expected_content_location='%s/%s' % (
|
||||||
|
self.env.tgt_cont, normalized_quoted_obj))
|
||||||
|
|
||||||
def test_symlink_put_head_get(self):
|
def test_symlink_put_head_get(self):
|
||||||
link_obj = uuid4().hex
|
link_obj = uuid4().hex
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ from copy import deepcopy
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import unittest2
|
import unittest2
|
||||||
from six.moves.urllib.parse import quote
|
from six.moves.urllib.parse import quote, unquote
|
||||||
|
|
||||||
import test.functional as tf
|
import test.functional as tf
|
||||||
|
|
||||||
|
|
@ -652,7 +652,7 @@ class TestObjectVersioning(Base):
|
||||||
tgt_b.write("bbbbb")
|
tgt_b.write("bbbbb")
|
||||||
|
|
||||||
symlink_name = Utils.create_name()
|
symlink_name = Utils.create_name()
|
||||||
sym_tgt_header = '%s/%s' % (container.name, tgt_a_name)
|
sym_tgt_header = quote(unquote('%s/%s' % (container.name, tgt_a_name)))
|
||||||
sym_headers_a = {'X-Symlink-Target': sym_tgt_header}
|
sym_headers_a = {'X-Symlink-Target': sym_tgt_header}
|
||||||
symlink = container.file(symlink_name)
|
symlink = container.file(symlink_name)
|
||||||
symlink.write("", hdrs=sym_headers_a)
|
symlink.write("", hdrs=sym_headers_a)
|
||||||
|
|
@ -684,8 +684,9 @@ class TestObjectVersioning(Base):
|
||||||
sym_info = symlink.info(parms={'symlink': 'get'})
|
sym_info = symlink.info(parms={'symlink': 'get'})
|
||||||
self.assertEqual("aaaaa", symlink.read())
|
self.assertEqual("aaaaa", symlink.read())
|
||||||
self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag'])
|
self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag'])
|
||||||
self.assertEqual('%s/%s' % (self.env.container.name, target.name),
|
self.assertEqual(
|
||||||
sym_info['x_symlink_target'])
|
quote(unquote('%s/%s' % (self.env.container.name, target.name))),
|
||||||
|
sym_info['x_symlink_target'])
|
||||||
|
|
||||||
def _setup_symlink(self):
|
def _setup_symlink(self):
|
||||||
target = self.env.container.file('target-object')
|
target = self.env.container.file('target-object')
|
||||||
|
|
|
||||||
|
|
@ -302,9 +302,14 @@ class TestRequest(S3ApiTestCase):
|
||||||
self.assertTrue(sw_req.environ['swift.proxy_access_log_made'])
|
self.assertTrue(sw_req.environ['swift.proxy_access_log_made'])
|
||||||
|
|
||||||
def test_get_container_info(self):
|
def test_get_container_info(self):
|
||||||
|
s3api_acl = '{"Owner":"owner","Grant":'\
|
||||||
|
'[{"Grantee":"owner","Permission":"FULL_CONTROL"}]}'
|
||||||
self.swift.register('HEAD', '/v1/AUTH_test/bucket', HTTPNoContent,
|
self.swift.register('HEAD', '/v1/AUTH_test/bucket', HTTPNoContent,
|
||||||
{'x-container-read': 'foo',
|
{'x-container-read': 'foo',
|
||||||
'X-container-object-count': 5,
|
'X-container-object-count': 5,
|
||||||
|
'x-container-sysmeta-versions-location':
|
||||||
|
'bucket2',
|
||||||
|
'x-container-sysmeta-s3api-acl': s3api_acl,
|
||||||
'X-container-meta-foo': 'bar'}, None)
|
'X-container-meta-foo': 'bar'}, None)
|
||||||
req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'GET'},
|
req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'GET'},
|
||||||
headers={'Authorization': 'AWS test:tester:hmac',
|
headers={'Authorization': 'AWS test:tester:hmac',
|
||||||
|
|
@ -316,6 +321,9 @@ class TestRequest(S3ApiTestCase):
|
||||||
self.assertEqual(204, info['status']) # sanity
|
self.assertEqual(204, info['status']) # sanity
|
||||||
self.assertEqual('foo', info['read_acl']) # sanity
|
self.assertEqual('foo', info['read_acl']) # sanity
|
||||||
self.assertEqual('5', info['object_count']) # sanity
|
self.assertEqual('5', info['object_count']) # sanity
|
||||||
|
self.assertEqual(
|
||||||
|
'bucket2', info['sysmeta']['versions-location']) # sanity
|
||||||
|
self.assertEqual(s3api_acl, info['sysmeta']['s3api-acl']) # sanity
|
||||||
self.assertEqual({'foo': 'bar'}, info['meta']) # sanity
|
self.assertEqual({'foo': 'bar'}, info['meta']) # sanity
|
||||||
with patch(
|
with patch(
|
||||||
'swift.common.middleware.s3api.s3request.get_container_info',
|
'swift.common.middleware.s3api.s3request.get_container_info',
|
||||||
|
|
|
||||||
|
|
@ -54,6 +54,8 @@ class TestResponse(unittest.TestCase):
|
||||||
expected_headers = HeaderKeyDict(
|
expected_headers = HeaderKeyDict(
|
||||||
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
||||||
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
|
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
|
||||||
|
self.assertIn('x-%s-sysmeta-test-s3api' % _server_type,
|
||||||
|
s3resp.sw_headers)
|
||||||
|
|
||||||
def test_response_s3api_sysmeta_from_swift3_sysmeta(self):
|
def test_response_s3api_sysmeta_from_swift3_sysmeta(self):
|
||||||
for _server_type in ('object', 'container'):
|
for _server_type in ('object', 'container'):
|
||||||
|
|
|
||||||
|
|
@ -372,6 +372,15 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
|
||||||
middleware = s3token.filter_factory(config)(self.app)
|
middleware = s3token.filter_factory(config)(self.app)
|
||||||
self.assertIs('false_ind', middleware._verify)
|
self.assertIs('false_ind', middleware._verify)
|
||||||
|
|
||||||
|
def test_reseller_prefix(self):
|
||||||
|
def do_test(conf, expected):
|
||||||
|
conf.update(self.conf)
|
||||||
|
middleware = s3token.filter_factory(conf)(self.app)
|
||||||
|
self.assertEqual(expected, middleware._reseller_prefix)
|
||||||
|
do_test({}, 'AUTH_')
|
||||||
|
do_test({'reseller_prefix': 'KEY_'}, 'KEY_')
|
||||||
|
do_test({'reseller_prefix': 'KEY'}, 'KEY_')
|
||||||
|
|
||||||
def test_auth_uris(self):
|
def test_auth_uris(self):
|
||||||
for conf, expected in [
|
for conf, expected in [
|
||||||
({'auth_uri': 'https://example.com/v2.0'},
|
({'auth_uri': 'https://example.com/v2.0'},
|
||||||
|
|
@ -456,12 +465,12 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
|
||||||
with self.assertRaises(ConfigFileError) as cm:
|
with self.assertRaises(ConfigFileError) as cm:
|
||||||
s3token.filter_factory({'auth_uri': auth_uri})(self.app)
|
s3token.filter_factory({'auth_uri': auth_uri})(self.app)
|
||||||
self.assertEqual('Invalid auth_uri; must include scheme and host',
|
self.assertEqual('Invalid auth_uri; must include scheme and host',
|
||||||
cm.exception.message)
|
cm.exception.args[0])
|
||||||
with self.assertRaises(ConfigFileError) as cm:
|
with self.assertRaises(ConfigFileError) as cm:
|
||||||
s3token.filter_factory({
|
s3token.filter_factory({
|
||||||
'auth_uri': 'nonhttp://example.com'})(self.app)
|
'auth_uri': 'nonhttp://example.com'})(self.app)
|
||||||
self.assertEqual('Invalid auth_uri; scheme must be http or https',
|
self.assertEqual('Invalid auth_uri; scheme must be http or https',
|
||||||
cm.exception.message)
|
cm.exception.args[0])
|
||||||
for auth_uri in [
|
for auth_uri in [
|
||||||
'http://user@example.com/',
|
'http://user@example.com/',
|
||||||
'http://example.com/?with=query',
|
'http://example.com/?with=query',
|
||||||
|
|
@ -469,7 +478,7 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
|
||||||
with self.assertRaises(ConfigFileError) as cm:
|
with self.assertRaises(ConfigFileError) as cm:
|
||||||
s3token.filter_factory({'auth_uri': auth_uri})(self.app)
|
s3token.filter_factory({'auth_uri': auth_uri})(self.app)
|
||||||
self.assertEqual('Invalid auth_uri; must not include username, '
|
self.assertEqual('Invalid auth_uri; must not include username, '
|
||||||
'query, or fragment', cm.exception.message)
|
'query, or fragment', cm.exception.args[0])
|
||||||
|
|
||||||
def test_unicode_path(self):
|
def test_unicode_path(self):
|
||||||
url = u'/v1/AUTH_cfa/c/euro\u20ac'.encode('utf8')
|
url = u'/v1/AUTH_cfa/c/euro\u20ac'.encode('utf8')
|
||||||
|
|
@ -568,7 +577,7 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
|
||||||
|
|
||||||
MOCK_REQUEST.return_value = TestResponse({
|
MOCK_REQUEST.return_value = TestResponse({
|
||||||
'status_code': 201,
|
'status_code': 201,
|
||||||
'text': json.dumps(GOOD_RESPONSE_V2)})
|
'text': json.dumps(GOOD_RESPONSE_V2).encode('ascii')})
|
||||||
|
|
||||||
req = Request.blank('/v1/AUTH_cfa/c/o')
|
req = Request.blank('/v1/AUTH_cfa/c/o')
|
||||||
req.environ['s3api.auth_details'] = {
|
req.environ['s3api.auth_details'] = {
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,8 @@ LIMIT = 'swift.common.constraints.CONTAINER_LISTING_LIMIT'
|
||||||
|
|
||||||
|
|
||||||
def md5hex(s):
|
def md5hex(s):
|
||||||
|
if not isinstance(s, bytes):
|
||||||
|
s = s.encode('utf-8')
|
||||||
return hashlib.md5(s).hexdigest()
|
return hashlib.md5(s).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -52,7 +54,7 @@ class DloTestCase(unittest.TestCase):
|
||||||
headers[0] = h
|
headers[0] = h
|
||||||
|
|
||||||
body_iter = app(req.environ, start_response)
|
body_iter = app(req.environ, start_response)
|
||||||
body = ''
|
body = b''
|
||||||
# appease the close-checker
|
# appease the close-checker
|
||||||
with closing_if_possible(body_iter):
|
with closing_if_possible(body_iter):
|
||||||
for chunk in body_iter:
|
for chunk in body_iter:
|
||||||
|
|
@ -69,36 +71,36 @@ class DloTestCase(unittest.TestCase):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_01',
|
'GET', '/v1/AUTH_test/c/seg_01',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")},
|
||||||
'aaaaa')
|
b'aaaaa')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_02',
|
'GET', '/v1/AUTH_test/c/seg_02',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
|
||||||
'bbbbb')
|
b'bbbbb')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_03',
|
'GET', '/v1/AUTH_test/c/seg_03',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")},
|
||||||
'ccccc')
|
b'ccccc')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_04',
|
'GET', '/v1/AUTH_test/c/seg_04',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")},
|
||||||
'ddddd')
|
b'ddddd')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_05',
|
'GET', '/v1/AUTH_test/c/seg_05',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")},
|
||||||
'eeeee')
|
b'eeeee')
|
||||||
|
|
||||||
# an unrelated object (not seg*) to test the prefix matching
|
# an unrelated object (not seg*) to test the prefix matching
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/catpicture.jpg',
|
'GET', '/v1/AUTH_test/c/catpicture.jpg',
|
||||||
swob.HTTPOk, {'Content-Length': '9',
|
swob.HTTPOk, {'Content-Length': '9',
|
||||||
'Etag': md5hex("meow meow meow meow")},
|
'Etag': md5hex("meow meow meow meow")},
|
||||||
'meow meow meow meow')
|
b'meow meow meow meow')
|
||||||
|
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/mancon/manifest',
|
'GET', '/v1/AUTH_test/mancon/manifest',
|
||||||
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
|
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
|
||||||
'X-Object-Manifest': 'c/seg'},
|
'X-Object-Manifest': 'c/seg'},
|
||||||
'manifest-contents')
|
b'manifest-contents')
|
||||||
|
|
||||||
lm = '2013-11-22T02:42:13.781760'
|
lm = '2013-11-22T02:42:13.781760'
|
||||||
ct = 'application/octet-stream'
|
ct = 'application/octet-stream'
|
||||||
|
|
@ -120,11 +122,11 @@ class DloTestCase(unittest.TestCase):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c',
|
'GET', '/v1/AUTH_test/c',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps(full_container_listing))
|
json.dumps(full_container_listing).encode('ascii'))
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=seg',
|
'GET', '/v1/AUTH_test/c?prefix=seg',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps(segs))
|
json.dumps(segs).encode('ascii'))
|
||||||
|
|
||||||
# This is to let us test multi-page container listings; we use the
|
# This is to let us test multi-page container listings; we use the
|
||||||
# trailing underscore to send small (pagesize=3) listings.
|
# trailing underscore to send small (pagesize=3) listings.
|
||||||
|
|
@ -135,26 +137,26 @@ class DloTestCase(unittest.TestCase):
|
||||||
'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
|
'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
|
||||||
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
|
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
|
||||||
'X-Object-Manifest': 'c/seg_'},
|
'X-Object-Manifest': 'c/seg_'},
|
||||||
'manyseg')
|
b'manyseg')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=seg_',
|
'GET', '/v1/AUTH_test/c?prefix=seg_',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps(segs[:3]))
|
json.dumps(segs[:3]).encode('ascii'))
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03',
|
'GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps(segs[3:]))
|
json.dumps(segs[3:]).encode('ascii'))
|
||||||
|
|
||||||
# Here's a manifest with 0 segments
|
# Here's a manifest with 0 segments
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
|
'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
|
||||||
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
|
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
|
||||||
'X-Object-Manifest': 'c/noseg_'},
|
'X-Object-Manifest': 'c/noseg_'},
|
||||||
'noseg')
|
b'noseg')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=noseg_',
|
'GET', '/v1/AUTH_test/c?prefix=noseg_',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps([]))
|
json.dumps([]).encode('ascii'))
|
||||||
|
|
||||||
|
|
||||||
class TestDloPutManifest(DloTestCase):
|
class TestDloPutManifest(DloTestCase):
|
||||||
|
|
@ -284,7 +286,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(headers["Etag"], expected_etag)
|
self.assertEqual(headers["Etag"], expected_etag)
|
||||||
self.assertEqual(headers["Content-Length"], "25")
|
self.assertEqual(headers["Content-Length"], "25")
|
||||||
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
|
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
|
||||||
|
|
||||||
for _, _, hdrs in self.app.calls_with_headers[1:]:
|
for _, _, hdrs in self.app.calls_with_headers[1:]:
|
||||||
ua = hdrs.get("User-Agent", "")
|
ua = hdrs.get("User-Agent", "")
|
||||||
|
|
@ -302,7 +304,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg',
|
req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg',
|
||||||
environ={'REQUEST_METHOD': 'GET'})
|
environ={'REQUEST_METHOD': 'GET'})
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
self.assertEqual(body, "meow meow meow meow")
|
self.assertEqual(body, b"meow meow meow meow")
|
||||||
|
|
||||||
def test_get_non_object_passthrough(self):
|
def test_get_non_object_passthrough(self):
|
||||||
self.app.register('GET', '/info', swob.HTTPOk,
|
self.app.register('GET', '/info', swob.HTTPOk,
|
||||||
|
|
@ -311,7 +313,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
environ={'REQUEST_METHOD': 'GET'})
|
environ={'REQUEST_METHOD': 'GET'})
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
self.assertEqual(status, '200 OK')
|
self.assertEqual(status, '200 OK')
|
||||||
self.assertEqual(body, 'useful stuff here')
|
self.assertEqual(body, b'useful stuff here')
|
||||||
self.assertEqual(self.app.call_count, 1)
|
self.assertEqual(self.app.call_count, 1)
|
||||||
|
|
||||||
def test_get_manifest_passthrough(self):
|
def test_get_manifest_passthrough(self):
|
||||||
|
|
@ -328,7 +330,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(headers["Etag"], "manifest-etag")
|
self.assertEqual(headers["Etag"], "manifest-etag")
|
||||||
self.assertEqual(body, "manifest-contents")
|
self.assertEqual(body, b'manifest-contents')
|
||||||
|
|
||||||
def test_error_passthrough(self):
|
def test_error_passthrough(self):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
|
|
@ -347,7 +349,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "10")
|
self.assertEqual(headers["Content-Length"], "10")
|
||||||
self.assertEqual(body, "bbcccccddd")
|
self.assertEqual(body, b'bbcccccddd')
|
||||||
expected_etag = '"%s"' % md5hex(
|
expected_etag = '"%s"' % md5hex(
|
||||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||||
md5hex("ddddd") + md5hex("eeeee"))
|
md5hex("ddddd") + md5hex("eeeee"))
|
||||||
|
|
@ -361,7 +363,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "10")
|
self.assertEqual(headers["Content-Length"], "10")
|
||||||
self.assertEqual(body, "cccccddddd")
|
self.assertEqual(body, b'cccccddddd')
|
||||||
|
|
||||||
def test_get_range_first_byte(self):
|
def test_get_range_first_byte(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
|
|
@ -371,7 +373,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "1")
|
self.assertEqual(headers["Content-Length"], "1")
|
||||||
self.assertEqual(body, "a")
|
self.assertEqual(body, b'a')
|
||||||
|
|
||||||
def test_get_range_last_byte(self):
|
def test_get_range_last_byte(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
|
|
@ -381,7 +383,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "1")
|
self.assertEqual(headers["Content-Length"], "1")
|
||||||
self.assertEqual(body, "e")
|
self.assertEqual(body, b'e')
|
||||||
|
|
||||||
def test_get_range_overlapping_end(self):
|
def test_get_range_overlapping_end(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
|
|
@ -392,7 +394,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "7")
|
self.assertEqual(headers["Content-Length"], "7")
|
||||||
self.assertEqual(headers["Content-Range"], "bytes 18-24/25")
|
self.assertEqual(headers["Content-Range"], "bytes 18-24/25")
|
||||||
self.assertEqual(body, "ddeeeee")
|
self.assertEqual(body, b'ddeeeee')
|
||||||
|
|
||||||
def test_get_range_unsatisfiable(self):
|
def test_get_range_unsatisfiable(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
|
|
@ -428,7 +430,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
#
|
#
|
||||||
# Since the truth is forbidden, we lie.
|
# Since the truth is forbidden, we lie.
|
||||||
self.assertEqual(headers["Content-Range"], "bytes 3-12/15")
|
self.assertEqual(headers["Content-Range"], "bytes 3-12/15")
|
||||||
self.assertEqual(body, "aabbbbbccc")
|
self.assertEqual(body, b"aabbbbbccc")
|
||||||
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.app.calls,
|
self.app.calls,
|
||||||
|
|
@ -449,7 +451,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
# this requires multiple pages of container listing, so we can't send
|
# this requires multiple pages of container listing, so we can't send
|
||||||
# a Content-Length header
|
# a Content-Length header
|
||||||
self.assertIsNone(headers.get("Content-Length"))
|
self.assertIsNone(headers.get("Content-Length"))
|
||||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
|
||||||
|
|
||||||
def test_get_suffix_range(self):
|
def test_get_suffix_range(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
|
|
@ -459,7 +461,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "25")
|
self.assertEqual(headers["Content-Length"], "25")
|
||||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
|
||||||
|
|
||||||
def test_get_suffix_range_many_segments(self):
|
def test_get_suffix_range_many_segments(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
|
||||||
|
|
@ -471,7 +473,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertIsNone(headers.get("Content-Length"))
|
self.assertIsNone(headers.get("Content-Length"))
|
||||||
self.assertIsNone(headers.get("Content-Range"))
|
self.assertIsNone(headers.get("Content-Range"))
|
||||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
|
||||||
|
|
||||||
def test_get_multi_range(self):
|
def test_get_multi_range(self):
|
||||||
# DLO doesn't support multi-range GETs. The way that you express that
|
# DLO doesn't support multi-range GETs. The way that you express that
|
||||||
|
|
@ -485,7 +487,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertIsNone(headers.get("Content-Length"))
|
self.assertIsNone(headers.get("Content-Length"))
|
||||||
self.assertIsNone(headers.get("Content-Range"))
|
self.assertIsNone(headers.get("Content-Range"))
|
||||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
|
||||||
|
|
||||||
def test_if_match_matches(self):
|
def test_if_match_matches(self):
|
||||||
manifest_etag = '"%s"' % md5hex(
|
manifest_etag = '"%s"' % md5hex(
|
||||||
|
|
@ -500,7 +502,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
|
|
||||||
self.assertEqual(status, '200 OK')
|
self.assertEqual(status, '200 OK')
|
||||||
self.assertEqual(headers['Content-Length'], '25')
|
self.assertEqual(headers['Content-Length'], '25')
|
||||||
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
|
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
|
||||||
|
|
||||||
def test_if_match_does_not_match(self):
|
def test_if_match_does_not_match(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
|
|
@ -512,7 +514,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
|
|
||||||
self.assertEqual(status, '412 Precondition Failed')
|
self.assertEqual(status, '412 Precondition Failed')
|
||||||
self.assertEqual(headers['Content-Length'], '0')
|
self.assertEqual(headers['Content-Length'], '0')
|
||||||
self.assertEqual(body, '')
|
self.assertEqual(body, b'')
|
||||||
|
|
||||||
def test_if_none_match_matches(self):
|
def test_if_none_match_matches(self):
|
||||||
manifest_etag = '"%s"' % md5hex(
|
manifest_etag = '"%s"' % md5hex(
|
||||||
|
|
@ -527,7 +529,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
|
|
||||||
self.assertEqual(status, '304 Not Modified')
|
self.assertEqual(status, '304 Not Modified')
|
||||||
self.assertEqual(headers['Content-Length'], '0')
|
self.assertEqual(headers['Content-Length'], '0')
|
||||||
self.assertEqual(body, '')
|
self.assertEqual(body, b'')
|
||||||
|
|
||||||
def test_if_none_match_does_not_match(self):
|
def test_if_none_match_does_not_match(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
|
|
@ -539,7 +541,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
|
|
||||||
self.assertEqual(status, '200 OK')
|
self.assertEqual(status, '200 OK')
|
||||||
self.assertEqual(headers['Content-Length'], '25')
|
self.assertEqual(headers['Content-Length'], '25')
|
||||||
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
|
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
|
||||||
|
|
||||||
def test_get_with_if_modified_since(self):
|
def test_get_with_if_modified_since(self):
|
||||||
# It's important not to pass the If-[Un]Modified-Since header to the
|
# It's important not to pass the If-[Un]Modified-Since header to the
|
||||||
|
|
@ -581,7 +583,8 @@ class TestDloGetManifest(DloTestCase):
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
|
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertEqual(''.join(body), "aaaaa") # first segment made it out
|
# first segment made it out
|
||||||
|
self.assertEqual(body, b'aaaaa')
|
||||||
self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [
|
self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [
|
||||||
'While processing manifest /v1/AUTH_test/mancon/manifest, '
|
'While processing manifest /v1/AUTH_test/mancon/manifest, '
|
||||||
'got 403 while retrieving /v1/AUTH_test/c/seg_02',
|
'got 403 while retrieving /v1/AUTH_test/c/seg_02',
|
||||||
|
|
@ -610,7 +613,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
with mock.patch(LIMIT, 3):
|
with mock.patch(LIMIT, 3):
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertEqual(body, "aaaaabbbbbccccc")
|
self.assertEqual(body, b'aaaaabbbbbccccc')
|
||||||
|
|
||||||
def test_error_listing_container_HEAD(self):
|
def test_error_listing_container_HEAD(self):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
|
|
@ -639,7 +642,8 @@ class TestDloGetManifest(DloTestCase):
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
|
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error
|
# stop after error
|
||||||
|
self.assertEqual(body, b"aaaaabbWRONGbb")
|
||||||
|
|
||||||
def test_etag_comparison_ignores_quotes(self):
|
def test_etag_comparison_ignores_quotes(self):
|
||||||
# a little future-proofing here in case we ever fix this in swob
|
# a little future-proofing here in case we ever fix this in swob
|
||||||
|
|
@ -662,7 +666,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(headers["Etag"],
|
self.assertEqual(headers["Etag"],
|
||||||
'"' + hashlib.md5("abcdef").hexdigest() + '"')
|
'"' + hashlib.md5(b"abcdef").hexdigest() + '"')
|
||||||
|
|
||||||
def test_object_prefix_quoting(self):
|
def test_object_prefix_quoting(self):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
|
|
@ -675,22 +679,24 @@ class TestDloGetManifest(DloTestCase):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=%C3%A9',
|
'GET', '/v1/AUTH_test/c?prefix=%C3%A9',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json'},
|
swob.HTTPOk, {'Content-Type': 'application/json'},
|
||||||
json.dumps(segs))
|
json.dumps(segs).encode('ascii'))
|
||||||
|
|
||||||
|
# NB: wsgi string
|
||||||
|
path = '/v1/AUTH_test/c/\xC3\xa9'
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/\xC3\xa91',
|
'GET', path + '1',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")},
|
||||||
"AAAAA")
|
b"AAAAA")
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/\xC3\xA92',
|
'GET', path + '2',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")},
|
||||||
"BBBBB")
|
b"BBBBB")
|
||||||
|
|
||||||
req = swob.Request.blank('/v1/AUTH_test/man/accent',
|
req = swob.Request.blank('/v1/AUTH_test/man/accent',
|
||||||
environ={'REQUEST_METHOD': 'GET'})
|
environ={'REQUEST_METHOD': 'GET'})
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertEqual(body, "AAAAABBBBB")
|
self.assertEqual(body, b'AAAAABBBBB')
|
||||||
|
|
||||||
def test_get_taking_too_long(self):
|
def test_get_taking_too_long(self):
|
||||||
the_time = [time.time()]
|
the_time = [time.time()]
|
||||||
|
|
@ -715,7 +721,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
|
|
||||||
self.assertEqual(status, '200 OK')
|
self.assertEqual(status, '200 OK')
|
||||||
self.assertEqual(body, 'aaaaabbbbbccccc')
|
self.assertEqual(body, b'aaaaabbbbbccccc')
|
||||||
|
|
||||||
def test_get_oversize_segment(self):
|
def test_get_oversize_segment(self):
|
||||||
# If we send a Content-Length header to the client, it's based on the
|
# If we send a Content-Length header to the client, it's based on the
|
||||||
|
|
@ -738,7 +744,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
|
|
||||||
self.assertEqual(status, '200 OK') # sanity check
|
self.assertEqual(status, '200 OK') # sanity check
|
||||||
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
|
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
|
||||||
self.assertEqual(body, 'aaaaabbbbbccccccccccccccc')
|
self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.app.calls,
|
self.app.calls,
|
||||||
[('GET', '/v1/AUTH_test/mancon/manifest'),
|
[('GET', '/v1/AUTH_test/mancon/manifest'),
|
||||||
|
|
@ -770,7 +776,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
|
|
||||||
self.assertEqual(status, '200 OK') # sanity check
|
self.assertEqual(status, '200 OK') # sanity check
|
||||||
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
|
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
|
||||||
self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee')
|
self.assertEqual(body, b'aaaaabbbbbccccdddddeeeee')
|
||||||
|
|
||||||
def test_get_undersize_segment_range(self):
|
def test_get_undersize_segment_range(self):
|
||||||
# Shrink it by a single byte
|
# Shrink it by a single byte
|
||||||
|
|
@ -788,7 +794,7 @@ class TestDloGetManifest(DloTestCase):
|
||||||
|
|
||||||
self.assertEqual(status, '206 Partial Content') # sanity check
|
self.assertEqual(status, '206 Partial Content') # sanity check
|
||||||
self.assertEqual(headers.get('Content-Length'), '15') # sanity check
|
self.assertEqual(headers.get('Content-Length'), '15') # sanity check
|
||||||
self.assertEqual(body, 'aaaaabbbbbcccc')
|
self.assertEqual(body, b'aaaaabbbbbcccc')
|
||||||
|
|
||||||
def test_get_with_auth_overridden(self):
|
def test_get_with_auth_overridden(self):
|
||||||
auth_got_called = [0]
|
auth_got_called = [0]
|
||||||
|
|
@ -840,7 +846,7 @@ class TestDloConfiguration(unittest.TestCase):
|
||||||
max_get_time = 2900
|
max_get_time = 2900
|
||||||
""")
|
""")
|
||||||
|
|
||||||
conffile = tempfile.NamedTemporaryFile()
|
conffile = tempfile.NamedTemporaryFile(mode='w')
|
||||||
conffile.write(proxy_conf)
|
conffile.write(proxy_conf)
|
||||||
conffile.flush()
|
conffile.flush()
|
||||||
|
|
||||||
|
|
@ -853,6 +859,8 @@ class TestDloConfiguration(unittest.TestCase):
|
||||||
self.assertEqual(10, mware.rate_limit_after_segment)
|
self.assertEqual(10, mware.rate_limit_after_segment)
|
||||||
self.assertEqual(3600, mware.max_get_time)
|
self.assertEqual(3600, mware.max_get_time)
|
||||||
|
|
||||||
|
conffile.close()
|
||||||
|
|
||||||
def test_finding_defaults_from_file(self):
|
def test_finding_defaults_from_file(self):
|
||||||
# If DLO has no config vars, go pull them from the proxy server's
|
# If DLO has no config vars, go pull them from the proxy server's
|
||||||
# config section
|
# config section
|
||||||
|
|
@ -875,7 +883,7 @@ class TestDloConfiguration(unittest.TestCase):
|
||||||
set max_get_time = 2900
|
set max_get_time = 2900
|
||||||
""")
|
""")
|
||||||
|
|
||||||
conffile = tempfile.NamedTemporaryFile()
|
conffile = tempfile.NamedTemporaryFile(mode='w')
|
||||||
conffile.write(proxy_conf)
|
conffile.write(proxy_conf)
|
||||||
conffile.flush()
|
conffile.flush()
|
||||||
|
|
||||||
|
|
@ -887,6 +895,8 @@ class TestDloConfiguration(unittest.TestCase):
|
||||||
self.assertEqual(13, mware.rate_limit_after_segment)
|
self.assertEqual(13, mware.rate_limit_after_segment)
|
||||||
self.assertEqual(2900, mware.max_get_time)
|
self.assertEqual(2900, mware.max_get_time)
|
||||||
|
|
||||||
|
conffile.close()
|
||||||
|
|
||||||
def test_finding_defaults_from_dir(self):
|
def test_finding_defaults_from_dir(self):
|
||||||
# If DLO has no config vars, go pull them from the proxy server's
|
# If DLO has no config vars, go pull them from the proxy server's
|
||||||
# config section
|
# config section
|
||||||
|
|
@ -913,11 +923,13 @@ class TestDloConfiguration(unittest.TestCase):
|
||||||
|
|
||||||
conf_dir = self.tmpdir
|
conf_dir = self.tmpdir
|
||||||
|
|
||||||
conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
|
conffile1 = tempfile.NamedTemporaryFile(mode='w',
|
||||||
|
dir=conf_dir, suffix='.conf')
|
||||||
conffile1.write(proxy_conf1)
|
conffile1.write(proxy_conf1)
|
||||||
conffile1.flush()
|
conffile1.flush()
|
||||||
|
|
||||||
conffile2 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
|
conffile2 = tempfile.NamedTemporaryFile(mode='w',
|
||||||
|
dir=conf_dir, suffix='.conf')
|
||||||
conffile2.write(proxy_conf2)
|
conffile2.write(proxy_conf2)
|
||||||
conffile2.flush()
|
conffile2.flush()
|
||||||
|
|
||||||
|
|
@ -929,6 +941,9 @@ class TestDloConfiguration(unittest.TestCase):
|
||||||
self.assertEqual(13, mware.rate_limit_after_segment)
|
self.assertEqual(13, mware.rate_limit_after_segment)
|
||||||
self.assertEqual(2900, mware.max_get_time)
|
self.assertEqual(2900, mware.max_get_time)
|
||||||
|
|
||||||
|
conffile1.close()
|
||||||
|
conffile2.close()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
||||||
|
|
@ -528,7 +528,7 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
def test_container3indexhtml(self):
|
def test_container3indexhtml(self):
|
||||||
resp = Request.blank('/v1/a/c3/').get_response(self.test_staticweb)
|
resp = Request.blank('/v1/a/c3/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue('Test main index.html file.' in resp.body)
|
self.assertIn(b'Test main index.html file.', resp.body)
|
||||||
|
|
||||||
def test_container3subsubdir(self):
|
def test_container3subsubdir(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
|
|
@ -539,16 +539,16 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c3/subdir3/subsubdir/').get_response(self.test_staticweb)
|
'/v1/a/c3/subdir3/subsubdir/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertEqual(resp.body, 'index file')
|
self.assertEqual(resp.body, b'index file')
|
||||||
|
|
||||||
def test_container3subdir(self):
|
def test_container3subdir(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c3/subdir/').get_response(self.test_staticweb)
|
'/v1/a/c3/subdir/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertIn('Listing of /v1/a/c3/subdir/', resp.body)
|
self.assertIn(b'Listing of /v1/a/c3/subdir/', resp.body)
|
||||||
self.assertIn('</style>', resp.body)
|
self.assertIn(b'</style>', resp.body)
|
||||||
self.assertNotIn('<link', resp.body)
|
self.assertNotIn(b'<link', resp.body)
|
||||||
self.assertNotIn('listing.css', resp.body)
|
self.assertNotIn(b'listing.css', resp.body)
|
||||||
|
|
||||||
def test_container3subdirx(self):
|
def test_container3subdirx(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
|
|
@ -569,18 +569,18 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c3/unknown').get_response(self.test_staticweb)
|
'/v1/a/c3/unknown').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 404)
|
self.assertEqual(resp.status_int, 404)
|
||||||
self.assertNotIn("Chrome's 404 fancy-page sucks.", resp.body)
|
self.assertNotIn(b"Chrome's 404 fancy-page sucks.", resp.body)
|
||||||
|
|
||||||
def test_container3bindexhtml(self):
|
def test_container3bindexhtml(self):
|
||||||
resp = Request.blank('/v1/a/c3b/').get_response(self.test_staticweb)
|
resp = Request.blank('/v1/a/c3b/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 204)
|
self.assertEqual(resp.status_int, 204)
|
||||||
self.assertEqual(resp.body, '')
|
self.assertEqual(resp.body, b'')
|
||||||
|
|
||||||
def test_container4indexhtml(self):
|
def test_container4indexhtml(self):
|
||||||
resp = Request.blank('/v1/a/c4/').get_response(self.test_staticweb)
|
resp = Request.blank('/v1/a/c4/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertIn('Listing of /v1/a/c4/', resp.body)
|
self.assertIn(b'Listing of /v1/a/c4/', resp.body)
|
||||||
self.assertIn('href="listing.css"', resp.body)
|
self.assertIn(b'href="listing.css"', resp.body)
|
||||||
|
|
||||||
def test_container4indexhtmlauthed(self):
|
def test_container4indexhtmlauthed(self):
|
||||||
resp = Request.blank('/v1/a/c4').get_response(self.test_staticweb)
|
resp = Request.blank('/v1/a/c4').get_response(self.test_staticweb)
|
||||||
|
|
@ -600,16 +600,16 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c4/unknown').get_response(self.test_staticweb)
|
'/v1/a/c4/unknown').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 404)
|
self.assertEqual(resp.status_int, 404)
|
||||||
self.assertIn("Chrome's 404 fancy-page sucks.", resp.body)
|
self.assertIn(b"Chrome's 404 fancy-page sucks.", resp.body)
|
||||||
|
|
||||||
def test_container4subdir(self):
|
def test_container4subdir(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c4/subdir/').get_response(self.test_staticweb)
|
'/v1/a/c4/subdir/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertIn('Listing of /v1/a/c4/subdir/', resp.body)
|
self.assertIn(b'Listing of /v1/a/c4/subdir/', resp.body)
|
||||||
self.assertNotIn('</style>', resp.body)
|
self.assertNotIn(b'</style>', resp.body)
|
||||||
self.assertIn('<link', resp.body)
|
self.assertIn(b'<link', resp.body)
|
||||||
self.assertIn('href="../listing.css"', resp.body)
|
self.assertIn(b'href="../listing.css"', resp.body)
|
||||||
self.assertEqual(resp.headers['content-type'],
|
self.assertEqual(resp.headers['content-type'],
|
||||||
'text/html; charset=UTF-8')
|
'text/html; charset=UTF-8')
|
||||||
|
|
||||||
|
|
@ -631,7 +631,7 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c5/unknown').get_response(self.test_staticweb)
|
'/v1/a/c5/unknown').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 404)
|
self.assertEqual(resp.status_int, 404)
|
||||||
self.assertNotIn("Chrome's 404 fancy-page sucks.", resp.body)
|
self.assertNotIn(b"Chrome's 404 fancy-page sucks.", resp.body)
|
||||||
|
|
||||||
def test_container6subdir(self):
|
def test_container6subdir(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
|
|
@ -649,7 +649,7 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
staticweb.filter_factory({})(self.app), deny_listing=True)
|
staticweb.filter_factory({})(self.app), deny_listing=True)
|
||||||
resp = Request.blank('/v1/a/c6/').get_response(test_staticweb)
|
resp = Request.blank('/v1/a/c6/').get_response(test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 401)
|
self.assertEqual(resp.status_int, 401)
|
||||||
self.assertIn("Hey, you're not authorized to see this!", resp.body)
|
self.assertIn(b"Hey, you're not authorized to see this!", resp.body)
|
||||||
|
|
||||||
# expect default 401 if request is not auth'd for listing or object GET
|
# expect default 401 if request is not auth'd for listing or object GET
|
||||||
test_staticweb = FakeAuthFilter(
|
test_staticweb = FakeAuthFilter(
|
||||||
|
|
@ -657,20 +657,20 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
deny_objects=True)
|
deny_objects=True)
|
||||||
resp = Request.blank('/v1/a/c6/').get_response(test_staticweb)
|
resp = Request.blank('/v1/a/c6/').get_response(test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 401)
|
self.assertEqual(resp.status_int, 401)
|
||||||
self.assertNotIn("Hey, you're not authorized to see this!", resp.body)
|
self.assertNotIn(b"Hey, you're not authorized to see this!", resp.body)
|
||||||
|
|
||||||
def test_container6blisting(self):
|
def test_container6blisting(self):
|
||||||
label = 'Listing of {0}/'.format(
|
label = 'Listing of {0}/'.format(
|
||||||
meta_map['c6b']['meta']['web-listings-label'])
|
meta_map['c6b']['meta']['web-listings-label'])
|
||||||
resp = Request.blank('/v1/a/c6b/').get_response(self.test_staticweb)
|
resp = Request.blank('/v1/a/c6b/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertIn(label, resp.body)
|
self.assertIn(label.encode('utf-8'), resp.body)
|
||||||
|
|
||||||
def test_container7listing(self):
|
def test_container7listing(self):
|
||||||
# container7 has web-listings = f, web-error=error.html
|
# container7 has web-listings = f, web-error=error.html
|
||||||
resp = Request.blank('/v1/a/c7/').get_response(self.test_staticweb)
|
resp = Request.blank('/v1/a/c7/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 404)
|
self.assertEqual(resp.status_int, 404)
|
||||||
self.assertIn("Web Listing Disabled", resp.body)
|
self.assertIn(b"Web Listing Disabled", resp.body)
|
||||||
|
|
||||||
# expect 301 if auth'd but no trailing '/'
|
# expect 301 if auth'd but no trailing '/'
|
||||||
resp = Request.blank('/v1/a/c7').get_response(self.test_staticweb)
|
resp = Request.blank('/v1/a/c7').get_response(self.test_staticweb)
|
||||||
|
|
@ -682,14 +682,14 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
deny_objects=True)
|
deny_objects=True)
|
||||||
resp = Request.blank('/v1/a/c7').get_response(test_staticweb)
|
resp = Request.blank('/v1/a/c7').get_response(test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 401)
|
self.assertEqual(resp.status_int, 401)
|
||||||
self.assertNotIn("Hey, you're not authorized to see this!", resp.body)
|
self.assertNotIn(b"Hey, you're not authorized to see this!", resp.body)
|
||||||
|
|
||||||
# expect custom 401 if request is not auth'd for listing
|
# expect custom 401 if request is not auth'd for listing
|
||||||
test_staticweb = FakeAuthFilter(
|
test_staticweb = FakeAuthFilter(
|
||||||
staticweb.filter_factory({})(self.app), deny_listing=True)
|
staticweb.filter_factory({})(self.app), deny_listing=True)
|
||||||
resp = Request.blank('/v1/a/c7/').get_response(test_staticweb)
|
resp = Request.blank('/v1/a/c7/').get_response(test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 401)
|
self.assertEqual(resp.status_int, 401)
|
||||||
self.assertIn("Hey, you're not authorized to see this!", resp.body)
|
self.assertIn(b"Hey, you're not authorized to see this!", resp.body)
|
||||||
|
|
||||||
# expect default 401 if request is not auth'd for listing or object GET
|
# expect default 401 if request is not auth'd for listing or object GET
|
||||||
test_staticweb = FakeAuthFilter(
|
test_staticweb = FakeAuthFilter(
|
||||||
|
|
@ -697,69 +697,69 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
deny_objects=True)
|
deny_objects=True)
|
||||||
resp = Request.blank('/v1/a/c7/').get_response(test_staticweb)
|
resp = Request.blank('/v1/a/c7/').get_response(test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 401)
|
self.assertEqual(resp.status_int, 401)
|
||||||
self.assertNotIn("Hey, you're not authorized to see this!", resp.body)
|
self.assertNotIn(b"Hey, you're not authorized to see this!", resp.body)
|
||||||
|
|
||||||
def test_container8listingcss(self):
|
def test_container8listingcss(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c8/').get_response(self.test_staticweb)
|
'/v1/a/c8/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue('Listing of /v1/a/c8/' in resp.body)
|
self.assertIn(b'Listing of /v1/a/c8/', resp.body)
|
||||||
self.assertTrue('<link' in resp.body)
|
self.assertIn(b'<link', resp.body)
|
||||||
self.assertTrue(
|
self.assertIn(b'href="http://localhost/stylesheets/listing.css"',
|
||||||
'href="http://localhost/stylesheets/listing.css"' in resp.body)
|
resp.body)
|
||||||
|
|
||||||
def test_container8subdirlistingcss(self):
|
def test_container8subdirlistingcss(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c8/subdir/').get_response(self.test_staticweb)
|
'/v1/a/c8/subdir/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue('Listing of /v1/a/c8/subdir/' in resp.body)
|
self.assertIn(b'Listing of /v1/a/c8/subdir/', resp.body)
|
||||||
self.assertTrue('<link' in resp.body)
|
self.assertIn(b'<link', resp.body)
|
||||||
self.assertTrue(
|
self.assertIn(b'href="http://localhost/stylesheets/listing.css"',
|
||||||
'href="http://localhost/stylesheets/listing.css"' in resp.body)
|
resp.body)
|
||||||
|
|
||||||
def test_container9listingcss(self):
|
def test_container9listingcss(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c9/').get_response(self.test_staticweb)
|
'/v1/a/c9/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue('Listing of /v1/a/c9/' in resp.body)
|
self.assertIn(b'Listing of /v1/a/c9/', resp.body)
|
||||||
self.assertTrue('<link' in resp.body)
|
self.assertIn(b'<link', resp.body)
|
||||||
self.assertTrue('href="/absolute/listing.css"' in resp.body)
|
self.assertIn(b'href="/absolute/listing.css"', resp.body)
|
||||||
|
|
||||||
def test_container9subdirlistingcss(self):
|
def test_container9subdirlistingcss(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c9/subdir/').get_response(self.test_staticweb)
|
'/v1/a/c9/subdir/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue('Listing of /v1/a/c9/subdir/' in resp.body)
|
self.assertIn(b'Listing of /v1/a/c9/subdir/', resp.body)
|
||||||
self.assertTrue('<link' in resp.body)
|
self.assertIn(b'<link', resp.body)
|
||||||
self.assertTrue('href="/absolute/listing.css"' in resp.body)
|
self.assertIn(b'href="/absolute/listing.css"', resp.body)
|
||||||
|
|
||||||
def test_container10unicodesubdirlisting(self):
|
def test_container10unicodesubdirlisting(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c10/').get_response(self.test_staticweb)
|
'/v1/a/c10/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue('Listing of /v1/a/c10/' in resp.body)
|
self.assertIn(b'Listing of /v1/a/c10/', resp.body)
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c10/\xe2\x98\x83/').get_response(self.test_staticweb)
|
'/v1/a/c10/\xe2\x98\x83/').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue('Listing of /v1/a/c10/\xe2\x98\x83/' in resp.body)
|
self.assertIn(b'Listing of /v1/a/c10/\xe2\x98\x83/', resp.body)
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/'
|
'/v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/'
|
||||||
).get_response(self.test_staticweb)
|
).get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue(
|
self.assertIn(
|
||||||
'Listing of /v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/' in resp.body)
|
b'Listing of /v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/', resp.body)
|
||||||
|
|
||||||
def test_container11subdirmarkerobjectindex(self):
|
def test_container11subdirmarkerobjectindex(self):
|
||||||
resp = Request.blank('/v1/a/c11/subdir/').get_response(
|
resp = Request.blank('/v1/a/c11/subdir/').get_response(
|
||||||
self.test_staticweb)
|
self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertTrue('<h2>c11 subdir index</h2>' in resp.body)
|
self.assertIn(b'<h2>c11 subdir index</h2>', resp.body)
|
||||||
|
|
||||||
def test_container11subdirmarkermatchdirtype(self):
|
def test_container11subdirmarkermatchdirtype(self):
|
||||||
resp = Request.blank('/v1/a/c11a/subdir/').get_response(
|
resp = Request.blank('/v1/a/c11a/subdir/').get_response(
|
||||||
self.test_staticweb)
|
self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 404)
|
self.assertEqual(resp.status_int, 404)
|
||||||
self.assertIn('Index File Not Found', resp.body)
|
self.assertIn(b'Index File Not Found', resp.body)
|
||||||
|
|
||||||
def test_container11subdirmarkeraltdirtype(self):
|
def test_container11subdirmarkeraltdirtype(self):
|
||||||
resp = Request.blank('/v1/a/c11a/subdir2/').get_response(
|
resp = Request.blank('/v1/a/c11a/subdir2/').get_response(
|
||||||
|
|
@ -775,27 +775,27 @@ class TestStaticWeb(unittest.TestCase):
|
||||||
resp = Request.blank('/v1/a/c12/').get_response(
|
resp = Request.blank('/v1/a/c12/').get_response(
|
||||||
self.test_staticweb)
|
self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertIn('index file', resp.body)
|
self.assertIn(b'index file', resp.body)
|
||||||
|
|
||||||
def test_container_404_has_css(self):
|
def test_container_404_has_css(self):
|
||||||
resp = Request.blank('/v1/a/c13/').get_response(
|
resp = Request.blank('/v1/a/c13/').get_response(
|
||||||
self.test_staticweb)
|
self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 404)
|
self.assertEqual(resp.status_int, 404)
|
||||||
self.assertIn('listing.css', resp.body)
|
self.assertIn(b'listing.css', resp.body)
|
||||||
|
|
||||||
def test_container_404_has_no_css(self):
|
def test_container_404_has_no_css(self):
|
||||||
resp = Request.blank('/v1/a/c7/').get_response(
|
resp = Request.blank('/v1/a/c7/').get_response(
|
||||||
self.test_staticweb)
|
self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 404)
|
self.assertEqual(resp.status_int, 404)
|
||||||
self.assertNotIn('listing.css', resp.body)
|
self.assertNotIn(b'listing.css', resp.body)
|
||||||
self.assertIn('<style', resp.body)
|
self.assertIn(b'<style', resp.body)
|
||||||
|
|
||||||
def test_subrequest_once_if_possible(self):
|
def test_subrequest_once_if_possible(self):
|
||||||
resp = Request.blank(
|
resp = Request.blank(
|
||||||
'/v1/a/c4/one.txt').get_response(self.test_staticweb)
|
'/v1/a/c4/one.txt').get_response(self.test_staticweb)
|
||||||
self.assertEqual(resp.status_int, 200)
|
self.assertEqual(resp.status_int, 200)
|
||||||
self.assertEqual(resp.headers['x-object-meta-test'], 'value')
|
self.assertEqual(resp.headers['x-object-meta-test'], 'value')
|
||||||
self.assertEqual(resp.body, '1')
|
self.assertEqual(resp.body, b'1')
|
||||||
self.assertEqual(self.app.calls, 1)
|
self.assertEqual(self.app.calls, 1)
|
||||||
|
|
||||||
def test_no_auth_middleware(self):
|
def test_no_auth_middleware(self):
|
||||||
|
|
|
||||||
|
|
@ -2463,28 +2463,6 @@ log_name = %(yarr)s'''
|
||||||
|
|
||||||
self.verify_under_pseudo_time(testfunc, target_runtime_ms=900)
|
self.verify_under_pseudo_time(testfunc, target_runtime_ms=900)
|
||||||
|
|
||||||
def test_urlparse(self):
|
|
||||||
parsed = utils.urlparse('http://127.0.0.1/')
|
|
||||||
self.assertEqual(parsed.scheme, 'http')
|
|
||||||
self.assertEqual(parsed.hostname, '127.0.0.1')
|
|
||||||
self.assertEqual(parsed.path, '/')
|
|
||||||
|
|
||||||
parsed = utils.urlparse('http://127.0.0.1:8080/')
|
|
||||||
self.assertEqual(parsed.port, 8080)
|
|
||||||
|
|
||||||
parsed = utils.urlparse('https://127.0.0.1/')
|
|
||||||
self.assertEqual(parsed.scheme, 'https')
|
|
||||||
|
|
||||||
parsed = utils.urlparse('http://[::1]/')
|
|
||||||
self.assertEqual(parsed.hostname, '::1')
|
|
||||||
|
|
||||||
parsed = utils.urlparse('http://[::1]:8080/')
|
|
||||||
self.assertEqual(parsed.hostname, '::1')
|
|
||||||
self.assertEqual(parsed.port, 8080)
|
|
||||||
|
|
||||||
parsed = utils.urlparse('www.example.com')
|
|
||||||
self.assertEqual(parsed.hostname, '')
|
|
||||||
|
|
||||||
def test_search_tree(self):
|
def test_search_tree(self):
|
||||||
# file match & ext miss
|
# file match & ext miss
|
||||||
with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t:
|
with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t:
|
||||||
|
|
|
||||||
|
|
@ -1171,7 +1171,7 @@ class TestReconciler(unittest.TestCase):
|
||||||
q_path = '.misplaced_objects/%s' % container
|
q_path = '.misplaced_objects/%s' % container
|
||||||
self._mock_listing({
|
self._mock_listing({
|
||||||
(None, "/%s/1:/AUTH_bob/c/o1" % q_path): q_ts,
|
(None, "/%s/1:/AUTH_bob/c/o1" % q_path): q_ts,
|
||||||
(1, '/AUTH_bob/c/o1'): q_ts - 0.00001, # slightly older
|
(1, '/AUTH_bob/c/o1'): q_ts - 1, # slightly older
|
||||||
})
|
})
|
||||||
self._mock_oldest_spi({'c': 0})
|
self._mock_oldest_spi({'c': 0})
|
||||||
deleted_container_entries = self._run_once()
|
deleted_container_entries = self._run_once()
|
||||||
|
|
@ -1411,7 +1411,7 @@ class TestReconciler(unittest.TestCase):
|
||||||
self._mock_listing({
|
self._mock_listing({
|
||||||
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3679.2019,
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3679.2019,
|
||||||
(1, "/AUTH_bob/c/o1"): 3679.2019,
|
(1, "/AUTH_bob/c/o1"): 3679.2019,
|
||||||
(0, "/AUTH_bob/c/o1"): 3679.2019 + 0.00001, # slightly newer
|
(0, "/AUTH_bob/c/o1"): 3679.2019 + 1, # slightly newer
|
||||||
})
|
})
|
||||||
self._mock_oldest_spi({'c': 0})
|
self._mock_oldest_spi({'c': 0})
|
||||||
deleted_container_entries = self._run_once()
|
deleted_container_entries = self._run_once()
|
||||||
|
|
@ -1452,7 +1452,7 @@ class TestReconciler(unittest.TestCase):
|
||||||
self._mock_listing({
|
self._mock_listing({
|
||||||
(None, "/.misplaced_objects/36000/1:/AUTH_bob/c/o1"): 36123.38393,
|
(None, "/.misplaced_objects/36000/1:/AUTH_bob/c/o1"): 36123.38393,
|
||||||
(1, "/AUTH_bob/c/o1"): 36123.38393,
|
(1, "/AUTH_bob/c/o1"): 36123.38393,
|
||||||
(0, "/AUTH_bob/c/o1"): 36123.38393 - 0.00001, # slightly older
|
(0, "/AUTH_bob/c/o1"): 36123.38393 - 1, # slightly older
|
||||||
})
|
})
|
||||||
self._mock_oldest_spi({'c': 0})
|
self._mock_oldest_spi({'c': 0})
|
||||||
deleted_container_entries = self._run_once()
|
deleted_container_entries = self._run_once()
|
||||||
|
|
|
||||||
|
|
@ -678,7 +678,7 @@ class TestAuditor(unittest.TestCase):
|
||||||
auditor_worker.audit_all_objects(device_dirs=['sda'])
|
auditor_worker.audit_all_objects(device_dirs=['sda'])
|
||||||
log_lines = self.logger.get_lines_for_level('info')
|
log_lines = self.logger.get_lines_for_level('info')
|
||||||
self.assertGreater(len(log_lines), 0)
|
self.assertGreater(len(log_lines), 0)
|
||||||
self.assertTrue(log_lines[0].index('ALL - parallel, sda'))
|
self.assertIn('ALL - parallel, sda', log_lines[0])
|
||||||
|
|
||||||
self.logger.clear()
|
self.logger.clear()
|
||||||
auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
|
auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
|
||||||
|
|
@ -687,7 +687,7 @@ class TestAuditor(unittest.TestCase):
|
||||||
auditor_worker.audit_all_objects(device_dirs=['sda'])
|
auditor_worker.audit_all_objects(device_dirs=['sda'])
|
||||||
log_lines = self.logger.get_lines_for_level('info')
|
log_lines = self.logger.get_lines_for_level('info')
|
||||||
self.assertGreater(len(log_lines), 0)
|
self.assertGreater(len(log_lines), 0)
|
||||||
self.assertTrue(log_lines[0].index('ZBF - sda'))
|
self.assertIn('ZBF - sda', log_lines[0])
|
||||||
|
|
||||||
def test_object_run_recon_cache(self):
|
def test_object_run_recon_cache(self):
|
||||||
ts = Timestamp(time.time())
|
ts = Timestamp(time.time())
|
||||||
|
|
|
||||||
|
|
@ -536,9 +536,27 @@ class TestObjectReplicator(unittest.TestCase):
|
||||||
self._write_disk_data('sdd', with_json=True)
|
self._write_disk_data('sdd', with_json=True)
|
||||||
_create_test_rings(self.testdir, devs)
|
_create_test_rings(self.testdir, devs)
|
||||||
|
|
||||||
self.replicator.collect_jobs()
|
self.replicator.collect_jobs(override_partitions=[1])
|
||||||
self.assertEqual(self.replicator.total_stats.failure, 0)
|
self.assertEqual(self.replicator.total_stats.failure, 0)
|
||||||
|
|
||||||
|
def test_collect_jobs_with_override_parts_and_unexpected_part_dir(self):
|
||||||
|
self.replicator.collect_jobs(override_partitions=[0, 2])
|
||||||
|
self.assertEqual(self.replicator.total_stats.failure, 0)
|
||||||
|
os.mkdir(os.path.join(self.objects_1, 'foo'))
|
||||||
|
jobs = self.replicator.collect_jobs(override_partitions=[0, 2])
|
||||||
|
found_jobs = set()
|
||||||
|
for j in jobs:
|
||||||
|
found_jobs.add((int(j['policy']), int(j['partition'])))
|
||||||
|
self.assertEqual(found_jobs, {
|
||||||
|
(0, 0),
|
||||||
|
(0, 2),
|
||||||
|
(1, 0),
|
||||||
|
(1, 2),
|
||||||
|
})
|
||||||
|
num_disks = len(POLICIES[1].object_ring.devs)
|
||||||
|
# N.B. it's not clear why the UUT increments failure per device
|
||||||
|
self.assertEqual(self.replicator.total_stats.failure, num_disks)
|
||||||
|
|
||||||
@mock.patch('swift.obj.replicator.random.shuffle', side_effect=lambda l: l)
|
@mock.patch('swift.obj.replicator.random.shuffle', side_effect=lambda l: l)
|
||||||
def test_collect_jobs_multi_disk(self, mock_shuffle):
|
def test_collect_jobs_multi_disk(self, mock_shuffle):
|
||||||
devs = [
|
devs = [
|
||||||
|
|
@ -1401,10 +1419,10 @@ class TestObjectReplicator(unittest.TestCase):
|
||||||
self.assertTrue(os.access(part_path, os.F_OK))
|
self.assertTrue(os.access(part_path, os.F_OK))
|
||||||
self.replicator.replicate(override_devices=['sdb'])
|
self.replicator.replicate(override_devices=['sdb'])
|
||||||
self.assertTrue(os.access(part_path, os.F_OK))
|
self.assertTrue(os.access(part_path, os.F_OK))
|
||||||
self.replicator.replicate(override_partitions=['9'])
|
self.replicator.replicate(override_partitions=[9])
|
||||||
self.assertTrue(os.access(part_path, os.F_OK))
|
self.assertTrue(os.access(part_path, os.F_OK))
|
||||||
self.replicator.replicate(override_devices=['sda'],
|
self.replicator.replicate(override_devices=['sda'],
|
||||||
override_partitions=['1'])
|
override_partitions=[1])
|
||||||
self.assertFalse(os.access(part_path, os.F_OK))
|
self.assertFalse(os.access(part_path, os.F_OK))
|
||||||
|
|
||||||
def test_delete_policy_override_params(self):
|
def test_delete_policy_override_params(self):
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,6 @@ function is_rhel7 {
|
||||||
if is_rhel7; then
|
if is_rhel7; then
|
||||||
# Install CentOS OpenStack repos so that we have access to some extra
|
# Install CentOS OpenStack repos so that we have access to some extra
|
||||||
# packages.
|
# packages.
|
||||||
sudo yum install -y centos-release-openstack-queens
|
sudo yum install -y centos-release-openstack-rocky
|
||||||
sudo yum install -y liberasurecode-devel
|
sudo yum install -y liberasurecode-devel
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
37
tox.ini
37
tox.ini
|
|
@ -1,18 +1,19 @@
|
||||||
[tox]
|
[tox]
|
||||||
envlist = py35,py27,pep8
|
envlist = py37,py27,pep8
|
||||||
minversion = 2.3.2
|
minversion = 2.3.2
|
||||||
skipsdist = True
|
skipsdist = True
|
||||||
|
|
||||||
[testenv]
|
[testenv]
|
||||||
usedevelop = True
|
usedevelop = True
|
||||||
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} -U {opts} {packages}
|
install_command = pip install -U {opts} {packages}
|
||||||
setenv = VIRTUAL_ENV={envdir}
|
setenv = VIRTUAL_ENV={envdir}
|
||||||
NOSE_WITH_COVERAGE=1
|
NOSE_WITH_COVERAGE=1
|
||||||
NOSE_COVER_BRANCHES=1
|
NOSE_COVER_BRANCHES=1
|
||||||
deps =
|
deps =
|
||||||
|
-c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
|
||||||
-r{toxinidir}/requirements.txt
|
-r{toxinidir}/requirements.txt
|
||||||
-r{toxinidir}/test-requirements.txt
|
-r{toxinidir}/test-requirements.txt
|
||||||
commands = find . -type f -name "*.py[c|o]" -delete
|
commands = find . ( -type f -o -type l ) -name "*.py[c|o]" -delete
|
||||||
find . -type d -name "__pycache__" -delete
|
find . -type d -name "__pycache__" -delete
|
||||||
nosetests {posargs:test/unit}
|
nosetests {posargs:test/unit}
|
||||||
whitelist_externals = find
|
whitelist_externals = find
|
||||||
|
|
@ -31,12 +32,15 @@ setenv = VIRTUAL_ENV={envdir}
|
||||||
# tests file by file, add them into the list below, and never go back.
|
# tests file by file, add them into the list below, and never go back.
|
||||||
# This list also serves as a shared task board for those helping with py3.
|
# This list also serves as a shared task board for those helping with py3.
|
||||||
# But mind that reviews expanding this list may be outstanding in Gerrit.
|
# But mind that reviews expanding this list may be outstanding in Gerrit.
|
||||||
[testenv:py35]
|
[testenv:py37]
|
||||||
commands =
|
commands =
|
||||||
|
find . ( -type f -o -type l ) -name "*.py[c|o]" -delete
|
||||||
|
find . -type d -name "__pycache__" -delete
|
||||||
nosetests {posargs:\
|
nosetests {posargs:\
|
||||||
test/unit/account \
|
test/unit/account \
|
||||||
test/unit/cli \
|
test/unit/cli \
|
||||||
test/unit/common/middleware/crypto \
|
test/unit/common/middleware/crypto \
|
||||||
|
test/unit/common/middleware/s3api/test_s3token.py \
|
||||||
test/unit/common/middleware/test_account_quotas.py \
|
test/unit/common/middleware/test_account_quotas.py \
|
||||||
test/unit/common/middleware/test_acl.py \
|
test/unit/common/middleware/test_acl.py \
|
||||||
test/unit/common/middleware/test_catch_errors.py \
|
test/unit/common/middleware/test_catch_errors.py \
|
||||||
|
|
@ -44,6 +48,7 @@ commands =
|
||||||
test/unit/common/middleware/test_container_sync.py \
|
test/unit/common/middleware/test_container_sync.py \
|
||||||
test/unit/common/middleware/test_copy.py \
|
test/unit/common/middleware/test_copy.py \
|
||||||
test/unit/common/middleware/test_crossdomain.py \
|
test/unit/common/middleware/test_crossdomain.py \
|
||||||
|
test/unit/common/middleware/test_dlo.py \
|
||||||
test/unit/common/middleware/test_domain_remap.py \
|
test/unit/common/middleware/test_domain_remap.py \
|
||||||
test/unit/common/middleware/test_formpost.py \
|
test/unit/common/middleware/test_formpost.py \
|
||||||
test/unit/common/middleware/test_gatekeeper.py \
|
test/unit/common/middleware/test_gatekeeper.py \
|
||||||
|
|
@ -59,6 +64,7 @@ commands =
|
||||||
test/unit/common/middleware/test_read_only.py \
|
test/unit/common/middleware/test_read_only.py \
|
||||||
test/unit/common/middleware/test_recon.py \
|
test/unit/common/middleware/test_recon.py \
|
||||||
test/unit/common/middleware/test_subrequest_logging.py \
|
test/unit/common/middleware/test_subrequest_logging.py \
|
||||||
|
test/unit/common/middleware/test_staticweb.py \
|
||||||
test/unit/common/middleware/test_tempauth.py \
|
test/unit/common/middleware/test_tempauth.py \
|
||||||
test/unit/common/middleware/test_versioned_writes.py \
|
test/unit/common/middleware/test_versioned_writes.py \
|
||||||
test/unit/common/middleware/test_xprofile.py \
|
test/unit/common/middleware/test_xprofile.py \
|
||||||
|
|
@ -95,31 +101,23 @@ commands =
|
||||||
test/unit/proxy/controllers/test_obj.py}
|
test/unit/proxy/controllers/test_obj.py}
|
||||||
|
|
||||||
[testenv:py36]
|
[testenv:py36]
|
||||||
commands = {[testenv:py35]commands}
|
commands = {[testenv:py37]commands}
|
||||||
|
|
||||||
[testenv:py37]
|
[testenv:py35]
|
||||||
commands = {[testenv:py35]commands}
|
commands = {[testenv:py37]commands}
|
||||||
|
|
||||||
[testenv:pep8]
|
[testenv:pep8]
|
||||||
basepython = python2.7
|
basepython = python2.7
|
||||||
commands =
|
commands =
|
||||||
flake8 {posargs:swift test doc setup.py}
|
flake8 {posargs:swift test doc setup.py}
|
||||||
flake8 --filename=swift* bin
|
flake8 --filename=swift* bin
|
||||||
|
python ./setup.py check --restructuredtext --strict
|
||||||
bandit -c bandit.yaml -r swift -n 5
|
bandit -c bandit.yaml -r swift -n 5
|
||||||
./.manpages {posargs}
|
./.manpages {posargs}
|
||||||
|
|
||||||
[testenv:py3pep8]
|
[testenv:py3pep8]
|
||||||
basepython = python3
|
basepython = python3
|
||||||
install_command = echo {packages}
|
commands = {[testenv:pep8]commands}
|
||||||
whitelist_externals = echo
|
|
||||||
commands =
|
|
||||||
# Gross hack. There's no other way to get it to /not/ install swift itself
|
|
||||||
# (which triggers installing eventlet) but also get flake8 installed.
|
|
||||||
pip install flake8
|
|
||||||
flake8 swift test doc setup.py
|
|
||||||
flake8 --filename=swift* bin
|
|
||||||
python ./setup.py check --restructuredtext --strict
|
|
||||||
./.manpages {posargs}
|
|
||||||
|
|
||||||
[testenv:func]
|
[testenv:func]
|
||||||
basepython = python2.7
|
basepython = python2.7
|
||||||
|
|
@ -149,14 +147,14 @@ setenv = SWIFT_TEST_IN_PROCESS=1
|
||||||
commands = {posargs}
|
commands = {posargs}
|
||||||
|
|
||||||
[testenv:docs]
|
[testenv:docs]
|
||||||
basepython = python2.7
|
basepython = python3
|
||||||
deps = -r{toxinidir}/doc/requirements.txt
|
deps = -r{toxinidir}/doc/requirements.txt
|
||||||
commands = sphinx-build -W -b html doc/source doc/build/html
|
commands = sphinx-build -W -b html doc/source doc/build/html
|
||||||
|
|
||||||
[testenv:api-ref]
|
[testenv:api-ref]
|
||||||
# This environment is called from CI scripts to test and publish
|
# This environment is called from CI scripts to test and publish
|
||||||
# the API Ref to developer.openstack.org.
|
# the API Ref to developer.openstack.org.
|
||||||
basepython = python2.7
|
basepython = python3
|
||||||
deps = -r{toxinidir}/doc/requirements.txt
|
deps = -r{toxinidir}/doc/requirements.txt
|
||||||
commands =
|
commands =
|
||||||
rm -rf api-ref/build
|
rm -rf api-ref/build
|
||||||
|
|
@ -191,6 +189,7 @@ deps = bindep
|
||||||
commands = bindep test
|
commands = bindep test
|
||||||
|
|
||||||
[testenv:releasenotes]
|
[testenv:releasenotes]
|
||||||
|
basepython = python3
|
||||||
deps = -r{toxinidir}/doc/requirements.txt
|
deps = -r{toxinidir}/doc/requirements.txt
|
||||||
commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
|
commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue