diff --git a/.zuul.yaml b/.zuul.yaml index 885c821793..5b9c3fc951 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -331,6 +331,7 @@ irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ - ^test/(functional|probe)/.*$ + voting: false - swift-tox-py36: irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ @@ -340,7 +341,6 @@ irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ - ^test/(functional|probe)/.*$ - voting: false - swift-tox-func: irrelevant-files: - ^(api-ref|doc|releasenotes)/.*$ @@ -412,7 +412,7 @@ # long-running jobs, like probetests or (once they move to # in-tree definitions) dsvm jobs. - swift-tox-py27 - - swift-tox-py35 + - swift-tox-py37 - swift-tox-func - swift-tox-func-encryption - swift-tox-func-domain-remap-staticweb diff --git a/CHANGELOG b/CHANGELOG index a66cb9b23e..8f2ebc87ac 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1888,7 +1888,7 @@ swift (1.13.1, OpenStack Icehouse) A new proxy config variable (strict_cors_mode, default to True) has been added. Setting it to False keeps the old behavior. For an overview of old versus new behavior, please see - https://review.openstack.org/#/c/69419/ + https://review.opendev.org/#/c/69419/ * Invert the responsibility of the two instances of proxy-logging in the proxy pipeline diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index bbffc24479..3aef4845f4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -52,7 +52,7 @@ Reviewing Someone Else's Code ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ All code reviews in OpenStack projects are done on -https://review.openstack.org/. Reviewing patches is one of the most effective +https://review.opendev.org/. Reviewing patches is one of the most effective ways you can contribute to the community. We've written REVIEW_GUIDELINES.rst (found in this source tree) to help you diff --git a/api-ref/source/conf.py b/api-ref/source/conf.py index 9c30acbb37..5d7f7f3edc 100644 --- a/api-ref/source/conf.py +++ b/api-ref/source/conf.py @@ -162,6 +162,11 @@ try: except OSError: warnings.warn('Cannot get last updated time from git repository. ' 'Not setting "html_last_updated_fmt".') +else: + if not isinstance(html_last_updated_fmt, str): + # for py3 + html_last_updated_fmt = html_last_updated_fmt.decode('ascii') + # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index bc5ed086f6..3950dcfc89 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -1109,6 +1109,8 @@ multipart-manifest_put: description: | If you include the ``multipart-manifest=put`` query parameter, the object is a static large object manifest and the body contains the manifest. + See `Static large objects `_ for more information. in: query required: false type: string diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index dbadef4670..c01e284d91 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -2252,7 +2252,7 @@ For distros with more recent kernels (for example Ubuntu 12.04 Precise), we recommend using the default settings (including the default inode size of 256 bytes) when creating the file system:: - mkfs.xfs /dev/sda1 + mkfs.xfs -L D1 /dev/sda1 In the last couple of years, XFS has made great improvements in how inodes are allocated and used. Using the default inode size no longer has an @@ -2262,7 +2262,7 @@ For distros with older kernels (for example Ubuntu 10.04 Lucid), some settings can dramatically impact performance. We recommend the following when creating the file system:: - mkfs.xfs -i size=1024 /dev/sda1 + mkfs.xfs -i size=1024 -L D1 /dev/sda1 Setting the inode size is important, as XFS stores xattr data in the inode. If the metadata is too large to fit in the inode, a new extent is created, @@ -2272,15 +2272,15 @@ headroom. The following example mount options are recommended when using XFS:: - mount -t xfs -o noatime,nodiratime,nobarrier,logbufs=8 /dev/sda1 /srv/node/sda + mount -t xfs -o noatime,nodiratime,nobarrier,logbufs=8 -L D1 /srv/node/d1 We do not recommend running Swift on RAID, but if you are using RAID it is also important to make sure that the proper sunit and swidth settings get set so that XFS can make most efficient use of the RAID array. For a standard Swift install, all data drives are mounted directly under -``/srv/node`` (as can be seen in the above example of mounting ``/dev/sda1`` as -``/srv/node/sda``). If you choose to mount the drives in another directory, +``/srv/node`` (as can be seen in the above example of mounting label ``D1`` +as ``/srv/node/d1``). If you choose to mount the drives in another directory, be sure to set the `devices` config option in all of the server configs to point to the correct directory. @@ -2322,7 +2322,7 @@ The following settings should be in `/etc/sysctl.conf`:: # double amount of allowed conntrack net.ipv4.netfilter.ip_conntrack_max = 262144 -To load the updated sysctl settings, run ``sudo sysctl -p`` +To load the updated sysctl settings, run ``sudo sysctl -p``. A note about changing the TIME_WAIT values. By default the OS will hold a port open for 60 seconds to ensure that any remaining packets can be diff --git a/doc/source/first_contribution_swift.rst b/doc/source/first_contribution_swift.rst index 5e00b7cc4c..cabfd0b3ca 100644 --- a/doc/source/first_contribution_swift.rst +++ b/doc/source/first_contribution_swift.rst @@ -117,7 +117,7 @@ Tracking your changes --------------------- After proposing changes to Swift, you can track them at -https://review.openstack.org. After logging in, you will see a dashboard of +https://review.opendev.org. After logging in, you will see a dashboard of "Outgoing reviews" for changes you have proposed, "Incoming reviews" for changes you are reviewing, and "Recently closed" changes for which you were either a reviewer or owner. diff --git a/doc/source/install/controller-install-debian.rst b/doc/source/install/controller-install-debian.rst index 9ab3294760..4d6ca523fe 100644 --- a/doc/source/install/controller-install-debian.rst +++ b/doc/source/install/controller-install-debian.rst @@ -47,4 +47,4 @@ Install and configure components .. code-block:: console - # curl -o /etc/swift/proxy-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/proxy-server.conf-sample?h=stable/queens + # curl -o /etc/swift/proxy-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/proxy-server.conf-sample diff --git a/doc/source/install/controller-install-rdo.rst b/doc/source/install/controller-install-rdo.rst index db082fd364..262577ffdc 100644 --- a/doc/source/install/controller-install-rdo.rst +++ b/doc/source/install/controller-install-rdo.rst @@ -45,6 +45,6 @@ Install and configure components .. code-block:: console - # curl -o /etc/swift/proxy-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/proxy-server.conf-sample?h=stable/queens + # curl -o /etc/swift/proxy-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/proxy-server.conf-sample 3. .. include:: controller-include.txt diff --git a/doc/source/install/controller-install-ubuntu.rst b/doc/source/install/controller-install-ubuntu.rst index d2c987a46f..caf2b8efa5 100644 --- a/doc/source/install/controller-install-ubuntu.rst +++ b/doc/source/install/controller-install-ubuntu.rst @@ -47,6 +47,6 @@ Install and configure components .. code-block:: console - # curl -o /etc/swift/proxy-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/proxy-server.conf-sample?h=stable/queens + # curl -o /etc/swift/proxy-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/proxy-server.conf-sample 4. .. include:: controller-include.txt diff --git a/doc/source/install/finalize-installation-rdo.rst b/doc/source/install/finalize-installation-rdo.rst index 893412a5f5..5101b27a03 100644 --- a/doc/source/install/finalize-installation-rdo.rst +++ b/doc/source/install/finalize-installation-rdo.rst @@ -19,7 +19,7 @@ This section applies to Red Hat Enterprise Linux 7 and CentOS 7. .. code-block:: console # curl -o /etc/swift/swift.conf \ - https://git.openstack.org/cgit/openstack/swift/plain/etc/swift.conf-sample?h=stable/queens + https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/swift.conf-sample #. Edit the ``/etc/swift/swift.conf`` file and complete the following actions: diff --git a/doc/source/install/finalize-installation-ubuntu-debian.rst b/doc/source/install/finalize-installation-ubuntu-debian.rst index 4178a77bc2..b20f55e636 100644 --- a/doc/source/install/finalize-installation-ubuntu-debian.rst +++ b/doc/source/install/finalize-installation-ubuntu-debian.rst @@ -19,7 +19,7 @@ This section applies to Ubuntu 14.04 (LTS) and Debian. .. code-block:: console # curl -o /etc/swift/swift.conf \ - https://git.openstack.org/cgit/openstack/swift/plain/etc/swift.conf-sample?h=stable/queens + https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/swift.conf-sample #. Edit the ``/etc/swift/swift.conf`` file and complete the following actions: diff --git a/doc/source/install/storage-install-rdo.rst b/doc/source/install/storage-install-rdo.rst index ad21955174..9e714d86b0 100644 --- a/doc/source/install/storage-install-rdo.rst +++ b/doc/source/install/storage-install-rdo.rst @@ -133,9 +133,9 @@ Install and configure components .. code-block:: console - # curl -o /etc/swift/account-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/account-server.conf-sample?h=stable/queens - # curl -o /etc/swift/container-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/container-server.conf-sample?h=stable/queens - # curl -o /etc/swift/object-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/object-server.conf-sample?h=stable/queens + # curl -o /etc/swift/account-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/account-server.conf-sample + # curl -o /etc/swift/container-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/container-server.conf-sample + # curl -o /etc/swift/object-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/object-server.conf-sample 3. .. include:: storage-include1.txt 4. .. include:: storage-include2.txt diff --git a/doc/source/install/storage-install-ubuntu-debian.rst b/doc/source/install/storage-install-ubuntu-debian.rst index 1399e3a520..6ac18a6c88 100644 --- a/doc/source/install/storage-install-ubuntu-debian.rst +++ b/doc/source/install/storage-install-ubuntu-debian.rst @@ -137,9 +137,9 @@ Install and configure components .. code-block:: console - # curl -o /etc/swift/account-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/account-server.conf-sample?h=stable/queens - # curl -o /etc/swift/container-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/container-server.conf-sample?h=stable/queens - # curl -o /etc/swift/object-server.conf https://git.openstack.org/cgit/openstack/swift/plain/etc/object-server.conf-sample?h=stable/queens + # curl -o /etc/swift/account-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/account-server.conf-sample + # curl -o /etc/swift/container-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/container-server.conf-sample + # curl -o /etc/swift/object-server.conf https://opendev.org/openstack/swift/raw/branch/stable/queens/etc/object-server.conf-sample 3. .. include:: storage-include1.txt 4. .. include:: storage-include2.txt diff --git a/lower-constraints.txt b/lower-constraints.txt index 8f698ee1ee..2d89f16625 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -10,7 +10,7 @@ chardet==3.0.4 cliff==2.11.0 cmd2==0.8.1 coverage==3.6 -cryptography==1.6 +cryptography==1.8.2 debtcollector==1.19.0 dnspython==1.14.0 docutils==0.11 @@ -40,7 +40,7 @@ mock==2.0 monotonic==1.4 msgpack==0.5.6 netaddr==0.7.19 -netifaces==0.5 +netifaces==0.8 nose==1.3.7 nosehtmloutput==0.0.3 nosexcover==1.0.10 @@ -49,6 +49,7 @@ os-api-ref==1.0.0 os-testr==0.8.0 oslo.config==4.0.0 oslo.i18n==3.20.0 +oslo.log==3.22.0 oslo.serialization==2.25.0 oslo.utils==3.36.0 PasteDeploy==1.3.3 diff --git a/requirements.txt b/requirements.txt index 65d78c0a69..70458dbdd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,18 +2,18 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -dnspython>=1.14.0;python_version=='2.7' # http://www.dnspython.org/LICENSE -eventlet>=0.17.4,!=0.23.0 # MIT +dnspython>=1.14.0;python_version=='2.7' # http://www.dnspython.org/LICENSE +eventlet>=0.17.4,!=0.23.0 # MIT greenlet>=0.3.1 -netifaces>=0.5,!=0.10.0,!=0.10.1 +netifaces>=0.8,!=0.10.0,!=0.10.1 PasteDeploy>=1.3.3 lxml>=3.4.1 -requests>=2.14.2 # Apache-2.0 +requests>=2.14.2 # Apache-2.0 six>=1.9.0 -xattr>=0.4 +xattr>=0.4;sys_platform!='win32' # MIT PyECLib>=1.3.1 # BSD -cryptography!=2.0,>=1.6 # BSD/Apache-2.0 -ipaddress>=1.0.16;python_version<'3.3' # PSF +cryptography!=2.0,>=1.8.2 # BSD/Apache-2.0 +ipaddress>=1.0.16;python_version<'3.3' # PSF # grpcio will fail to work with eventlet starting with 1.3.5. # see this for a similar issue with gevent: https://github.com/grpc/grpc/issues/4629 and https://github.com/gevent/gevent/issues/786 # don't use eventlet for the object-server ? diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py index 8333bbab67..cef7022642 100644 --- a/swift/common/middleware/acl.py +++ b/swift/common/middleware/acl.py @@ -15,9 +15,7 @@ import json import six -from six.moves.urllib.parse import unquote - -from swift.common.utils import urlparse +from six.moves.urllib.parse import unquote, urlparse def clean_acl(name, value): diff --git a/swift/common/middleware/dlo.py b/swift/common/middleware/dlo.py index 6a15f1390c..5334b1037f 100644 --- a/swift/common/middleware/dlo.py +++ b/swift/common/middleware/dlo.py @@ -121,14 +121,14 @@ Here's an example using ``curl`` with tiny 1-byte segments:: import json import six -from six.moves.urllib.parse import unquote from hashlib import md5 from swift.common import constraints from swift.common.exceptions import ListingIterError, SegmentError from swift.common.http import is_success from swift.common.swob import Request, Response, \ - HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict + HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \ + str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote from swift.common.utils import get_logger, \ RateLimitedIterator, quote, close_if_possible, closing_if_possible from swift.common.request_helpers import SegmentedIterable @@ -143,9 +143,18 @@ class GetContext(WSGIContext): def _get_container_listing(self, req, version, account, container, prefix, marker=''): + ''' + :param version: whatever + :param account: native + :param container: native + :param prefix: native + :param marker: native + ''' con_req = make_subrequest( req.environ, - path=quote('/'.join(['', version, account, container])), + path=wsgi_quote('/'.join([ + '', str_to_wsgi(version), + str_to_wsgi(account), str_to_wsgi(container)])), method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO') @@ -156,14 +165,24 @@ class GetContext(WSGIContext): con_resp = con_req.get_response(self.dlo.app) if not is_success(con_resp.status_int): if req.method == 'HEAD': - con_resp.body = '' + con_resp.body = b'' return con_resp, None with closing_if_possible(con_resp.app_iter): - return None, json.loads(''.join(con_resp.app_iter)) + return None, json.loads(b''.join(con_resp.app_iter)) def _segment_listing_iterator(self, req, version, account, container, prefix, segments, first_byte=None, last_byte=None): + ''' + :param req: upstream request + :param version: native + :param account: native + :param container: native + :param prefix: native + :param segments: array of dicts, with native strings + :param first_byte: number + :param last_byte: number + ''' # It's sort of hokey that this thing takes in the first page of # segments as an argument, but we need to compute the etag and content # length from the first page, and it's better to have a hokey @@ -173,7 +192,6 @@ class GetContext(WSGIContext): if last_byte is None: last_byte = float("inf") - marker = '' while True: for segment in segments: seg_length = int(segment['bytes']) @@ -188,7 +206,7 @@ class GetContext(WSGIContext): break seg_name = segment['name'] - if isinstance(seg_name, six.text_type): + if six.PY2: seg_name = seg_name.encode("utf-8") # We deliberately omit the etag and size here; @@ -227,16 +245,18 @@ class GetContext(WSGIContext): "Got status %d listing container /%s/%s" % (error_response.status_int, account, container)) - def get_or_head_response(self, req, x_object_manifest, - response_headers=None): - if response_headers is None: - response_headers = self._response_headers + def get_or_head_response(self, req, x_object_manifest): + ''' + :param req: user's request + :param x_object_manifest: as unquoted, native string + ''' + response_headers = self._response_headers container, obj_prefix = x_object_manifest.split('/', 1) - container = unquote(container) - obj_prefix = unquote(obj_prefix) version, account, _junk = req.split_path(2, 3, True) + version = wsgi_to_str(version) + account = wsgi_to_str(account) error_response, segments = self._get_container_listing( req, version, account, container, obj_prefix) if error_response: @@ -311,7 +331,7 @@ class GetContext(WSGIContext): if h.lower() != "etag"] etag = md5() for seg_dict in segments: - etag.update(seg_dict['hash'].strip('"')) + etag.update(seg_dict['hash'].strip('"').encode('utf8')) response_headers.append(('Etag', '"%s"' % etag.hexdigest())) app_iter = None @@ -353,7 +373,8 @@ class GetContext(WSGIContext): for header, value in self._response_headers: if (header.lower() == 'x-object-manifest'): close_if_possible(resp_iter) - response = self.get_or_head_response(req, value) + response = self.get_or_head_response( + req, wsgi_to_str(wsgi_unquote(value))) return response(req.environ, start_response) # Not a dynamic large object manifest; just pass it through. start_response(self._response_status, diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index ffb2f88442..65051ebcdf 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -90,6 +90,8 @@ def _header_strip(value): # behave as though it wasn't provided return None return stripped + + _header_strip.re = re.compile('^[\x00-\x20]*|[\x00-\x20]*$') @@ -1427,8 +1429,10 @@ class S3Request(swob.Request): else: # otherwise we do naive HEAD request with the authentication resp = self.get_response(app, 'HEAD', self.container_name, '') + headers = resp.sw_headers.copy() + headers.update(resp.sysmeta_headers) return headers_to_container_info( - resp.sw_headers, resp.status_int) # pylint: disable-msg=E1101 + headers, resp.status_int) # pylint: disable-msg=E1101 def gen_multipart_manifest_delete_query(self, app, obj=None): if not self.allow_multipart_uploads: diff --git a/swift/common/middleware/s3api/s3response.py b/swift/common/middleware/s3api/s3response.py index 38bde84242..9e6a759bd7 100644 --- a/swift/common/middleware/s3api/s3response.py +++ b/swift/common/middleware/s3api/s3response.py @@ -80,7 +80,7 @@ class S3Response(S3ResponseBase, swob.Response): def __init__(self, *args, **kwargs): swob.Response.__init__(self, *args, **kwargs) - sw_sysmeta_headers = swob.HeaderKeyDict() + s3_sysmeta_headers = swob.HeaderKeyDict() sw_headers = swob.HeaderKeyDict() headers = HeaderKeyDict() self.is_slo = False @@ -103,12 +103,14 @@ class S3Response(S3ResponseBase, swob.Response): key = sysmeta_prefix(_server_type) + \ key[len('x-%s-sysmeta-swift3-' % _server_type):] - if key not in sw_sysmeta_headers: + if key not in s3_sysmeta_headers: # To avoid overwrite s3api sysmeta by older swift3 # sysmeta set the key only when the key does not exist - sw_sysmeta_headers[key] = val + s3_sysmeta_headers[key] = val elif is_s3api_sysmeta(key, _server_type): - sw_sysmeta_headers[key] = val + s3_sysmeta_headers[key] = val + else: + sw_headers[key] = val else: sw_headers[key] = val @@ -132,7 +134,7 @@ class S3Response(S3ResponseBase, swob.Response): self.is_slo = config_true_value(val) # Check whether we stored the AWS-style etag on upload - override_etag = sw_sysmeta_headers.get( + override_etag = s3_sysmeta_headers.get( sysmeta_header('object', 'etag')) if override_etag is not None: # Multipart uploads in AWS have ETags like @@ -153,7 +155,7 @@ class S3Response(S3ResponseBase, swob.Response): # Used for pure swift header handling at the request layer self.sw_headers = sw_headers - self.sysmeta_headers = sw_sysmeta_headers + self.sysmeta_headers = s3_sysmeta_headers @classmethod def from_swift_resp(cls, sw_resp): diff --git a/swift/common/middleware/s3api/s3token.py b/swift/common/middleware/s3api/s3token.py index dd4ecfe526..503f276922 100644 --- a/swift/common/middleware/s3api/s3token.py +++ b/swift/common/middleware/s3api/s3token.py @@ -67,7 +67,7 @@ from six.moves import urllib from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \ HTTPException from swift.common.utils import config_true_value, split_path, get_logger, \ - cache_from_env + cache_from_env, append_underscore from swift.common.wsgi import ConfigFileError @@ -149,7 +149,8 @@ class S3Token(object): self._timeout = float(conf.get('http_timeout', '10.0')) if not (0 < self._timeout <= 60): raise ValueError('http_timeout must be between 0 and 60 seconds') - self._reseller_prefix = conf.get('reseller_prefix', 'AUTH_') + self._reseller_prefix = append_underscore( + conf.get('reseller_prefix', 'AUTH')) self._delay_auth_decision = config_true_value( conf.get('delay_auth_decision')) @@ -274,7 +275,9 @@ class S3Token(object): string_to_sign = s3_auth_details['string_to_sign'] if isinstance(string_to_sign, six.text_type): string_to_sign = string_to_sign.encode('utf-8') - token = base64.urlsafe_b64encode(string_to_sign).encode('ascii') + token = base64.urlsafe_b64encode(string_to_sign) + if isinstance(token, six.binary_type): + token = token.decode('ascii') # NOTE(chmou): This is to handle the special case with nova # when we have the option s3_affix_tenant. We will force it to diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index d01c753b34..43815c820f 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -125,14 +125,17 @@ Example usage of this middleware via ``swift``: import cgi import json +import six import time +from six.moves.urllib.parse import urlparse + from swift.common.utils import human_readable, split_path, config_true_value, \ - quote, register_swift_info, get_logger, urlparse + quote, register_swift_info, get_logger from swift.common.wsgi import make_env, WSGIContext from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound, \ - Request + Request, wsgi_quote, wsgi_to_str from swift.proxy.controllers.base import get_container_info @@ -145,6 +148,12 @@ class _StaticWebContext(WSGIContext): that might need to be handled to make keeping contextual information about the request a bit simpler than storing it in the WSGI env. + + :param staticweb: The staticweb middleware object in use. + :param version: A WSGI string representation of the swift api version. + :param account: A WSGI string representation of the account name. + :param container: A WSGI string representation of the container name. + :param obj: A WSGI string representation of the object name. """ def __init__(self, staticweb, version, account, container, obj): @@ -223,9 +232,9 @@ class _StaticWebContext(WSGIContext): :param start_response: The original WSGI start_response hook. :param prefix: Any prefix desired for the container listing. """ - label = env['PATH_INFO'] + label = wsgi_to_str(env['PATH_INFO']) if self._listings_label: - groups = env['PATH_INFO'].split('/') + groups = wsgi_to_str(env['PATH_INFO']).split('/') label = '{0}/{1}'.format(self._listings_label, '/'.join(groups[4:])) @@ -262,14 +271,14 @@ class _StaticWebContext(WSGIContext): self.agent, swift_source='SW') tmp_env['QUERY_STRING'] = 'delimiter=/' if prefix: - tmp_env['QUERY_STRING'] += '&prefix=%s' % quote(prefix) + tmp_env['QUERY_STRING'] += '&prefix=%s' % wsgi_quote(prefix) else: prefix = '' resp = self._app_call(tmp_env) if not is_success(self._get_status_int()): return self._error_response(resp, env, start_response) listing = None - body = ''.join(resp) + body = b''.join(resp) if body: listing = json.loads(body) if not listing: @@ -280,7 +289,8 @@ class _StaticWebContext(WSGIContext): 'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \ '\n' \ ' \n' \ - ' Listing of %s\n' % cgi.escape(label) + ' Listing of %s\n' % \ + cgi.escape(label) if self._listings_css: body += ' \n' % (self._build_css_path(prefix)) @@ -308,7 +318,8 @@ class _StaticWebContext(WSGIContext): ' \n' for item in listing: if 'subdir' in item: - subdir = item['subdir'].encode("utf-8") + subdir = item['subdir'] if six.PY3 else \ + item['subdir'].encode('utf-8') if prefix: subdir = subdir[len(prefix):] body += ' \n' \ @@ -319,13 +330,16 @@ class _StaticWebContext(WSGIContext): (quote(subdir), cgi.escape(subdir)) for item in listing: if 'name' in item: - name = item['name'].encode("utf-8") + name = item['name'] if six.PY3 else \ + item['name'].encode('utf-8') if prefix: name = name[len(prefix):] - content_type = item['content_type'].encode("utf-8") + content_type = item['content_type'] if six.PY3 else \ + item['content_type'].encode('utf-8') bytes = human_readable(item['bytes']) last_modified = ( - cgi.escape(item['last_modified'].encode("utf-8")). + cgi.escape(item['last_modified'] if six.PY3 else + item['last_modified'].encode('utf-8')). split('.')[0].replace('T', ' ')) body += ' \n' \ ' %s\n' \ @@ -362,7 +376,8 @@ class _StaticWebContext(WSGIContext): env['wsgi.url_scheme'] = self.url_scheme if self.url_host: env['HTTP_HOST'] = self.url_host - resp = HTTPMovedPermanently(location=(env['PATH_INFO'] + '/')) + resp = HTTPMovedPermanently( + location=wsgi_quote(env['PATH_INFO'] + '/')) return resp(env, start_response) def handle_container(self, env, start_response): @@ -466,9 +481,9 @@ class _StaticWebContext(WSGIContext): self.version, self.account, self.container), self.agent, swift_source='SW') tmp_env['QUERY_STRING'] = 'limit=1&delimiter=/&prefix=%s' % ( - quote(self.obj + '/'), ) + quote(wsgi_to_str(self.obj) + '/'), ) resp = self._app_call(tmp_env) - body = ''.join(resp) + body = b''.join(resp) if not is_success(self._get_status_int()) or not body or \ not json.loads(body): resp = HTTPNotFound()(env, self._start_response) diff --git a/swift/common/middleware/symlink.py b/swift/common/middleware/symlink.py index 94e6c8edfe..a0bb56de39 100644 --- a/swift/common/middleware/symlink.py +++ b/swift/common/middleware/symlink.py @@ -161,7 +161,7 @@ from cgi import parse_header from six.moves.urllib.parse import unquote from swift.common.utils import get_logger, register_swift_info, split_path, \ - MD5_OF_EMPTY_STRING, closing_if_possible + MD5_OF_EMPTY_STRING, closing_if_possible, quote from swift.common.constraints import check_account_format from swift.common.wsgi import WSGIContext, make_subrequest from swift.common.request_helpers import get_sys_meta_prefix, \ @@ -208,6 +208,7 @@ def _check_symlink_header(req): req, TGT_OBJ_SYMLINK_HDR, 2, 'X-Symlink-Target header must be of the ' 'form /') + req.headers[TGT_OBJ_SYMLINK_HDR] = quote('%s/%s' % (container, obj)) # Check account format if it exists account = check_account_format( @@ -217,7 +218,9 @@ def _check_symlink_header(req): # Extract request path _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True) - if not account: + if account: + req.headers[TGT_ACCT_SYMLINK_HDR] = quote(account) + else: account = req_acc # Check if symlink targets the symlink itself or not @@ -378,9 +381,9 @@ class SymlinkObjectContext(WSGIContext): :returns: new request for target path if it's symlink otherwise None """ - version, account, _junk = split_path(req.path, 2, 3, True) + version, account, _junk = req.split_path(2, 3, True) account = self._response_header_value( - TGT_ACCT_SYSMETA_SYMLINK_HDR) or account + TGT_ACCT_SYSMETA_SYMLINK_HDR) or quote(account) target_path = os.path.join( '/', version, account, symlink_target.lstrip('/')) @@ -485,7 +488,7 @@ class SymlinkObjectContext(WSGIContext): if tgt_co: version, account, _junk = req.split_path(2, 3, True) target_acc = self._response_header_value( - TGT_ACCT_SYSMETA_SYMLINK_HDR) or account + TGT_ACCT_SYSMETA_SYMLINK_HDR) or quote(account) location_hdr = os.path.join( '/', version, target_acc, tgt_co) req.environ['swift.leave_relative_location'] = True diff --git a/swift/common/utils.py b/swift/common/utils.py index 2413eecd15..9431631c9f 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -75,9 +75,8 @@ from six.moves import cPickle as pickle from six.moves.configparser import (ConfigParser, NoSectionError, NoOptionError, RawConfigParser) from six.moves import range, http_client -from six.moves.urllib.parse import ParseResult from six.moves.urllib.parse import quote as _quote -from six.moves.urllib.parse import urlparse as stdlib_urlparse +from six.moves.urllib.parse import urlparse from swift import gettext_ as _ import swift.common.exceptions @@ -2265,7 +2264,7 @@ def get_hub(): Note about epoll: - Review: https://review.openstack.org/#/c/18806/ + Review: https://review.opendev.org/#/c/18806/ There was a problem where once out of every 30 quadrillion connections, a coroutine wouldn't wake up when the client @@ -3244,38 +3243,6 @@ class StreamingPile(GreenAsyncPile): self.pool.__exit__(type, value, traceback) -class ModifiedParseResult(ParseResult): - """Parse results class for urlparse.""" - - @property - def hostname(self): - netloc = self.netloc.split('@', 1)[-1] - if netloc.startswith('['): - return netloc[1:].split(']')[0] - elif ':' in netloc: - return netloc.rsplit(':')[0] - return netloc - - @property - def port(self): - netloc = self.netloc.split('@', 1)[-1] - if netloc.startswith('['): - netloc = netloc.rsplit(']')[1] - if ':' in netloc: - return int(netloc.rsplit(':')[1]) - return None - - -def urlparse(url): - """ - urlparse augmentation. - This is necessary because urlparse can't handle RFC 2732 URLs. - - :param url: URL to parse. - """ - return ModifiedParseResult(*stdlib_urlparse(url)) - - def validate_sync_to(value, allowed_sync_hosts, realms_conf): """ Validates an X-Container-Sync-To header value, returning the diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 7cce7f98b4..6f0f600f54 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -430,6 +430,7 @@ class SwiftHttpProtocol(wsgi.HttpProtocol): # for py3: def get_default_type(self): + '''If the client didn't provide a content type, leave it blank.''' return '' diff --git a/swift/container/sharder.py b/swift/container/sharder.py index d5f125968a..c87912e10f 100644 --- a/swift/container/sharder.py +++ b/swift/container/sharder.py @@ -365,8 +365,9 @@ class ContainerSharder(ContainerReplicator): 'Swift Container Sharder', request_tries, allow_modify_pipeline=False) - except IOError as err: - if err.errno != errno.ENOENT: + except (OSError, IOError) as err: + if err.errno != errno.ENOENT and \ + not str(err).endswith(' not found'): raise raise SystemExit( 'Unable to load internal client from config: %r (%s)' % diff --git a/swift/container/sync.py b/swift/container/sync.py index 0f96c496a1..a17c28612c 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -23,6 +23,7 @@ from random import choice, random from struct import unpack_from from eventlet import sleep, Timeout +from six.moves.urllib.parse import urlparse import swift.common.db from swift.common.db import DatabaseConnectionError @@ -37,7 +38,7 @@ from swift.common.ring import Ring from swift.common.ring.utils import is_local_device from swift.common.utils import ( clean_content_type, config_true_value, - FileLikeIter, get_logger, hash_path, quote, urlparse, validate_sync_to, + FileLikeIter, get_logger, hash_path, quote, validate_sync_to, whataremyips, Timestamp, decode_timestamps) from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND @@ -238,8 +239,9 @@ class ContainerSync(Daemon): try: self.swift = InternalClient( internal_client_conf, 'Swift Container Sync', request_tries) - except IOError as err: - if err.errno != errno.ENOENT: + except (OSError, IOError) as err: + if err.errno != errno.ENOENT and \ + not str(err).endswith(' not found'): raise raise SystemExit( _('Unable to load internal client from config: ' diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index d34dd7ae48..b62611a250 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -818,9 +818,10 @@ class ObjectReplicator(Daemon): except Exception: self.logger.exception('ERROR creating %s' % obj_path) continue + for partition in df_mgr.listdir(obj_path): - if (override_partitions is not None - and partition not in override_partitions): + if (override_partitions is not None and partition.isdigit() + and int(partition) not in override_partitions): continue if (partition.startswith('auditor_status_') and diff --git a/test/functional/s3api/s3_test_client.py b/test/functional/s3api/s3_test_client.py index f5f79ce8b5..8ceccbff4b 100644 --- a/test/functional/s3api/s3_test_client.py +++ b/test/functional/s3api/s3_test_client.py @@ -16,8 +16,10 @@ import os import test.functional as tf from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \ - BotoClientError, S3ResponseError + S3ResponseError import six +import sys +import traceback RETRY_COUNT = 3 @@ -92,11 +94,12 @@ class Connection(object): # 404 means NoSuchBucket, NoSuchKey, or NoSuchUpload if e.status != 404: raise - except (BotoClientError, S3ResponseError) as e: - exceptions.append(e) + except Exception as e: + exceptions.append(''.join( + traceback.format_exception(*sys.exc_info()))) if exceptions: - # raise the first exception - raise exceptions.pop(0) + exceptions.insert(0, 'Too many errors to continue:') + raise Exception('\n========\n'.join(exceptions)) def make_request(self, method, bucket='', obj='', headers=None, body='', query=None): diff --git a/test/functional/test_staticweb.py b/test/functional/test_staticweb.py index ee1639806c..20ff080c99 100644 --- a/test/functional/test_staticweb.py +++ b/test/functional/test_staticweb.py @@ -16,6 +16,8 @@ import functools from unittest2 import SkipTest +from six.moves.urllib.parse import unquote +from swift.common.utils import quote import test.functional as tf from test.functional import cluster_info from test.functional.tests import Utils, Base, BaseEnv @@ -74,8 +76,8 @@ class TestStaticWebEnv(BaseEnv): 'listings_css', 'dir/', 'dir/obj', - 'dir/subdir/', - 'dir/subdir/obj'] + 'dir/some sub%dir/', + 'dir/some sub%dir/obj'] cls.objects = {} for item in sorted(objects): @@ -168,12 +170,12 @@ class TestStaticWeb(Base): def _test_redirect_slash_direct(self, anonymous): host = self.env.account.conn.storage_netloc path = '%s/%s' % (self.env.account.conn.storage_path, - self.env.container.name) + quote(self.env.container.name)) self._test_redirect_with_slash(host, path, anonymous=anonymous) path = '%s/%s/%s' % (self.env.account.conn.storage_path, - self.env.container.name, - self.env.objects['dir/'].name) + quote(self.env.container.name), + quote(self.env.objects['dir/'].name)) self._test_redirect_with_slash(host, path, anonymous=anonymous) def test_redirect_slash_auth_direct(self): @@ -185,10 +187,10 @@ class TestStaticWeb(Base): @requires_domain_remap def _test_redirect_slash_remap_acct(self, anonymous): host = self.domain_remap_acct - path = '/%s' % self.env.container.name + path = '/%s' % quote(self.env.container.name) self._test_redirect_with_slash(host, path, anonymous=anonymous) - path = '/%s/%s' % (self.env.container.name, + path = '/%s/%s' % (quote(self.env.container.name), self.env.objects['dir/'].name) self._test_redirect_with_slash(host, path, anonymous=anonymous) @@ -229,13 +231,14 @@ class TestStaticWeb(Base): self._set_staticweb_headers(listings=True, listings_css=(css is not None)) if title is None: - title = path + title = unquote(path) expected_in = ['Listing of %s' % title] + [ - '{0}'.format(link) for link in links] + '{1}'.format(quote(link), link) + for link in links] expected_not_in = notins if css: expected_in.append('' % css) + 'href="%s" />' % quote(css)) self._test_get_path(host, path, anonymous=anonymous, expected_in=expected_in, expected_not_in=expected_not_in) @@ -244,7 +247,7 @@ class TestStaticWeb(Base): objects = self.env.objects host = self.env.account.conn.storage_netloc path = '%s/%s/' % (self.env.account.conn.storage_path, - self.env.container.name) + quote(self.env.container.name)) css = objects['listings_css'].name if listings_css else None self._test_listing(host, path, anonymous=True, css=css, links=[objects['index'].name, @@ -252,15 +255,15 @@ class TestStaticWeb(Base): notins=[objects['dir/obj'].name]) path = '%s/%s/%s/' % (self.env.account.conn.storage_path, - self.env.container.name, - objects['dir/'].name) + quote(self.env.container.name), + quote(objects['dir/'].name)) css = '../%s' % objects['listings_css'].name if listings_css else None - self._test_listing(host, path, anonymous=anonymous, css=css, - links=[objects['dir/obj'].name.split('/')[-1], - objects['dir/subdir/'].name.split('/')[-1] - + '/'], - notins=[objects['index'].name, - objects['dir/subdir/obj'].name]) + self._test_listing( + host, path, anonymous=anonymous, css=css, + links=[objects['dir/obj'].name.split('/')[-1], + objects['dir/some sub%dir/'].name.split('/')[-1] + '/'], + notins=[objects['index'].name, + objects['dir/some sub%dir/obj'].name]) def test_listing_auth_direct_without_css(self): self._test_listing_direct(False, False) @@ -293,13 +296,12 @@ class TestStaticWeb(Base): title = '%s/%s/%s/' % (self.env.account.conn.storage_path, self.env.container.name, objects['dir/']) - self._test_listing(host, path, title=title, anonymous=anonymous, - css=css, - links=[objects['dir/obj'].name.split('/')[-1], - objects['dir/subdir/'].name.split('/')[-1] - + '/'], - notins=[objects['index'].name, - objects['dir/subdir/obj'].name]) + self._test_listing( + host, path, title=title, anonymous=anonymous, css=css, + links=[objects['dir/obj'].name.split('/')[-1], + objects['dir/some sub%dir/'].name.split('/')[-1] + '/'], + notins=[objects['index'].name, + objects['dir/some sub%dir/obj'].name]) def test_listing_auth_remap_acct_without_css(self): self._test_listing_remap_acct(False, False) @@ -332,13 +334,12 @@ class TestStaticWeb(Base): title = '%s/%s/%s/' % (self.env.account.conn.storage_path, self.env.container.name, objects['dir/']) - self._test_listing(host, path, title=title, anonymous=anonymous, - css=css, - links=[objects['dir/obj'].name.split('/')[-1], - objects['dir/subdir/'].name.split('/')[-1] - + '/'], - notins=[objects['index'].name, - objects['dir/subdir/obj'].name]) + self._test_listing( + host, path, title=title, anonymous=anonymous, css=css, + links=[objects['dir/obj'].name.split('/')[-1], + objects['dir/some sub%dir/'].name.split('/')[-1] + '/'], + notins=[objects['index'].name, + objects['dir/some sub%dir/obj'].name]) def test_listing_auth_remap_cont_without_css(self): self._test_listing_remap_cont(False, False) @@ -369,12 +370,12 @@ class TestStaticWeb(Base): objects = self.env.objects host = self.env.account.conn.storage_netloc path = '%s/%s/' % (self.env.account.conn.storage_path, - self.env.container.name) + quote(self.env.container.name)) self._test_index(host, path, anonymous=anonymous) path = '%s/%s/%s/' % (self.env.account.conn.storage_path, - self.env.container.name, - objects['dir/'].name) + quote(self.env.container.name), + quote(objects['dir/'].name)) self._test_index(host, path, anonymous=anonymous, expected_status=404) def test_index_auth_direct(self): diff --git a/test/functional/test_symlink.py b/test/functional/test_symlink.py index cfc0031b2c..721b449741 100755 --- a/test/functional/test_symlink.py +++ b/test/functional/test_symlink.py @@ -270,23 +270,45 @@ class TestSymlink(Base): target_obj = 'dealde%2Fl04 011e%204c8df/flash.png' link_obj = uuid4().hex - # Now let's write a new target object and symlink will be able to - # return object + # create target using unnormalized path resp = retry( self._make_request, method='PUT', container=self.env.tgt_cont, obj=target_obj, body=TARGET_BODY) - self.assertEqual(resp.status, 201) + # you can get it using either name + resp = retry( + self._make_request, method='GET', container=self.env.tgt_cont, + obj=target_obj) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, TARGET_BODY) + normalized_quoted_obj = 'dealde/l04%20011e%204c8df/flash.png' + self.assertEqual(normalized_quoted_obj, urllib.parse.quote( + urllib.parse.unquote(target_obj))) + resp = retry( + self._make_request, method='GET', container=self.env.tgt_cont, + obj=normalized_quoted_obj) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.content, TARGET_BODY) - # PUT symlink + # create a symlink using the un-normalized target path self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, tgt_cont=self.env.tgt_cont, tgt_obj=target_obj) - + # and it's normalized self._assertSymlink( self.env.link_cont, link_obj, - expected_content_location="%s/%s" % (self.env.tgt_cont, - target_obj)) + expected_content_location='%s/%s' % ( + self.env.tgt_cont, normalized_quoted_obj)) + + # create a symlink using the normalized target path + self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj, + tgt_cont=self.env.tgt_cont, + tgt_obj=normalized_quoted_obj) + # and it's ALSO normalized + self._assertSymlink( + self.env.link_cont, link_obj, + expected_content_location='%s/%s' % ( + self.env.tgt_cont, normalized_quoted_obj)) def test_symlink_put_head_get(self): link_obj = uuid4().hex diff --git a/test/functional/test_versioned_writes.py b/test/functional/test_versioned_writes.py index 205ba06633..1ae0058401 100644 --- a/test/functional/test_versioned_writes.py +++ b/test/functional/test_versioned_writes.py @@ -18,7 +18,7 @@ from copy import deepcopy import json import time import unittest2 -from six.moves.urllib.parse import quote +from six.moves.urllib.parse import quote, unquote import test.functional as tf @@ -652,7 +652,7 @@ class TestObjectVersioning(Base): tgt_b.write("bbbbb") symlink_name = Utils.create_name() - sym_tgt_header = '%s/%s' % (container.name, tgt_a_name) + sym_tgt_header = quote(unquote('%s/%s' % (container.name, tgt_a_name))) sym_headers_a = {'X-Symlink-Target': sym_tgt_header} symlink = container.file(symlink_name) symlink.write("", hdrs=sym_headers_a) @@ -684,8 +684,9 @@ class TestObjectVersioning(Base): sym_info = symlink.info(parms={'symlink': 'get'}) self.assertEqual("aaaaa", symlink.read()) self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag']) - self.assertEqual('%s/%s' % (self.env.container.name, target.name), - sym_info['x_symlink_target']) + self.assertEqual( + quote(unquote('%s/%s' % (self.env.container.name, target.name))), + sym_info['x_symlink_target']) def _setup_symlink(self): target = self.env.container.file('target-object') diff --git a/test/unit/common/middleware/s3api/test_s3request.py b/test/unit/common/middleware/s3api/test_s3request.py index 64b72d1627..3cec613586 100644 --- a/test/unit/common/middleware/s3api/test_s3request.py +++ b/test/unit/common/middleware/s3api/test_s3request.py @@ -302,9 +302,14 @@ class TestRequest(S3ApiTestCase): self.assertTrue(sw_req.environ['swift.proxy_access_log_made']) def test_get_container_info(self): + s3api_acl = '{"Owner":"owner","Grant":'\ + '[{"Grantee":"owner","Permission":"FULL_CONTROL"}]}' self.swift.register('HEAD', '/v1/AUTH_test/bucket', HTTPNoContent, {'x-container-read': 'foo', 'X-container-object-count': 5, + 'x-container-sysmeta-versions-location': + 'bucket2', + 'x-container-sysmeta-s3api-acl': s3api_acl, 'X-container-meta-foo': 'bar'}, None) req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', @@ -316,6 +321,9 @@ class TestRequest(S3ApiTestCase): self.assertEqual(204, info['status']) # sanity self.assertEqual('foo', info['read_acl']) # sanity self.assertEqual('5', info['object_count']) # sanity + self.assertEqual( + 'bucket2', info['sysmeta']['versions-location']) # sanity + self.assertEqual(s3api_acl, info['sysmeta']['s3api-acl']) # sanity self.assertEqual({'foo': 'bar'}, info['meta']) # sanity with patch( 'swift.common.middleware.s3api.s3request.get_container_info', diff --git a/test/unit/common/middleware/s3api/test_s3response.py b/test/unit/common/middleware/s3api/test_s3response.py index 9ffdaf1db1..4c6a854b18 100644 --- a/test/unit/common/middleware/s3api/test_s3response.py +++ b/test/unit/common/middleware/s3api/test_s3response.py @@ -54,6 +54,8 @@ class TestResponse(unittest.TestCase): expected_headers = HeaderKeyDict( {sysmeta_prefix(_server_type) + 'test': 'ok'}) self.assertEqual(expected_headers, s3resp.sysmeta_headers) + self.assertIn('x-%s-sysmeta-test-s3api' % _server_type, + s3resp.sw_headers) def test_response_s3api_sysmeta_from_swift3_sysmeta(self): for _server_type in ('object', 'container'): diff --git a/test/unit/common/middleware/s3api/test_s3token.py b/test/unit/common/middleware/s3api/test_s3token.py index e84099d83d..aaf15cba6e 100644 --- a/test/unit/common/middleware/s3api/test_s3token.py +++ b/test/unit/common/middleware/s3api/test_s3token.py @@ -372,6 +372,15 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): middleware = s3token.filter_factory(config)(self.app) self.assertIs('false_ind', middleware._verify) + def test_reseller_prefix(self): + def do_test(conf, expected): + conf.update(self.conf) + middleware = s3token.filter_factory(conf)(self.app) + self.assertEqual(expected, middleware._reseller_prefix) + do_test({}, 'AUTH_') + do_test({'reseller_prefix': 'KEY_'}, 'KEY_') + do_test({'reseller_prefix': 'KEY'}, 'KEY_') + def test_auth_uris(self): for conf, expected in [ ({'auth_uri': 'https://example.com/v2.0'}, @@ -456,12 +465,12 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): with self.assertRaises(ConfigFileError) as cm: s3token.filter_factory({'auth_uri': auth_uri})(self.app) self.assertEqual('Invalid auth_uri; must include scheme and host', - cm.exception.message) + cm.exception.args[0]) with self.assertRaises(ConfigFileError) as cm: s3token.filter_factory({ 'auth_uri': 'nonhttp://example.com'})(self.app) self.assertEqual('Invalid auth_uri; scheme must be http or https', - cm.exception.message) + cm.exception.args[0]) for auth_uri in [ 'http://user@example.com/', 'http://example.com/?with=query', @@ -469,7 +478,7 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): with self.assertRaises(ConfigFileError) as cm: s3token.filter_factory({'auth_uri': auth_uri})(self.app) self.assertEqual('Invalid auth_uri; must not include username, ' - 'query, or fragment', cm.exception.message) + 'query, or fragment', cm.exception.args[0]) def test_unicode_path(self): url = u'/v1/AUTH_cfa/c/euro\u20ac'.encode('utf8') @@ -568,7 +577,7 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): MOCK_REQUEST.return_value = TestResponse({ 'status_code': 201, - 'text': json.dumps(GOOD_RESPONSE_V2)}) + 'text': json.dumps(GOOD_RESPONSE_V2).encode('ascii')}) req = Request.blank('/v1/AUTH_cfa/c/o') req.environ['s3api.auth_details'] = { diff --git a/test/unit/common/middleware/test_dlo.py b/test/unit/common/middleware/test_dlo.py index 8ac99827d9..4f4636900a 100644 --- a/test/unit/common/middleware/test_dlo.py +++ b/test/unit/common/middleware/test_dlo.py @@ -34,6 +34,8 @@ LIMIT = 'swift.common.constraints.CONTAINER_LISTING_LIMIT' def md5hex(s): + if not isinstance(s, bytes): + s = s.encode('utf-8') return hashlib.md5(s).hexdigest() @@ -52,7 +54,7 @@ class DloTestCase(unittest.TestCase): headers[0] = h body_iter = app(req.environ, start_response) - body = '' + body = b'' # appease the close-checker with closing_if_possible(body_iter): for chunk in body_iter: @@ -69,36 +71,36 @@ class DloTestCase(unittest.TestCase): self.app.register( 'GET', '/v1/AUTH_test/c/seg_01', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")}, - 'aaaaa') + b'aaaaa') self.app.register( 'GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")}, - 'bbbbb') + b'bbbbb') self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")}, - 'ccccc') + b'ccccc') self.app.register( 'GET', '/v1/AUTH_test/c/seg_04', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")}, - 'ddddd') + b'ddddd') self.app.register( 'GET', '/v1/AUTH_test/c/seg_05', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")}, - 'eeeee') + b'eeeee') # an unrelated object (not seg*) to test the prefix matching self.app.register( 'GET', '/v1/AUTH_test/c/catpicture.jpg', swob.HTTPOk, {'Content-Length': '9', 'Etag': md5hex("meow meow meow meow")}, - 'meow meow meow meow') + b'meow meow meow meow') self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest', swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag', 'X-Object-Manifest': 'c/seg'}, - 'manifest-contents') + b'manifest-contents') lm = '2013-11-22T02:42:13.781760' ct = 'application/octet-stream' @@ -120,11 +122,11 @@ class DloTestCase(unittest.TestCase): self.app.register( 'GET', '/v1/AUTH_test/c', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, - json.dumps(full_container_listing)) + json.dumps(full_container_listing).encode('ascii')) self.app.register( 'GET', '/v1/AUTH_test/c?prefix=seg', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, - json.dumps(segs)) + json.dumps(segs).encode('ascii')) # This is to let us test multi-page container listings; we use the # trailing underscore to send small (pagesize=3) listings. @@ -135,26 +137,26 @@ class DloTestCase(unittest.TestCase): 'GET', '/v1/AUTH_test/mancon/manifest-many-segments', swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg', 'X-Object-Manifest': 'c/seg_'}, - 'manyseg') + b'manyseg') self.app.register( 'GET', '/v1/AUTH_test/c?prefix=seg_', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, - json.dumps(segs[:3])) + json.dumps(segs[:3]).encode('ascii')) self.app.register( 'GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, - json.dumps(segs[3:])) + json.dumps(segs[3:]).encode('ascii')) # Here's a manifest with 0 segments self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest-no-segments', swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg', 'X-Object-Manifest': 'c/noseg_'}, - 'noseg') + b'noseg') self.app.register( 'GET', '/v1/AUTH_test/c?prefix=noseg_', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, - json.dumps([])) + json.dumps([]).encode('ascii')) class TestDloPutManifest(DloTestCase): @@ -284,7 +286,7 @@ class TestDloGetManifest(DloTestCase): headers = HeaderKeyDict(headers) self.assertEqual(headers["Etag"], expected_etag) self.assertEqual(headers["Content-Length"], "25") - self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') + self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee') for _, _, hdrs in self.app.calls_with_headers[1:]: ua = hdrs.get("User-Agent", "") @@ -302,7 +304,7 @@ class TestDloGetManifest(DloTestCase): req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) - self.assertEqual(body, "meow meow meow meow") + self.assertEqual(body, b"meow meow meow meow") def test_get_non_object_passthrough(self): self.app.register('GET', '/info', swob.HTTPOk, @@ -311,7 +313,7 @@ class TestDloGetManifest(DloTestCase): environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) self.assertEqual(status, '200 OK') - self.assertEqual(body, 'useful stuff here') + self.assertEqual(body, b'useful stuff here') self.assertEqual(self.app.call_count, 1) def test_get_manifest_passthrough(self): @@ -328,7 +330,7 @@ class TestDloGetManifest(DloTestCase): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(headers["Etag"], "manifest-etag") - self.assertEqual(body, "manifest-contents") + self.assertEqual(body, b'manifest-contents') def test_error_passthrough(self): self.app.register( @@ -347,7 +349,7 @@ class TestDloGetManifest(DloTestCase): headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "10") - self.assertEqual(body, "bbcccccddd") + self.assertEqual(body, b'bbcccccddd') expected_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) @@ -361,7 +363,7 @@ class TestDloGetManifest(DloTestCase): headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "10") - self.assertEqual(body, "cccccddddd") + self.assertEqual(body, b'cccccddddd') def test_get_range_first_byte(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', @@ -371,7 +373,7 @@ class TestDloGetManifest(DloTestCase): headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "1") - self.assertEqual(body, "a") + self.assertEqual(body, b'a') def test_get_range_last_byte(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', @@ -381,7 +383,7 @@ class TestDloGetManifest(DloTestCase): headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "1") - self.assertEqual(body, "e") + self.assertEqual(body, b'e') def test_get_range_overlapping_end(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', @@ -392,7 +394,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "7") self.assertEqual(headers["Content-Range"], "bytes 18-24/25") - self.assertEqual(body, "ddeeeee") + self.assertEqual(body, b'ddeeeee') def test_get_range_unsatisfiable(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', @@ -428,7 +430,7 @@ class TestDloGetManifest(DloTestCase): # # Since the truth is forbidden, we lie. self.assertEqual(headers["Content-Range"], "bytes 3-12/15") - self.assertEqual(body, "aabbbbbccc") + self.assertEqual(body, b"aabbbbbccc") self.assertEqual( self.app.calls, @@ -449,7 +451,7 @@ class TestDloGetManifest(DloTestCase): # this requires multiple pages of container listing, so we can't send # a Content-Length header self.assertIsNone(headers.get("Content-Length")) - self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") + self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee") def test_get_suffix_range(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', @@ -459,7 +461,7 @@ class TestDloGetManifest(DloTestCase): headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "25") - self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") + self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee") def test_get_suffix_range_many_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', @@ -471,7 +473,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, "200 OK") self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Range")) - self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") + self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee") def test_get_multi_range(self): # DLO doesn't support multi-range GETs. The way that you express that @@ -485,7 +487,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, "200 OK") self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Range")) - self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") + self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee') def test_if_match_matches(self): manifest_etag = '"%s"' % md5hex( @@ -500,7 +502,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '25') - self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') + self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee') def test_if_match_does_not_match(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', @@ -512,7 +514,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, '412 Precondition Failed') self.assertEqual(headers['Content-Length'], '0') - self.assertEqual(body, '') + self.assertEqual(body, b'') def test_if_none_match_matches(self): manifest_etag = '"%s"' % md5hex( @@ -527,7 +529,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, '304 Not Modified') self.assertEqual(headers['Content-Length'], '0') - self.assertEqual(body, '') + self.assertEqual(body, b'') def test_if_none_match_does_not_match(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', @@ -539,7 +541,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '25') - self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') + self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee') def test_get_with_if_modified_since(self): # It's important not to pass the If-[Un]Modified-Since header to the @@ -581,7 +583,8 @@ class TestDloGetManifest(DloTestCase): headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") - self.assertEqual(''.join(body), "aaaaa") # first segment made it out + # first segment made it out + self.assertEqual(body, b'aaaaa') self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [ 'While processing manifest /v1/AUTH_test/mancon/manifest, ' 'got 403 while retrieving /v1/AUTH_test/c/seg_02', @@ -610,7 +613,7 @@ class TestDloGetManifest(DloTestCase): with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) self.assertEqual(status, "200 OK") - self.assertEqual(body, "aaaaabbbbbccccc") + self.assertEqual(body, b'aaaaabbbbbccccc') def test_error_listing_container_HEAD(self): self.app.register( @@ -639,7 +642,8 @@ class TestDloGetManifest(DloTestCase): headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") - self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error + # stop after error + self.assertEqual(body, b"aaaaabbWRONGbb") def test_etag_comparison_ignores_quotes(self): # a little future-proofing here in case we ever fix this in swob @@ -662,7 +666,7 @@ class TestDloGetManifest(DloTestCase): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(headers["Etag"], - '"' + hashlib.md5("abcdef").hexdigest() + '"') + '"' + hashlib.md5(b"abcdef").hexdigest() + '"') def test_object_prefix_quoting(self): self.app.register( @@ -675,22 +679,24 @@ class TestDloGetManifest(DloTestCase): self.app.register( 'GET', '/v1/AUTH_test/c?prefix=%C3%A9', swob.HTTPOk, {'Content-Type': 'application/json'}, - json.dumps(segs)) + json.dumps(segs).encode('ascii')) + # NB: wsgi string + path = '/v1/AUTH_test/c/\xC3\xa9' self.app.register( - 'GET', '/v1/AUTH_test/c/\xC3\xa91', + 'GET', path + '1', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")}, - "AAAAA") + b"AAAAA") self.app.register( - 'GET', '/v1/AUTH_test/c/\xC3\xA92', + 'GET', path + '2', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")}, - "BBBBB") + b"BBBBB") req = swob.Request.blank('/v1/AUTH_test/man/accent', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) self.assertEqual(status, "200 OK") - self.assertEqual(body, "AAAAABBBBB") + self.assertEqual(body, b'AAAAABBBBB') def test_get_taking_too_long(self): the_time = [time.time()] @@ -715,7 +721,7 @@ class TestDloGetManifest(DloTestCase): status, headers, body = self.call_dlo(req) self.assertEqual(status, '200 OK') - self.assertEqual(body, 'aaaaabbbbbccccc') + self.assertEqual(body, b'aaaaabbbbbccccc') def test_get_oversize_segment(self): # If we send a Content-Length header to the client, it's based on the @@ -738,7 +744,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, '200 OK') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check - self.assertEqual(body, 'aaaaabbbbbccccccccccccccc') + self.assertEqual(body, b'aaaaabbbbbccccccccccccccc') self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/mancon/manifest'), @@ -770,7 +776,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, '200 OK') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check - self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee') + self.assertEqual(body, b'aaaaabbbbbccccdddddeeeee') def test_get_undersize_segment_range(self): # Shrink it by a single byte @@ -788,7 +794,7 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(status, '206 Partial Content') # sanity check self.assertEqual(headers.get('Content-Length'), '15') # sanity check - self.assertEqual(body, 'aaaaabbbbbcccc') + self.assertEqual(body, b'aaaaabbbbbcccc') def test_get_with_auth_overridden(self): auth_got_called = [0] @@ -840,7 +846,7 @@ class TestDloConfiguration(unittest.TestCase): max_get_time = 2900 """) - conffile = tempfile.NamedTemporaryFile() + conffile = tempfile.NamedTemporaryFile(mode='w') conffile.write(proxy_conf) conffile.flush() @@ -853,6 +859,8 @@ class TestDloConfiguration(unittest.TestCase): self.assertEqual(10, mware.rate_limit_after_segment) self.assertEqual(3600, mware.max_get_time) + conffile.close() + def test_finding_defaults_from_file(self): # If DLO has no config vars, go pull them from the proxy server's # config section @@ -875,7 +883,7 @@ class TestDloConfiguration(unittest.TestCase): set max_get_time = 2900 """) - conffile = tempfile.NamedTemporaryFile() + conffile = tempfile.NamedTemporaryFile(mode='w') conffile.write(proxy_conf) conffile.flush() @@ -887,6 +895,8 @@ class TestDloConfiguration(unittest.TestCase): self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(2900, mware.max_get_time) + conffile.close() + def test_finding_defaults_from_dir(self): # If DLO has no config vars, go pull them from the proxy server's # config section @@ -913,11 +923,13 @@ class TestDloConfiguration(unittest.TestCase): conf_dir = self.tmpdir - conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf') + conffile1 = tempfile.NamedTemporaryFile(mode='w', + dir=conf_dir, suffix='.conf') conffile1.write(proxy_conf1) conffile1.flush() - conffile2 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf') + conffile2 = tempfile.NamedTemporaryFile(mode='w', + dir=conf_dir, suffix='.conf') conffile2.write(proxy_conf2) conffile2.flush() @@ -929,6 +941,9 @@ class TestDloConfiguration(unittest.TestCase): self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(2900, mware.max_get_time) + conffile1.close() + conffile2.close() + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_staticweb.py b/test/unit/common/middleware/test_staticweb.py index ba6d1a705d..b72b34bf29 100644 --- a/test/unit/common/middleware/test_staticweb.py +++ b/test/unit/common/middleware/test_staticweb.py @@ -528,7 +528,7 @@ class TestStaticWeb(unittest.TestCase): def test_container3indexhtml(self): resp = Request.blank('/v1/a/c3/').get_response(self.test_staticweb) self.assertEqual(resp.status_int, 200) - self.assertTrue('Test main index.html file.' in resp.body) + self.assertIn(b'Test main index.html file.', resp.body) def test_container3subsubdir(self): resp = Request.blank( @@ -539,16 +539,16 @@ class TestStaticWeb(unittest.TestCase): resp = Request.blank( '/v1/a/c3/subdir3/subsubdir/').get_response(self.test_staticweb) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.body, 'index file') + self.assertEqual(resp.body, b'index file') def test_container3subdir(self): resp = Request.blank( '/v1/a/c3/subdir/').get_response(self.test_staticweb) self.assertEqual(resp.status_int, 200) - self.assertIn('Listing of /v1/a/c3/subdir/', resp.body) - self.assertIn('', resp.body) - self.assertNotIn('', resp.body) + self.assertNotIn(b'', resp.body) - self.assertIn('', resp.body) + self.assertIn(b'c11 subdir index' in resp.body) + self.assertIn(b'

c11 subdir index

', resp.body) def test_container11subdirmarkermatchdirtype(self): resp = Request.blank('/v1/a/c11a/subdir/').get_response( self.test_staticweb) self.assertEqual(resp.status_int, 404) - self.assertIn('Index File Not Found', resp.body) + self.assertIn(b'Index File Not Found', resp.body) def test_container11subdirmarkeraltdirtype(self): resp = Request.blank('/v1/a/c11a/subdir2/').get_response( @@ -775,27 +775,27 @@ class TestStaticWeb(unittest.TestCase): resp = Request.blank('/v1/a/c12/').get_response( self.test_staticweb) self.assertEqual(resp.status_int, 200) - self.assertIn('index file', resp.body) + self.assertIn(b'index file', resp.body) def test_container_404_has_css(self): resp = Request.blank('/v1/a/c13/').get_response( self.test_staticweb) self.assertEqual(resp.status_int, 404) - self.assertIn('listing.css', resp.body) + self.assertIn(b'listing.css', resp.body) def test_container_404_has_no_css(self): resp = Request.blank('/v1/a/c7/').get_response( self.test_staticweb) self.assertEqual(resp.status_int, 404) - self.assertNotIn('listing.css', resp.body) - self.assertIn('