2013-09-20 01:00:54 +08:00
|
|
|
# Copyright (c) 2010-2012 OpenStack Foundation
|
2010-07-12 17:03:45 -05:00
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
|
|
# implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
|
|
|
import unittest
|
Remove extra lstat() calls from check_mount
The os.path.exists call performs an lstat, but os.path.ismount already
performs the same check. However, it performs a separate lstat() call
to check for a symlink, which we remove as well, cutting the number
performed in half.
Sample program to be straced for comparison:
from swift.common.constraints import check_mount
import os
os.write(1, "Starting\n")
if check_mount("/", "tmp"):
os.write(1, "Mounted\n")
Here is the output of a check on a mounted file system (common case)
using the new method:
---- strace new ----
write(1, "Starting\n", 9) = 9
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp/..", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
write(1, "Mounted\n", 8) = 8
---- strace old ----
write(1, "Starting\n", 9) = 9
stat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp/..", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
write(1, "Mounted\n", 8) = 8
Change-Id: I027c862a2b7d9ff99d7f61bd43ccc0825dba525c
Signed-off-by: Peter Portante <peter.portante@redhat.com>
2013-07-19 11:34:12 -04:00
|
|
|
import mock
|
2014-03-11 14:18:55 -07:00
|
|
|
import tempfile
|
2014-06-10 22:17:47 -07:00
|
|
|
import time
|
Remove extra lstat() calls from check_mount
The os.path.exists call performs an lstat, but os.path.ismount already
performs the same check. However, it performs a separate lstat() call
to check for a symlink, which we remove as well, cutting the number
performed in half.
Sample program to be straced for comparison:
from swift.common.constraints import check_mount
import os
os.write(1, "Starting\n")
if check_mount("/", "tmp"):
os.write(1, "Mounted\n")
Here is the output of a check on a mounted file system (common case)
using the new method:
---- strace new ----
write(1, "Starting\n", 9) = 9
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp/..", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
write(1, "Mounted\n", 8) = 8
---- strace old ----
write(1, "Starting\n", 9) = 9
stat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp/..", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
write(1, "Mounted\n", 8) = 8
Change-Id: I027c862a2b7d9ff99d7f61bd43ccc0825dba525c
Signed-off-by: Peter Portante <peter.portante@redhat.com>
2013-07-19 11:34:12 -04:00
|
|
|
|
2015-05-25 18:28:02 +02:00
|
|
|
from six.moves import range
|
2017-04-19 15:09:40 +02:00
|
|
|
from test.unit import mock_check_drive
|
2010-07-12 17:03:45 -05:00
|
|
|
|
2016-02-28 01:18:07 +00:00
|
|
|
from swift.common.swob import Request, HTTPException
|
2012-09-04 14:02:19 -07:00
|
|
|
from swift.common.http import HTTP_REQUEST_ENTITY_TOO_LARGE, \
|
2014-08-21 10:33:30 -04:00
|
|
|
HTTP_BAD_REQUEST, HTTP_LENGTH_REQUIRED, HTTP_NOT_IMPLEMENTED
|
2014-04-02 16:16:21 -04:00
|
|
|
from swift.common import constraints, utils
|
2016-09-29 20:24:23 +08:00
|
|
|
from swift.common.constraints import MAX_OBJECT_NAME_LENGTH
|
2010-07-12 17:03:45 -05:00
|
|
|
|
2010-11-16 15:35:39 -08:00
|
|
|
|
2010-07-12 17:03:45 -05:00
|
|
|
class TestConstraints(unittest.TestCase):
|
|
|
|
def test_check_metadata_empty(self):
|
|
|
|
headers = {}
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object'))
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_metadata_good(self):
|
|
|
|
headers = {'X-Object-Meta-Name': 'Value'}
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object'))
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_metadata_empty_name(self):
|
|
|
|
headers = {'X-Object-Meta-': 'Value'}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Metadata name cannot be empty', resp.body)
|
2016-02-28 01:18:07 +00:00
|
|
|
|
|
|
|
def test_check_metadata_non_utf8(self):
|
2018-06-20 15:45:50 -07:00
|
|
|
# Consciously using native "WSGI strings" in headers
|
|
|
|
headers = {'X-Account-Meta-Foo': '\xff'}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'account')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Metadata must be valid UTF-8', resp.body)
|
2016-09-29 20:24:23 +08:00
|
|
|
|
2018-06-20 15:45:50 -07:00
|
|
|
headers = {'X-Container-Meta-\xff': 'foo'}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'container')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Metadata must be valid UTF-8', resp.body)
|
2016-02-28 01:18:07 +00:00
|
|
|
# Object's OK; its metadata isn't serialized as JSON
|
2018-06-20 15:45:50 -07:00
|
|
|
headers = {'X-Object-Meta-Foo': '\xff'}
|
2016-02-28 01:18:07 +00:00
|
|
|
self.assertIsNone(constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object'))
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_metadata_name_length(self):
|
|
|
|
name = 'a' * constraints.MAX_META_NAME_LENGTH
|
|
|
|
headers = {'X-Object-Meta-%s' % name: 'v'}
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object'))
|
2016-09-29 20:24:23 +08:00
|
|
|
|
2010-07-12 17:03:45 -05:00
|
|
|
name = 'a' * (constraints.MAX_META_NAME_LENGTH + 1)
|
|
|
|
headers = {'X-Object-Meta-%s' % name: 'v'}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2013-09-01 01:14:40 -04:00
|
|
|
self.assertIn(
|
2018-06-20 15:45:50 -07:00
|
|
|
b'x-object-meta-%s' % name.encode('ascii'), resp.body.lower())
|
|
|
|
self.assertIn(b'Metadata name too long', resp.body)
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_metadata_value_length(self):
|
|
|
|
value = 'a' * constraints.MAX_META_VALUE_LENGTH
|
|
|
|
headers = {'X-Object-Meta-Name': value}
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object'))
|
2016-09-29 20:24:23 +08:00
|
|
|
|
2010-07-12 17:03:45 -05:00
|
|
|
value = 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)
|
|
|
|
headers = {'X-Object-Meta-Name': value}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'x-object-meta-name', resp.body.lower())
|
2013-09-01 01:14:40 -04:00
|
|
|
self.assertIn(
|
2018-06-20 15:45:50 -07:00
|
|
|
str(constraints.MAX_META_VALUE_LENGTH).encode('ascii'), resp.body)
|
|
|
|
self.assertIn(b'Metadata value longer than 256', resp.body)
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_metadata_count(self):
|
|
|
|
headers = {}
|
2015-05-25 18:28:02 +02:00
|
|
|
for x in range(constraints.MAX_META_COUNT):
|
2010-07-12 17:03:45 -05:00
|
|
|
headers['X-Object-Meta-%d' % x] = 'v'
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object'))
|
2016-09-29 20:24:23 +08:00
|
|
|
|
2010-07-12 17:03:45 -05:00
|
|
|
headers['X-Object-Meta-Too-Many'] = 'v'
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Too many metadata items', resp.body)
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_metadata_size(self):
|
|
|
|
headers = {}
|
|
|
|
size = 0
|
|
|
|
chunk = constraints.MAX_META_NAME_LENGTH + \
|
2013-09-01 01:14:40 -04:00
|
|
|
constraints.MAX_META_VALUE_LENGTH
|
2010-07-12 17:03:45 -05:00
|
|
|
x = 0
|
|
|
|
while size + chunk < constraints.MAX_META_OVERALL_SIZE:
|
|
|
|
headers['X-Object-Meta-%04d%s' %
|
|
|
|
(x, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
|
2013-09-01 01:14:40 -04:00
|
|
|
'v' * constraints.MAX_META_VALUE_LENGTH
|
2010-07-12 17:03:45 -05:00
|
|
|
size += chunk
|
|
|
|
x += 1
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object'))
|
2012-09-05 20:49:50 -07:00
|
|
|
# add two more headers in case adding just one falls exactly on the
|
|
|
|
# limit (eg one header adds 1024 and the limit is 2048)
|
|
|
|
headers['X-Object-Meta-%04d%s' %
|
|
|
|
(x, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
|
2013-09-01 01:14:40 -04:00
|
|
|
'v' * constraints.MAX_META_VALUE_LENGTH
|
2012-09-05 20:49:50 -07:00
|
|
|
headers['X-Object-Meta-%04d%s' %
|
|
|
|
(x + 1, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
|
2013-09-01 01:14:40 -04:00
|
|
|
'v' * constraints.MAX_META_VALUE_LENGTH
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_metadata(Request.blank(
|
|
|
|
'/', headers=headers), 'object')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Total metadata too large', resp.body)
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_object_creation_content_length(self):
|
|
|
|
headers = {'Content-Length': str(constraints.MAX_FILE_SIZE),
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_object_creation(Request.blank(
|
|
|
|
'/', headers=headers), 'object_name'))
|
2014-08-21 10:33:30 -04:00
|
|
|
|
2010-07-12 17:03:45 -05:00
|
|
|
headers = {'Content-Length': str(constraints.MAX_FILE_SIZE + 1),
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_object_creation(
|
|
|
|
Request.blank('/', headers=headers), 'object_name')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_REQUEST_ENTITY_TOO_LARGE)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
2010-07-12 17:03:45 -05:00
|
|
|
headers = {'Transfer-Encoding': 'chunked',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_object_creation(Request.blank(
|
|
|
|
'/', headers=headers), 'object_name'))
|
2014-08-21 10:33:30 -04:00
|
|
|
|
|
|
|
headers = {'Transfer-Encoding': 'gzip',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_object_creation(Request.blank(
|
|
|
|
'/', headers=headers), 'object_name')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Invalid Transfer-Encoding header value', resp.body)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
headers = {'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_object_creation(
|
|
|
|
Request.blank('/', headers=headers), 'object_name')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_LENGTH_REQUIRED)
|
2010-07-12 17:03:45 -05:00
|
|
|
|
2014-08-21 10:33:30 -04:00
|
|
|
headers = {'Content-Length': 'abc',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_object_creation(Request.blank(
|
|
|
|
'/', headers=headers), 'object_name')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Invalid Content-Length header value', resp.body)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
|
|
|
headers = {'Transfer-Encoding': 'gzip,chunked',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_object_creation(Request.blank(
|
|
|
|
'/', headers=headers), 'object_name')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_NOT_IMPLEMENTED)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
2010-07-12 17:03:45 -05:00
|
|
|
def test_check_object_creation_name_length(self):
|
|
|
|
headers = {'Transfer-Encoding': 'chunked',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2010-07-12 17:03:45 -05:00
|
|
|
name = 'o' * constraints.MAX_OBJECT_NAME_LENGTH
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_object_creation(Request.blank(
|
|
|
|
'/', headers=headers), name))
|
2016-09-29 20:24:23 +08:00
|
|
|
|
|
|
|
name = 'o' * (MAX_OBJECT_NAME_LENGTH + 1)
|
|
|
|
resp = constraints.check_object_creation(
|
|
|
|
Request.blank('/', headers=headers), name)
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Object name length of %d longer than %d' %
|
2016-09-29 20:24:23 +08:00
|
|
|
(MAX_OBJECT_NAME_LENGTH + 1, MAX_OBJECT_NAME_LENGTH),
|
|
|
|
resp.body)
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_object_creation_content_type(self):
|
|
|
|
headers = {'Transfer-Encoding': 'chunked',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2017-06-25 10:29:23 +08:00
|
|
|
self.assertIsNone(constraints.check_object_creation(Request.blank(
|
|
|
|
'/', headers=headers), 'object_name'))
|
2016-09-29 20:24:23 +08:00
|
|
|
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
headers = {'Transfer-Encoding': 'chunked',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2016-09-29 20:24:23 +08:00
|
|
|
resp = constraints.check_object_creation(
|
|
|
|
Request.blank('/', headers=headers), 'object_name')
|
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'No content type', resp.body)
|
2010-07-12 17:03:45 -05:00
|
|
|
|
|
|
|
def test_check_object_creation_bad_content_type(self):
|
|
|
|
headers = {'Transfer-Encoding': 'chunked',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'Content-Type': '\xff\xff',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2010-07-12 17:03:45 -05:00
|
|
|
resp = constraints.check_object_creation(
|
|
|
|
Request.blank('/', headers=headers), 'object_name')
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Content-Type', resp.body)
|
2010-07-12 17:03:45 -05:00
|
|
|
|
2014-08-21 10:33:30 -04:00
|
|
|
def test_check_object_creation_bad_delete_headers(self):
|
|
|
|
headers = {'Transfer-Encoding': 'chunked',
|
|
|
|
'Content-Type': 'text/plain',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'X-Delete-After': 'abc',
|
|
|
|
'X-Timestamp': str(time.time())}
|
2014-08-21 10:33:30 -04:00
|
|
|
resp = constraints.check_object_creation(
|
|
|
|
Request.blank('/', headers=headers), 'object_name')
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Non-integer X-Delete-After', resp.body)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
|
|
|
t = str(int(time.time() - 60))
|
|
|
|
headers = {'Transfer-Encoding': 'chunked',
|
|
|
|
'Content-Type': 'text/plain',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'X-Delete-At': t,
|
|
|
|
'X-Timestamp': str(time.time())}
|
2014-08-21 10:33:30 -04:00
|
|
|
resp = constraints.check_object_creation(
|
|
|
|
Request.blank('/', headers=headers), 'object_name')
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(resp.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'X-Delete-At in past', resp.body)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
|
|
|
def test_check_delete_headers(self):
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
# x-delete-at value should be relative to the request timestamp rather
|
|
|
|
# than time.time() so separate the two to ensure the checks are robust
|
2018-01-08 14:36:53 +00:00
|
|
|
ts = utils.Timestamp(time.time() + 100)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
|
|
|
# X-Delete-After
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
headers = {'X-Delete-After': '600',
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
req = constraints.check_delete_headers(
|
2014-08-21 10:33:30 -04:00
|
|
|
Request.blank('/', headers=headers))
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertIsInstance(req, Request)
|
|
|
|
self.assertIn('x-delete-at', req.headers)
|
|
|
|
self.assertNotIn('x-delete-after', req.headers)
|
|
|
|
expected_delete_at = str(int(ts) + 600)
|
|
|
|
self.assertEqual(req.headers.get('X-Delete-At'), expected_delete_at)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
headers = {'X-Delete-After': 'abc',
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
|
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
2014-08-21 10:33:30 -04:00
|
|
|
Request.blank('/', headers=headers))
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Non-integer X-Delete-After', cm.exception.body)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
|
|
|
|
headers = {'X-Delete-After': '60.1',
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
2014-08-21 10:33:30 -04:00
|
|
|
Request.blank('/', headers=headers))
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Non-integer X-Delete-After', cm.exception.body)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
|
|
|
|
headers = {'X-Delete-After': '-1',
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
2014-08-21 10:33:30 -04:00
|
|
|
Request.blank('/', headers=headers))
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'X-Delete-After in past', cm.exception.body)
|
2014-08-21 10:33:30 -04:00
|
|
|
|
2018-01-05 14:44:52 +00:00
|
|
|
headers = {'X-Delete-After': '0',
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
2018-01-05 14:44:52 +00:00
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
|
|
|
Request.blank('/', headers=headers))
|
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'X-Delete-After in past', cm.exception.body)
|
2018-01-05 14:44:52 +00:00
|
|
|
|
2018-01-08 10:14:06 +00:00
|
|
|
# x-delete-after = 0 disallowed when it results in x-delete-at equal to
|
|
|
|
# the timestamp
|
|
|
|
headers = {'X-Delete-After': '0',
|
|
|
|
'X-Timestamp': utils.Timestamp(int(ts)).internal}
|
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
|
|
|
Request.blank('/', headers=headers))
|
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'X-Delete-After in past', cm.exception.body)
|
2018-01-08 10:14:06 +00:00
|
|
|
|
2014-08-21 10:33:30 -04:00
|
|
|
# X-Delete-At
|
2018-01-08 14:36:53 +00:00
|
|
|
delete_at = str(int(ts) + 100)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
headers = {'X-Delete-At': delete_at,
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
req = constraints.check_delete_headers(
|
2014-08-21 10:33:30 -04:00
|
|
|
Request.blank('/', headers=headers))
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertIsInstance(req, Request)
|
|
|
|
self.assertIn('x-delete-at', req.headers)
|
|
|
|
self.assertEqual(req.headers.get('X-Delete-At'), delete_at)
|
|
|
|
|
|
|
|
headers = {'X-Delete-At': 'abc',
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
2014-08-21 10:33:30 -04:00
|
|
|
Request.blank('/', headers=headers))
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Non-integer X-Delete-At', cm.exception.body)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
|
2018-01-08 14:36:53 +00:00
|
|
|
delete_at = str(int(ts) + 100) + '.1'
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
headers = {'X-Delete-At': delete_at,
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
2014-08-21 10:33:30 -04:00
|
|
|
Request.blank('/', headers=headers))
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'Non-integer X-Delete-At', cm.exception.body)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
|
2018-01-08 14:36:53 +00:00
|
|
|
delete_at = str(int(ts) - 1)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
headers = {'X-Delete-At': delete_at,
|
2018-01-08 14:36:53 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
2014-08-21 10:33:30 -04:00
|
|
|
Request.blank('/', headers=headers))
|
2018-01-08 10:14:06 +00:00
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'X-Delete-At in past', cm.exception.body)
|
2018-01-08 10:14:06 +00:00
|
|
|
|
|
|
|
# x-delete-at disallowed when exactly equal to timestamp
|
|
|
|
delete_at = str(int(ts))
|
|
|
|
headers = {'X-Delete-At': delete_at,
|
|
|
|
'X-Timestamp': utils.Timestamp(int(ts)).internal}
|
|
|
|
with self.assertRaises(HTTPException) as cm:
|
|
|
|
constraints.check_delete_headers(
|
|
|
|
Request.blank('/', headers=headers))
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertEqual(cm.exception.status_int, HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertIn(b'X-Delete-At in past', cm.exception.body)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
|
|
|
|
def test_check_delete_headers_removes_delete_after(self):
|
2018-01-17 15:25:33 +00:00
|
|
|
ts = utils.Timestamp.now()
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
headers = {'Content-Length': '0',
|
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Delete-After': '42',
|
2018-01-17 15:25:33 +00:00
|
|
|
'X-Delete-At': str(int(ts) + 40),
|
|
|
|
'X-Timestamp': ts.internal}
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
req = Request.blank('/', headers=headers)
|
|
|
|
constraints.check_delete_headers(req)
|
|
|
|
self.assertNotIn('X-Delete-After', req.headers)
|
2018-01-17 15:25:33 +00:00
|
|
|
self.assertEqual(req.headers['X-Delete-At'], str(int(ts) + 42))
|
2014-08-21 10:33:30 -04:00
|
|
|
|
2014-09-08 14:06:00 +01:00
|
|
|
def test_check_delete_headers_sets_delete_at(self):
|
2018-02-07 05:50:12 +00:00
|
|
|
ts = utils.Timestamp.now()
|
|
|
|
expected = str(int(ts) + 1000)
|
2014-09-08 14:06:00 +01:00
|
|
|
# check delete-at is passed through
|
|
|
|
headers = {'Content-Length': '0',
|
|
|
|
'Content-Type': 'text/plain',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'X-Delete-At': expected,
|
2018-02-07 05:50:12 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
2014-09-08 14:06:00 +01:00
|
|
|
req = Request.blank('/', headers=headers)
|
|
|
|
constraints.check_delete_headers(req)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertIn('X-Delete-At', req.headers)
|
|
|
|
self.assertEqual(req.headers['X-Delete-At'], expected)
|
2014-09-08 14:06:00 +01:00
|
|
|
|
|
|
|
# check delete-after is converted to delete-at
|
|
|
|
headers = {'Content-Length': '0',
|
|
|
|
'Content-Type': 'text/plain',
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
'X-Delete-After': '42',
|
2018-02-07 05:50:12 +00:00
|
|
|
'X-Timestamp': ts.internal}
|
2014-09-08 14:06:00 +01:00
|
|
|
req = Request.blank('/', headers=headers)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
constraints.check_delete_headers(req)
|
|
|
|
self.assertIn('X-Delete-At', req.headers)
|
2018-02-07 05:50:12 +00:00
|
|
|
expected = str(int(ts) + 42)
|
2014-09-08 14:06:00 +01:00
|
|
|
self.assertEqual(req.headers['X-Delete-At'], expected)
|
|
|
|
|
|
|
|
# check delete-after takes precedence over delete-at
|
|
|
|
headers = {'Content-Length': '0',
|
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Delete-After': '42',
|
2018-02-07 05:50:12 +00:00
|
|
|
'X-Delete-At': str(int(ts) + 40),
|
|
|
|
'X-Timestamp': ts.internal}
|
2014-09-08 14:06:00 +01:00
|
|
|
req = Request.blank('/', headers=headers)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
constraints.check_delete_headers(req)
|
|
|
|
self.assertIn('X-Delete-At', req.headers)
|
2014-09-08 14:06:00 +01:00
|
|
|
self.assertEqual(req.headers['X-Delete-At'], expected)
|
|
|
|
|
|
|
|
headers = {'Content-Length': '0',
|
|
|
|
'Content-Type': 'text/plain',
|
|
|
|
'X-Delete-After': '42',
|
2018-02-07 05:50:12 +00:00
|
|
|
'X-Delete-At': str(int(ts) + 44),
|
|
|
|
'X-Timestamp': ts.internal}
|
2014-09-08 14:06:00 +01:00
|
|
|
req = Request.blank('/', headers=headers)
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
constraints.check_delete_headers(req)
|
|
|
|
self.assertIn('X-Delete-At', req.headers)
|
2014-09-08 14:06:00 +01:00
|
|
|
self.assertEqual(req.headers['X-Delete-At'], expected)
|
|
|
|
|
2017-04-19 15:09:40 +02:00
|
|
|
def test_check_drive_invalid_path(self):
|
|
|
|
root = '/srv/'
|
|
|
|
with mock_check_drive() as mocks:
|
2017-09-15 06:15:40 +00:00
|
|
|
drive = 'foo?bar'
|
|
|
|
with self.assertRaises(ValueError) as exc_mgr:
|
|
|
|
constraints.check_dir(root, drive)
|
|
|
|
self.assertEqual(str(exc_mgr.exception),
|
|
|
|
'%s is not a valid drive name' % drive)
|
|
|
|
|
|
|
|
drive = 'foo bar'
|
|
|
|
with self.assertRaises(ValueError) as exc_mgr:
|
|
|
|
constraints.check_mount(root, drive)
|
|
|
|
self.assertEqual(str(exc_mgr.exception),
|
|
|
|
'%s is not a valid drive name' % drive)
|
|
|
|
|
|
|
|
drive = 'foo/bar'
|
|
|
|
with self.assertRaises(ValueError) as exc_mgr:
|
|
|
|
constraints.check_drive(root, drive, True)
|
|
|
|
self.assertEqual(str(exc_mgr.exception),
|
|
|
|
'%s is not a valid drive name' % drive)
|
|
|
|
|
|
|
|
drive = 'foo%bar'
|
|
|
|
with self.assertRaises(ValueError) as exc_mgr:
|
|
|
|
constraints.check_drive(root, drive, False)
|
|
|
|
self.assertEqual(str(exc_mgr.exception),
|
|
|
|
'%s is not a valid drive name' % drive)
|
2017-04-19 15:09:40 +02:00
|
|
|
self.assertEqual([], mocks['isdir'].call_args_list)
|
|
|
|
self.assertEqual([], mocks['ismount'].call_args_list)
|
|
|
|
|
|
|
|
def test_check_drive_ismount(self):
|
|
|
|
root = '/srv'
|
|
|
|
path = 'sdb1'
|
|
|
|
with mock_check_drive(ismount=True) as mocks:
|
2017-09-15 06:15:40 +00:00
|
|
|
with self.assertRaises(ValueError) as exc_mgr:
|
|
|
|
constraints.check_dir(root, path)
|
|
|
|
self.assertEqual(str(exc_mgr.exception),
|
|
|
|
'/srv/sdb1 is not a directory')
|
|
|
|
|
|
|
|
with self.assertRaises(ValueError) as exc_mgr:
|
|
|
|
constraints.check_drive(root, path, False)
|
|
|
|
self.assertEqual(str(exc_mgr.exception),
|
|
|
|
'/srv/sdb1 is not a directory')
|
|
|
|
|
2017-04-19 15:09:40 +02:00
|
|
|
self.assertEqual([mock.call('/srv/sdb1'), mock.call('/srv/sdb1')],
|
|
|
|
mocks['isdir'].call_args_list)
|
|
|
|
self.assertEqual([], mocks['ismount'].call_args_list)
|
2017-09-15 06:15:40 +00:00
|
|
|
|
2017-04-19 15:09:40 +02:00
|
|
|
with mock_check_drive(ismount=True) as mocks:
|
|
|
|
self.assertEqual('/srv/sdb1', constraints.check_mount(root, path))
|
|
|
|
self.assertEqual('/srv/sdb1', constraints.check_drive(
|
|
|
|
root, path, True))
|
|
|
|
self.assertEqual([], mocks['isdir'].call_args_list)
|
|
|
|
self.assertEqual([mock.call('/srv/sdb1'), mock.call('/srv/sdb1')],
|
|
|
|
mocks['ismount'].call_args_list)
|
|
|
|
|
|
|
|
def test_check_drive_isdir(self):
|
|
|
|
root = '/srv'
|
|
|
|
path = 'sdb2'
|
|
|
|
with mock_check_drive(isdir=True) as mocks:
|
|
|
|
self.assertEqual('/srv/sdb2', constraints.check_dir(root, path))
|
|
|
|
self.assertEqual('/srv/sdb2', constraints.check_drive(
|
|
|
|
root, path, False))
|
|
|
|
self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')],
|
|
|
|
mocks['isdir'].call_args_list)
|
|
|
|
self.assertEqual([], mocks['ismount'].call_args_list)
|
2017-09-15 06:15:40 +00:00
|
|
|
|
2017-04-19 15:09:40 +02:00
|
|
|
with mock_check_drive(isdir=True) as mocks:
|
2017-09-15 06:15:40 +00:00
|
|
|
with self.assertRaises(ValueError) as exc_mgr:
|
|
|
|
constraints.check_mount(root, path)
|
|
|
|
self.assertEqual(str(exc_mgr.exception),
|
|
|
|
'/srv/sdb2 is not mounted')
|
|
|
|
|
|
|
|
with self.assertRaises(ValueError) as exc_mgr:
|
|
|
|
constraints.check_drive(root, path, True)
|
|
|
|
self.assertEqual(str(exc_mgr.exception),
|
|
|
|
'/srv/sdb2 is not mounted')
|
|
|
|
|
2017-04-19 15:09:40 +02:00
|
|
|
self.assertEqual([], mocks['isdir'].call_args_list)
|
|
|
|
self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')],
|
|
|
|
mocks['ismount'].call_args_list)
|
2010-07-29 13:30:16 -05:00
|
|
|
|
|
|
|
def test_check_float(self):
|
|
|
|
self.assertFalse(constraints.check_float(''))
|
|
|
|
self.assertTrue(constraints.check_float('0'))
|
2010-07-12 17:03:45 -05:00
|
|
|
|
2014-06-10 22:17:47 -07:00
|
|
|
def test_valid_timestamp(self):
|
|
|
|
self.assertRaises(HTTPException,
|
|
|
|
constraints.valid_timestamp,
|
|
|
|
Request.blank('/'))
|
|
|
|
self.assertRaises(HTTPException,
|
|
|
|
constraints.valid_timestamp,
|
|
|
|
Request.blank('/', headers={
|
|
|
|
'X-Timestamp': 'asdf'}))
|
2017-04-27 14:19:00 -07:00
|
|
|
timestamp = utils.Timestamp.now()
|
2014-06-10 22:17:47 -07:00
|
|
|
req = Request.blank('/', headers={'X-Timestamp': timestamp.internal})
|
|
|
|
self.assertEqual(timestamp, constraints.valid_timestamp(req))
|
|
|
|
req = Request.blank('/', headers={'X-Timestamp': timestamp.normal})
|
|
|
|
self.assertEqual(timestamp, constraints.valid_timestamp(req))
|
|
|
|
|
2012-06-06 03:39:53 +09:00
|
|
|
def test_check_utf8(self):
|
|
|
|
unicode_sample = u'\uc77c\uc601'
|
2013-01-12 06:54:17 +00:00
|
|
|
unicode_with_null = u'abc\u0000def'
|
2012-06-06 03:39:53 +09:00
|
|
|
|
2018-06-20 15:45:50 -07:00
|
|
|
# Some false-y values
|
|
|
|
self.assertFalse(constraints.check_utf8(None))
|
|
|
|
self.assertFalse(constraints.check_utf8(''))
|
|
|
|
self.assertFalse(constraints.check_utf8(b''))
|
|
|
|
self.assertFalse(constraints.check_utf8(u''))
|
|
|
|
|
|
|
|
# invalid utf8 bytes
|
|
|
|
self.assertFalse(constraints.check_utf8(
|
|
|
|
unicode_sample.encode('utf-8')[::-1]))
|
|
|
|
# unicode with null
|
|
|
|
self.assertFalse(constraints.check_utf8(unicode_with_null))
|
|
|
|
# utf8 bytes with null
|
|
|
|
self.assertFalse(constraints.check_utf8(
|
|
|
|
unicode_with_null.encode('utf8')))
|
|
|
|
|
|
|
|
self.assertTrue(constraints.check_utf8('this is ascii and utf-8, too'))
|
|
|
|
self.assertTrue(constraints.check_utf8(unicode_sample))
|
|
|
|
self.assertTrue(constraints.check_utf8(unicode_sample.encode('utf8')))
|
2012-06-06 03:39:53 +09:00
|
|
|
|
2014-10-27 16:29:07 +00:00
|
|
|
def test_check_utf8_non_canonical(self):
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertFalse(constraints.check_utf8(b'\xed\xa0\xbc\xed\xbc\xb8'))
|
|
|
|
self.assertTrue(constraints.check_utf8(u'\U0001f338'))
|
|
|
|
self.assertTrue(constraints.check_utf8(b'\xf0\x9f\x8c\xb8'))
|
|
|
|
self.assertTrue(constraints.check_utf8(u'\U0001f338'.encode('utf8')))
|
|
|
|
self.assertFalse(constraints.check_utf8(b'\xed\xa0\xbd\xed\xb9\x88'))
|
|
|
|
self.assertTrue(constraints.check_utf8(u'\U0001f648'))
|
2014-10-27 16:29:07 +00:00
|
|
|
|
Reject object names with Unicode surrogates
Technically, you can't encode surrogates into UTF-8 at all, but Python
2 lets you get away with it. Python 3 does not.
We already have a check for surrogate pairs (commit 0080337), but not
one for lone surrogates. This commit forbids object names with lone
surrogates in them.
The problem with surrogates is trivially reproducible:
swift@saio:~$ python2.7
Python 2.7.3 (default, Feb 27 2014, 19:58:35)
[GCC 4.6.3] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> b'\xed\xa0\xbc'.decode('utf-8')
u'\ud83c'
>>>
swift@saio:~$ python3.3
Python 3.3.5 (default, Aug 4 2014, 15:27:24)
[GCC 4.6.3] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> b'\xed\xa0\xbc'.decode('utf-8')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xed in position 0: invalid continuation byte
>>>
See also http://bugs.python.org/issue9133
Change-Id: I7c31022e8a028c3cdf2ed1586349509d96cfded9
2014-11-07 13:53:46 -08:00
|
|
|
def test_check_utf8_lone_surrogates(self):
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertFalse(constraints.check_utf8(b'\xed\xa0\xbc'))
|
|
|
|
self.assertFalse(constraints.check_utf8(u'\ud83c'))
|
|
|
|
self.assertFalse(constraints.check_utf8(b'\xed\xb9\x88'))
|
|
|
|
self.assertFalse(constraints.check_utf8(u'\ude48'))
|
|
|
|
|
|
|
|
self.assertFalse(constraints.check_utf8(u'\ud800'))
|
|
|
|
self.assertFalse(constraints.check_utf8(u'\udc00'))
|
|
|
|
self.assertFalse(constraints.check_utf8(u'\udcff'))
|
|
|
|
self.assertFalse(constraints.check_utf8(u'\udfff'))
|
Reject object names with Unicode surrogates
Technically, you can't encode surrogates into UTF-8 at all, but Python
2 lets you get away with it. Python 3 does not.
We already have a check for surrogate pairs (commit 0080337), but not
one for lone surrogates. This commit forbids object names with lone
surrogates in them.
The problem with surrogates is trivially reproducible:
swift@saio:~$ python2.7
Python 2.7.3 (default, Feb 27 2014, 19:58:35)
[GCC 4.6.3] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> b'\xed\xa0\xbc'.decode('utf-8')
u'\ud83c'
>>>
swift@saio:~$ python3.3
Python 3.3.5 (default, Aug 4 2014, 15:27:24)
[GCC 4.6.3] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> b'\xed\xa0\xbc'.decode('utf-8')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xed in position 0: invalid continuation byte
>>>
See also http://bugs.python.org/issue9133
Change-Id: I7c31022e8a028c3cdf2ed1586349509d96cfded9
2014-11-07 13:53:46 -08:00
|
|
|
|
2013-02-13 12:31:55 -08:00
|
|
|
def test_validate_bad_meta(self):
|
|
|
|
req = Request.blank(
|
|
|
|
'/v/a/c/o',
|
|
|
|
headers={'x-object-meta-hello':
|
|
|
|
'ab' * constraints.MAX_HEADER_SIZE})
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(constraints.check_metadata(req, 'object').status_int,
|
|
|
|
HTTP_BAD_REQUEST)
|
2018-06-20 15:45:50 -07:00
|
|
|
resp = constraints.check_metadata(req, 'object')
|
|
|
|
self.assertIsNotNone(resp)
|
|
|
|
self.assertIn(b'x-object-meta-hello', resp.body.lower())
|
2013-02-13 12:31:55 -08:00
|
|
|
|
2013-06-19 01:02:37 +08:00
|
|
|
def test_validate_constraints(self):
|
|
|
|
c = constraints
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertGreater(c.MAX_META_OVERALL_SIZE, c.MAX_META_NAME_LENGTH)
|
|
|
|
self.assertGreater(c.MAX_META_OVERALL_SIZE, c.MAX_META_VALUE_LENGTH)
|
|
|
|
self.assertGreater(c.MAX_HEADER_SIZE, c.MAX_META_NAME_LENGTH)
|
|
|
|
self.assertGreater(c.MAX_HEADER_SIZE, c.MAX_META_VALUE_LENGTH)
|
2013-06-19 01:02:37 +08:00
|
|
|
|
2014-04-30 15:00:49 +03:00
|
|
|
def test_check_account_format(self):
|
|
|
|
req = Request.blank(
|
|
|
|
'/v/a/c/o',
|
|
|
|
headers={'X-Copy-From-Account': 'account/with/slashes'})
|
|
|
|
self.assertRaises(HTTPException,
|
|
|
|
constraints.check_account_format,
|
|
|
|
req, req.headers['X-Copy-From-Account'])
|
2015-02-11 11:37:31 -08:00
|
|
|
req = Request.blank(
|
|
|
|
'/v/a/c/o',
|
|
|
|
headers={'X-Copy-From-Account': ''})
|
|
|
|
self.assertRaises(HTTPException,
|
|
|
|
constraints.check_account_format,
|
|
|
|
req, req.headers['X-Copy-From-Account'])
|
2014-04-30 15:00:49 +03:00
|
|
|
|
2014-11-09 13:13:27 -05:00
|
|
|
def test_check_container_format(self):
|
|
|
|
invalid_versions_locations = (
|
|
|
|
'container/with/slashes',
|
|
|
|
'', # empty
|
|
|
|
)
|
|
|
|
for versions_location in invalid_versions_locations:
|
|
|
|
req = Request.blank(
|
|
|
|
'/v/a/c/o', headers={
|
|
|
|
'X-Versions-Location': versions_location})
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
with self.assertRaises(HTTPException) as cm:
|
2014-11-09 13:13:27 -05:00
|
|
|
constraints.check_container_format(
|
|
|
|
req, req.headers['X-Versions-Location'])
|
Fix time skew when using X-Delete-After
When a client sent "X-Delete-After: <n>", the proxy and all object
servers would each compute X-Delete-At as "int(time.time() +
n)". However, since they don't all compute it at exactly the same
time, the objects stored on disk can end up with differing values for
X-Delete-At, and in that case, the object-expirer queue has multiple
entries for the same object (one for each distinct X-Delete-At value).
This commit makes two changes, either one of which is sufficient to
fix the bug.
First, after computing X-Delete-At from X-Delete-After, X-Delete-After
is removed from the request's headers. Thus, the proxy computes
X-Delete-At, and the object servers don't, so there's only a single
value.
Second, computation of X-Delete-At now uses the request's X-Timestamp
instead of time.time(). In the proxy, these values are essentially the
same; the proxy is responsible for setting X-Timestamp. In the object
server, this ensures that all computed X-Delete-At values are
identical, even if the object servers' clocks are not, or if one
object server takes an extra second to respond to a PUT request.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: I9a1b6826c4c553f0442cfe2bb78cdf49508fa4a5
Closes-Bug: 1741371
2018-01-04 20:28:28 -08:00
|
|
|
self.assertTrue(cm.exception.body.startswith(
|
2018-06-20 15:45:50 -07:00
|
|
|
b'Container name cannot'))
|
2014-11-09 13:13:27 -05:00
|
|
|
|
2016-09-06 11:55:41 +08:00
|
|
|
def test_valid_api_version(self):
|
|
|
|
version = 'v1'
|
|
|
|
self.assertTrue(constraints.valid_api_version(version))
|
|
|
|
|
|
|
|
version = 'v1.0'
|
|
|
|
self.assertTrue(constraints.valid_api_version(version))
|
|
|
|
|
|
|
|
version = 'v2'
|
|
|
|
self.assertFalse(constraints.valid_api_version(version))
|
|
|
|
|
Remove extra lstat() calls from check_mount
The os.path.exists call performs an lstat, but os.path.ismount already
performs the same check. However, it performs a separate lstat() call
to check for a symlink, which we remove as well, cutting the number
performed in half.
Sample program to be straced for comparison:
from swift.common.constraints import check_mount
import os
os.write(1, "Starting\n")
if check_mount("/", "tmp"):
os.write(1, "Mounted\n")
Here is the output of a check on a mounted file system (common case)
using the new method:
---- strace new ----
write(1, "Starting\n", 9) = 9
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp/..", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
write(1, "Mounted\n", 8) = 8
---- strace old ----
write(1, "Starting\n", 9) = 9
stat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=8460, ...}) = 0
lstat("/tmp/..", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
write(1, "Mounted\n", 8) = 8
Change-Id: I027c862a2b7d9ff99d7f61bd43ccc0825dba525c
Signed-off-by: Peter Portante <peter.portante@redhat.com>
2013-07-19 11:34:12 -04:00
|
|
|
|
2014-03-11 14:18:55 -07:00
|
|
|
class TestConstraintsConfig(unittest.TestCase):
|
|
|
|
|
|
|
|
def test_default_constraints(self):
|
|
|
|
for key in constraints.DEFAULT_CONSTRAINTS:
|
|
|
|
# if there is local over-rides in swift.conf we just continue on
|
|
|
|
if key in constraints.OVERRIDE_CONSTRAINTS:
|
|
|
|
continue
|
|
|
|
# module level attrs (that aren't in OVERRIDE) should have the
|
|
|
|
# same value as the DEFAULT map
|
|
|
|
module_level_value = getattr(constraints, key.upper())
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(constraints.DEFAULT_CONSTRAINTS[key],
|
|
|
|
module_level_value)
|
2014-03-11 14:18:55 -07:00
|
|
|
|
|
|
|
def test_effective_constraints(self):
|
|
|
|
for key in constraints.DEFAULT_CONSTRAINTS:
|
|
|
|
# module level attrs should always mirror the same value as the
|
|
|
|
# EFFECTIVE map
|
|
|
|
module_level_value = getattr(constraints, key.upper())
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(constraints.EFFECTIVE_CONSTRAINTS[key],
|
|
|
|
module_level_value)
|
2014-03-11 14:18:55 -07:00
|
|
|
# if there are local over-rides in swift.conf those should be
|
|
|
|
# reflected in the EFFECTIVE, otherwise we expect the DEFAULTs
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(constraints.EFFECTIVE_CONSTRAINTS[key],
|
|
|
|
constraints.OVERRIDE_CONSTRAINTS.get(
|
|
|
|
key, constraints.DEFAULT_CONSTRAINTS[key]))
|
2014-03-11 14:18:55 -07:00
|
|
|
|
|
|
|
def test_override_constraints(self):
|
|
|
|
try:
|
|
|
|
with tempfile.NamedTemporaryFile() as f:
|
2018-06-20 15:45:50 -07:00
|
|
|
f.write(b'[swift-constraints]\n')
|
2014-03-11 14:18:55 -07:00
|
|
|
# set everything to 1
|
|
|
|
for key in constraints.DEFAULT_CONSTRAINTS:
|
2018-06-20 15:45:50 -07:00
|
|
|
f.write(b'%s = 1\n' % key.encode('ascii'))
|
2014-03-11 14:18:55 -07:00
|
|
|
f.flush()
|
2014-04-02 16:16:21 -04:00
|
|
|
with mock.patch.object(utils, 'SWIFT_CONF_FILE', f.name):
|
2014-03-11 14:18:55 -07:00
|
|
|
constraints.reload_constraints()
|
|
|
|
for key in constraints.DEFAULT_CONSTRAINTS:
|
|
|
|
# module level attrs should all be 1
|
|
|
|
module_level_value = getattr(constraints, key.upper())
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(module_level_value, 1)
|
2014-03-11 14:18:55 -07:00
|
|
|
# all keys should be in OVERRIDE
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(constraints.OVERRIDE_CONSTRAINTS[key],
|
|
|
|
module_level_value)
|
2014-03-11 14:18:55 -07:00
|
|
|
# module level attrs should always mirror the same value as
|
|
|
|
# the EFFECTIVE map
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(constraints.EFFECTIVE_CONSTRAINTS[key],
|
|
|
|
module_level_value)
|
2014-03-11 14:18:55 -07:00
|
|
|
finally:
|
|
|
|
constraints.reload_constraints()
|
|
|
|
|
|
|
|
def test_reload_reset(self):
|
|
|
|
try:
|
|
|
|
with tempfile.NamedTemporaryFile() as f:
|
2018-06-20 15:45:50 -07:00
|
|
|
f.write(b'[swift-constraints]\n')
|
2014-03-11 14:18:55 -07:00
|
|
|
# set everything to 1
|
|
|
|
for key in constraints.DEFAULT_CONSTRAINTS:
|
2018-06-20 15:45:50 -07:00
|
|
|
f.write(b'%s = 1\n' % key.encode('ascii'))
|
2014-03-11 14:18:55 -07:00
|
|
|
f.flush()
|
2014-04-02 16:16:21 -04:00
|
|
|
with mock.patch.object(utils, 'SWIFT_CONF_FILE', f.name):
|
2014-03-11 14:18:55 -07:00
|
|
|
constraints.reload_constraints()
|
|
|
|
self.assertTrue(constraints.SWIFT_CONSTRAINTS_LOADED)
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(sorted(constraints.DEFAULT_CONSTRAINTS.keys()),
|
|
|
|
sorted(constraints.OVERRIDE_CONSTRAINTS.keys()))
|
2014-03-11 14:18:55 -07:00
|
|
|
# file is now deleted...
|
2014-04-02 16:16:21 -04:00
|
|
|
with mock.patch.object(utils, 'SWIFT_CONF_FILE', f.name):
|
2014-03-11 14:18:55 -07:00
|
|
|
constraints.reload_constraints()
|
2014-04-25 19:54:49 -07:00
|
|
|
# no constraints have been loaded from non-existent swift.conf
|
2014-03-11 14:18:55 -07:00
|
|
|
self.assertFalse(constraints.SWIFT_CONSTRAINTS_LOADED)
|
|
|
|
# no constraints are in OVERRIDE
|
2018-06-20 15:45:50 -07:00
|
|
|
self.assertEqual([], list(constraints.OVERRIDE_CONSTRAINTS.keys()))
|
2014-03-11 14:18:55 -07:00
|
|
|
# the EFFECTIVE constraints mirror DEFAULT
|
2015-08-05 23:58:14 +05:30
|
|
|
self.assertEqual(constraints.EFFECTIVE_CONSTRAINTS,
|
|
|
|
constraints.DEFAULT_CONSTRAINTS)
|
2014-03-11 14:18:55 -07:00
|
|
|
finally:
|
|
|
|
constraints.reload_constraints()
|
|
|
|
|
|
|
|
|
2010-07-12 17:03:45 -05:00
|
|
|
if __name__ == '__main__':
|
|
|
|
unittest.main()
|