279 lines
10 KiB
Python
279 lines
10 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
# use this file except in compliance with the License. You may obtain a copy
|
|
# of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import collections
|
|
import itertools
|
|
import json
|
|
import mock
|
|
import six
|
|
import unittest
|
|
|
|
from swift.cli import container_deleter
|
|
from swift.common import internal_client
|
|
from swift.common import swob
|
|
from swift.common import utils
|
|
|
|
AppCall = collections.namedtuple('AppCall', [
|
|
'method', 'path', 'query', 'headers', 'body'])
|
|
|
|
|
|
class FakeInternalClient(internal_client.InternalClient):
|
|
def __init__(self, responses):
|
|
self.resp_iter = iter(responses)
|
|
self.calls = []
|
|
|
|
def make_request(self, method, path, headers, acceptable_statuses,
|
|
body_file=None, params=None):
|
|
if body_file is None:
|
|
body = None
|
|
else:
|
|
body = body_file.read()
|
|
path, _, query = path.partition('?')
|
|
self.calls.append(AppCall(method, path, query, headers, body))
|
|
resp = next(self.resp_iter)
|
|
if isinstance(resp, Exception):
|
|
raise resp
|
|
return resp
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
unused_responses = [r for r in self.resp_iter]
|
|
if unused_responses:
|
|
raise Exception('Unused responses: %r' % unused_responses)
|
|
|
|
|
|
class TestContainerDeleter(unittest.TestCase):
|
|
def setUp(self):
|
|
patcher = mock.patch.object(container_deleter.time, 'time',
|
|
side_effect=itertools.count())
|
|
patcher.__enter__()
|
|
self.addCleanup(patcher.__exit__)
|
|
|
|
patcher = mock.patch.object(container_deleter, 'OBJECTS_PER_UPDATE', 5)
|
|
patcher.__enter__()
|
|
self.addCleanup(patcher.__exit__)
|
|
|
|
def test_make_delete_jobs(self):
|
|
ts = '1558463777.42739'
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
'acct', 'cont', ['obj1', 'obj2'],
|
|
utils.Timestamp(ts)),
|
|
[{'name': ts + '-acct/cont/obj1',
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': ts + '-acct/cont/obj2',
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_make_delete_jobs_native_utf8(self):
|
|
ts = '1558463777.42739'
|
|
uacct = acct = u'acct-\U0001f334'
|
|
ucont = cont = u'cont-\N{SNOWMAN}'
|
|
uobj1 = obj1 = u'obj-\N{GREEK CAPITAL LETTER ALPHA}'
|
|
uobj2 = obj2 = u'/obj-\N{GREEK CAPITAL LETTER OMEGA}'
|
|
if six.PY2:
|
|
acct = acct.encode('utf8')
|
|
cont = cont.encode('utf8')
|
|
obj1 = obj1.encode('utf8')
|
|
obj2 = obj2.encode('utf8')
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
acct, cont, [obj1, obj2], utils.Timestamp(ts)),
|
|
[{'name': u'%s-%s/%s/%s' % (ts, uacct, ucont, uobj1),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': u'%s-%s/%s/%s' % (ts, uacct, ucont, uobj2),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_make_delete_jobs_unicode_utf8(self):
|
|
ts = '1558463777.42739'
|
|
acct = u'acct-\U0001f334'
|
|
cont = u'cont-\N{SNOWMAN}'
|
|
obj1 = u'obj-\N{GREEK CAPITAL LETTER ALPHA}'
|
|
obj2 = u'obj-\N{GREEK CAPITAL LETTER OMEGA}'
|
|
self.assertEqual(
|
|
container_deleter.make_delete_jobs(
|
|
acct, cont, [obj1, obj2], utils.Timestamp(ts)),
|
|
[{'name': u'%s-%s/%s/%s' % (ts, acct, cont, obj1),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'},
|
|
{'name': u'%s-%s/%s/%s' % (ts, acct, cont, obj2),
|
|
'deleted': 0,
|
|
'created_at': ts,
|
|
'etag': utils.MD5_OF_EMPTY_STRING,
|
|
'size': 0,
|
|
'storage_policy_index': 0,
|
|
'content_type': 'application/async-deleted'}])
|
|
|
|
def test_mark_for_deletion_empty_no_yield(self):
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
])),
|
|
]) as swift:
|
|
self.assertEqual(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'marker',
|
|
'end',
|
|
'prefix',
|
|
timestamp=None,
|
|
yield_time=None,
|
|
), 0)
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=marker&end_marker=end&prefix=prefix',
|
|
{}, None),
|
|
])
|
|
|
|
def test_mark_for_deletion_empty_with_yield(self):
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
])),
|
|
]) as swift:
|
|
self.assertEqual(list(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'marker',
|
|
'end',
|
|
'prefix',
|
|
timestamp=None,
|
|
yield_time=0.5,
|
|
)), [(0, None)])
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=marker&end_marker=end&prefix=prefix',
|
|
{}, None),
|
|
])
|
|
|
|
def test_mark_for_deletion_one_update_no_yield(self):
|
|
ts = '1558463777.42739'
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
{'name': '/obj1'},
|
|
{'name': 'obj2'},
|
|
{'name': 'obj3'},
|
|
])),
|
|
swob.Response(json.dumps([
|
|
])),
|
|
swob.Response(status=202),
|
|
]) as swift:
|
|
self.assertEqual(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'',
|
|
'',
|
|
'',
|
|
timestamp=utils.Timestamp(ts),
|
|
yield_time=None,
|
|
), 3)
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=&end_marker=&prefix=', {}, None),
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=obj3&end_marker=&prefix=', {}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
])
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-1].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container', ['/obj1', 'obj2', 'obj3'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|
|
|
|
def test_mark_for_deletion_two_updates_with_yield(self):
|
|
ts = '1558463777.42739'
|
|
with FakeInternalClient([
|
|
swob.Response(json.dumps([
|
|
{'name': 'obj1'},
|
|
{'name': 'obj2'},
|
|
{'name': 'obj3'},
|
|
{'name': u'obj4-\N{SNOWMAN}'},
|
|
{'name': 'obj5'},
|
|
{'name': 'obj6'},
|
|
])),
|
|
swob.Response(status=202),
|
|
swob.Response(json.dumps([
|
|
])),
|
|
swob.Response(status=202),
|
|
]) as swift:
|
|
self.assertEqual(list(container_deleter.mark_for_deletion(
|
|
swift,
|
|
'account',
|
|
'container',
|
|
'',
|
|
'end',
|
|
'pre',
|
|
timestamp=utils.Timestamp(ts),
|
|
yield_time=0,
|
|
)), [(5, 'obj5'), (6, 'obj6'), (6, None)])
|
|
self.assertEqual(swift.calls, [
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=&end_marker=end&prefix=pre', {}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
('GET', '/v1/account/container',
|
|
'format=json&marker=obj6&end_marker=end&prefix=pre',
|
|
{}, None),
|
|
('UPDATE', '/v1/.expiring_objects/' + ts.split('.')[0], '', {
|
|
'X-Backend-Allow-Private-Methods': 'True',
|
|
'X-Backend-Storage-Policy-Index': '0',
|
|
'X-Timestamp': ts}, mock.ANY),
|
|
])
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-3].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container',
|
|
['obj1', 'obj2', 'obj3', u'obj4-\N{SNOWMAN}', 'obj5'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
json.loads(swift.calls[-1].body),
|
|
container_deleter.make_delete_jobs(
|
|
'account', 'container', ['obj6'],
|
|
utils.Timestamp(ts)
|
|
)
|
|
)
|