Allow reconciler to handle reserved names

Change-Id: Ib918f10e95970b9f562b88e923c25608b826b83f
This commit is contained in:
Clay Gerrard 2019-12-20 11:47:13 -06:00
parent 4601548dab
commit b1178b4a96
6 changed files with 237 additions and 67 deletions

View File

@ -789,13 +789,16 @@ class InternalClient(object):
def upload_object(
self, fobj, account, container, obj, headers=None,
acceptable_statuses=(2,)):
acceptable_statuses=(2,), params=None):
"""
:param fobj: File object to read object's content from.
:param account: The object's account.
:param container: The object's container.
:param obj: The object.
:param headers: Headers to send with request, defaults to empty dict.
:param acceptable_statuses: List of acceptable statuses for request.
:param params: A dict of params to be set in request query string,
defaults to None.
:raises UnexpectedResponse: Exception raised when requests fail
to get a response with an acceptable status
@ -807,7 +810,8 @@ class InternalClient(object):
if 'Content-Length' not in headers:
headers['Transfer-Encoding'] = 'chunked'
path = self.make_path(account, container, obj)
self.make_request('PUT', path, headers, acceptable_statuses, fobj)
self.make_request('PUT', path, headers, acceptable_statuses, fobj,
params=params)
def get_auth(url, user, key, auth_version='1.0', **kwargs):

View File

@ -29,6 +29,7 @@ import six
from swift.common.header_key_dict import HeaderKeyDict
from swift import gettext_ as _
from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
from swift.common.storage_policy import POLICIES
from swift.common.exceptions import ListingIterError, SegmentError
from swift.common.http import is_success
@ -41,6 +42,7 @@ from swift.common.utils import split_path, validate_device_partition, \
parse_content_range, csv_append, list_from_csv, Spliterator, quote, \
RESERVED
from swift.common.wsgi import make_subrequest
from swift.container.reconciler import MISPLACED_OBJECTS_ACCOUNT
OBJECT_TRANSIENT_SYSMETA_PREFIX = 'x-object-transient-sysmeta-'
@ -121,7 +123,8 @@ def validate_internal_obj(account, container, obj):
if not container:
raise ValueError('Container is required')
validate_internal_container(account, container)
if obj:
if obj and not (account.startswith(AUTO_CREATE_ACCOUNT_PREFIX) or
account == MISPLACED_OBJECTS_ACCOUNT):
_validate_internal_name(obj, 'object')
if container.startswith(RESERVED) and not obj.startswith(RESERVED):
raise HTTPBadRequest(body='Invalid user-namespace object '

View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import functools
import sys
import itertools
import uuid
@ -19,10 +20,12 @@ from optparse import OptionParser
import random
import six
from six.moves.urllib.parse import urlparse
from six import StringIO
from six.moves.urllib.parse import urlparse, parse_qs, quote
from swift.common.manager import Manager
from swift.common import utils, ring
from swift.common.internal_client import InternalClient, UnexpectedResponse
from swift.common.storage_policy import POLICIES
from swift.common.http import HTTP_NOT_FOUND
@ -64,13 +67,10 @@ def command(f):
return f
@six.add_metaclass(meta_command)
class BrainSplitter(object):
def __init__(self, url, token, container_name='test', object_name='test',
server_type='container', policy=None):
self.url = url
self.token = token
self.account = utils.split_path(urlparse(url).path, 2, 2)[1]
class BaseBrain(object):
def _setup(self, account, container_name, object_name,
server_type, policy):
self.account = account
self.container_name = container_name
self.object_name = object_name
server_list = ['%s-server' % server_type] if server_type else ['all']
@ -153,36 +153,168 @@ class BrainSplitter(object):
policy = self.policy
headers = {'X-Storage-Policy': policy.name}
client.put_container(self.url, self.token, self.container_name,
headers=headers)
self.client.put_container(self.container_name, headers=headers)
@command
def delete_container(self):
"""
delete container
"""
client.delete_container(self.url, self.token, self.container_name)
self.client.delete_container(self.container_name)
@command
def put_object(self, headers=None, contents=None):
"""
issue put for test object
"""
client.put_object(self.url, self.token, self.container_name,
self.object_name, headers=headers, contents=contents)
self.client.put_object(self.container_name, self.object_name,
headers=headers, contents=contents)
@command
def delete_object(self):
"""
issue delete for test object
"""
self.client.delete_object(self.container_name, self.object_name)
@command
def get_object(self):
"""
issue GET for test object
"""
return self.client.get_object(self.container_name, self.object_name)
class PublicBrainClient(object):
def __init__(self, url, token):
self.url = url
self.token = token
self.account = utils.split_path(urlparse(url).path, 2, 2)[1]
def put_container(self, container_name, headers):
return client.put_container(self.url, self.token, container_name,
headers=headers)
def post_container(self, container_name, headers):
return client.post_container(self.url, self.token, container_name,
headers)
def delete_container(self, container_name):
return client.delete_container(self.url, self.token, container_name)
def put_object(self, container_name, object_name, headers, contents,
query_string=None):
return client.put_object(self.url, self.token, container_name,
object_name, headers=headers,
contents=contents, query_string=query_string)
def delete_object(self, container_name, object_name):
try:
client.delete_object(self.url, self.token, self.container_name,
self.object_name)
client.delete_object(self.url, self.token,
container_name, object_name)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
def head_object(self, container_name, object_name):
return client.head_object(self.url, self.token, container_name,
object_name)
def get_object(self, container_name, object_name, query_string=None):
return client.get_object(self.url, self.token,
container_name, object_name,
query_string=query_string)
def translate_client_exception(m):
@functools.wraps(m)
def wrapper(*args, **kwargs):
try:
return m(*args, **kwargs)
except UnexpectedResponse as err:
raise ClientException(
err.message,
http_scheme=err.resp.environ['wsgi.url_scheme'],
http_host=err.resp.environ['SERVER_NAME'],
http_port=err.resp.environ['SERVER_PORT'],
http_path=quote(err.resp.environ['PATH_INFO']),
http_query=err.resp.environ['QUERY_STRING'],
http_status=err.resp.status_int,
http_reason=err.resp.explanation,
http_response_content=err.resp.body,
http_response_headers=err.resp.headers,
)
return wrapper
class InternalBrainClient(object):
def __init__(self, conf_file, account='AUTH_test'):
self.swift = InternalClient(conf_file, 'probe-test', 3)
self.account = account
@translate_client_exception
def put_container(self, container_name, headers):
return self.swift.create_container(self.account, container_name,
headers=headers)
@translate_client_exception
def post_container(self, container_name, headers):
return self.swift.set_container_metadata(self.account, container_name,
headers)
@translate_client_exception
def delete_container(self, container_name):
return self.swift.delete_container(self.account, container_name)
def parse_qs(self, query_string):
if query_string is not None:
return {k: v[-1] for k, v in parse_qs(query_string).items()}
@translate_client_exception
def put_object(self, container_name, object_name, headers, contents,
query_string=None):
return self.swift.upload_object(StringIO(contents), self.account,
container_name, object_name,
headers=headers,
params=self.parse_qs(query_string))
@translate_client_exception
def delete_object(self, container_name, object_name):
return self.swift.delete_object(
self.account, container_name, object_name)
@translate_client_exception
def head_object(self, container_name, object_name):
return self.swift.get_object_metadata(
self.account, container_name, object_name)
@translate_client_exception
def get_object(self, container_name, object_name, query_string=None):
status, headers, resp_iter = self.swift.get_object(
self.account, container_name, object_name,
params=self.parse_qs(query_string))
return headers, ''.join(resp_iter)
@six.add_metaclass(meta_command)
class BrainSplitter(BaseBrain):
def __init__(self, url, token, container_name='test', object_name='test',
server_type='container', policy=None):
self.client = PublicBrainClient(url, token)
self._setup(self.client.account, container_name, object_name,
server_type, policy)
@six.add_metaclass(meta_command)
class InternalBrainSplitter(BaseBrain):
def __init__(self, conf, container_name='test', object_name='test',
server_type='container', policy=None):
self.client = InternalBrainClient(conf)
self._setup(self.client.account, container_name, object_name,
server_type, policy)
parser = OptionParser('%prog [options] '
'<command>[:<args>[,<args>...]] [<command>...]')
parser.usage += '\n\nCommands:\n\t' + \

View File

@ -24,11 +24,12 @@ from swift.common import utils, direct_client
from swift.common.storage_policy import POLICIES
from swift.common.http import HTTP_NOT_FOUND
from swift.container.reconciler import MISPLACED_OBJECTS_ACCOUNT
from test.probe.brain import BrainSplitter
from test.probe.brain import BrainSplitter, InternalBrainSplitter
from swift.common.request_helpers import get_reserved_name
from test.probe.common import (ReplProbeTest, ENABLED_POLICIES,
POLICIES_BY_TYPE, REPL_POLICY)
from swiftclient import client, ClientException
from swiftclient import ClientException
TIMEOUT = 60
@ -48,15 +49,13 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
timeout = time.time() + TIMEOUT
while time.time() < timeout:
try:
return client.get_object(self.url, self.token,
self.container_name,
self.object_name)
return self.brain.get_object()
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
time.sleep(1)
else:
self.fail('could not HEAD /%s/%s/%s/ from policy %s '
self.fail('could not GET /%s/%s/%s/ from policy %s '
'after %s seconds.' % (
self.account, self.container_name, self.object_name,
int(policy_index), TIMEOUT))
@ -238,6 +237,12 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
self.account, self.container_name, self.object_name,
orig_policy_index, node))
def get_object_name(self, name):
"""
hook for sublcass to translate object names
"""
return name
def test_reconcile_manifest(self):
if 'slo' not in self.cluster_info:
raise unittest.SkipTest(
@ -258,14 +263,14 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
def write_part(i):
body = b'VERIFY%0.2d' % i + b'\x00' * 1048576
part_name = 'manifest_part_%0.2d' % i
part_name = self.get_object_name('manifest_part_%0.2d' % i)
manifest_entry = {
"path": "/%s/%s" % (self.container_name, part_name),
"etag": md5(body).hexdigest(),
"size_bytes": len(body),
}
client.put_object(self.url, self.token, self.container_name,
part_name, contents=body)
self.brain.client.put_object(self.container_name, part_name, {},
body)
manifest_data.append(manifest_entry)
# get an old container stashed
@ -284,10 +289,10 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
# write manifest
with self.assertRaises(ClientException) as catcher:
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
self.brain.client.put_object(
self.container_name, self.object_name,
{}, utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
# so as it works out, you can't really upload a multi-part
# manifest for objects that are currently misplaced - you have to
@ -334,18 +339,18 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
self.get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
self.brain.client.post_container(self.container_name, {})
# let's see how that direct upload worked out...
metadata, body = client.get_object(
self.url, self.token, self.container_name, direct_manifest_name,
metadata, body = self.brain.client.get_object(
self.container_name, direct_manifest_name,
query_string='multipart-manifest=get')
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
for i, entry in enumerate(utils.json.loads(body)):
for key in ('hash', 'bytes', 'name'):
self.assertEqual(entry[key], direct_manifest_data[i][key])
metadata, body = client.get_object(
self.url, self.token, self.container_name, direct_manifest_name)
metadata, body = self.brain.client.get_object(
self.container_name, direct_manifest_name)
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
@ -353,13 +358,11 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
for i in range(20)))
# and regular upload should work now too
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
self.brain.client.put_object(self.container_name, self.object_name, {},
utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
metadata = self.brain.client.head_object(self.container_name,
self.object_name)
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
@ -376,66 +379,69 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
self.brain.put_container(int(policy))
self.brain.start_primary_half()
# write some target data
client.put_object(self.url, self.token, self.container_name, 'target',
contents=b'this is the target data')
target_name = self.get_object_name('target')
self.brain.client.put_object(self.container_name, target_name, {},
b'this is the target data')
# write the symlink
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
client.put_object(
self.url, self.token, self.container_name, 'symlink',
headers={
'X-Symlink-Target': '%s/target' % self.container_name,
symlink_name = self.get_object_name('symlink')
self.brain.client.put_object(
self.container_name, symlink_name, {
'X-Symlink-Target': '%s/%s' % (
self.container_name, target_name),
'Content-Type': 'application/symlink',
})
}, '')
# at this point we have a broken symlink (the container_info has the
# proxy looking for the target in the wrong policy)
with self.assertRaises(ClientException) as ctx:
client.get_object(self.url, self.token, self.container_name,
'symlink')
self.brain.client.get_object(self.container_name, symlink_name)
self.assertEqual(ctx.exception.http_status, 404)
# of course the symlink itself is fine
metadata, body = client.get_object(self.url, self.token,
self.container_name, 'symlink',
query_string='symlink=get')
metadata, body = self.brain.client.get_object(
self.container_name, symlink_name, query_string='symlink=get')
self.assertEqual(metadata['x-symlink-target'],
'%s/target' % self.container_name)
utils.quote('%s/%s' % (
self.container_name, target_name)))
self.assertEqual(metadata['content-type'], 'application/symlink')
self.assertEqual(body, b'')
# ... although in the wrong policy
object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, 'symlink')
self.account, self.container_name, symlink_name)
for node in nodes:
metadata = direct_client.direct_head_object(
node, part, self.account, self.container_name, 'symlink',
node, part, self.account, self.container_name, symlink_name,
headers={'X-Backend-Storage-Policy-Index': int(wrong_policy)})
self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'],
'%s/target' % self.container_name)
utils.quote('%s/%s' % (
self.container_name, target_name)))
# let the reconciler run
self.brain.start_handoff_half()
self.get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
self.brain.client.post_container(self.container_name, {})
# now the symlink works
metadata, body = client.get_object(self.url, self.token,
self.container_name, 'symlink')
metadata, body = self.brain.client.get_object(
self.container_name, symlink_name)
self.assertEqual(body, b'this is the target data')
# and it's in the correct policy
object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, 'symlink')
self.account, self.container_name, symlink_name)
for node in nodes:
metadata = direct_client.direct_head_object(
node, part, self.account, self.container_name, 'symlink',
node, part, self.account, self.container_name, symlink_name,
headers={'X-Backend-Storage-Policy-Index': int(policy)})
self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'],
'%s/target' % self.container_name)
utils.quote('%s/%s' % (
self.container_name, target_name)))
def test_reconciler_move_object_twice(self):
# select some policies
@ -552,5 +558,24 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
self.assertEqual('custom-meta', headers['x-object-meta-test'])
class TestReservedNamespaceMergePolicyIndex(TestContainerMergePolicyIndex):
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
def setUp(self):
super(TestReservedNamespaceMergePolicyIndex, self).setUp()
self.container_name = get_reserved_name('container', str(uuid.uuid4()))
self.object_name = get_reserved_name('object', str(uuid.uuid4()))
self.brain = InternalBrainSplitter('/etc/swift/internal-client.conf',
self.container_name,
self.object_name, 'container')
def get_object_name(self, name):
return get_reserved_name(name)
def test_reconcile_manifest(self):
raise unittest.SkipTest(
'SLO does not allow parts in the reserved namespace')
if __name__ == "__main__":
unittest.main()

View File

@ -1330,7 +1330,7 @@ class TestInternalClient(unittest.TestCase):
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
body_file=None, params=None):
self.make_request_called += 1
self.test.assertEqual(self.path, path)
exp_headers = dict(self.headers)
@ -1358,7 +1358,7 @@ class TestInternalClient(unittest.TestCase):
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
body_file=None, params=None):
self.make_request_called += 1
self.test.assertEqual(self.path, path)
exp_headers = dict(self.headers)

View File

@ -19,6 +19,7 @@ import unittest
from swift.common.swob import Request, HTTPException, HeaderKeyDict
from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY
from swift.common import request_helpers as rh
from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
from test.unit import patch_policies
from test.unit.common.test_utils import FakeResponse
@ -280,6 +281,11 @@ class TestRequestHelpers(unittest.TestCase):
'AUTH_foo', cont, 'baz')
self.assertEqual(raised.exception.args[0], 'Container is required')
def test_invalid_names_in_system_accounts(self):
self.assertIsNone(rh.validate_internal_obj(
AUTO_CREATE_ACCOUNT_PREFIX + 'system_account', 'foo',
'crazy%stown' % rh.RESERVED))
def test_invalid_reserved_names(self):
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj('AUTH_foo' + rh.RESERVED, 'bar', 'baz')