openstack/swift
Code Issues Proposed changes
b33f8ff2c5
swift/test/functional/test_symlink.py
Tim Burke 118cf2ba8a tempurl: Deprecate sha1 signatures 
We've known this would eventually be necessary for a while [1], and
way back in 2017 we started seeing SHA-1 collisions [2].

[1] https://www.schneier.com/blog/archives/2012/10/when_will_we_se.html
[2] https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html

UpgradeImpact:
==============
"sha1" has been removed from the default set of `allowed_digests` in the
tempurl middleware config. If your cluster still has clients requiring
the use of SHA-1,

- explicitly configure `allowed_digests` to include "sha1" and
- encourage your clients to move to more-secure algorithms.

Depends-On: https://review.opendev.org/c/openstack/tempest/+/832771
Change-Id: I6e6fa76671c860191a2ce921cb6caddc859b1066
Related-Change: Ia9dd1a91cc3c9c946f5f029cdefc9e66bcf01046
Closes-Bug: #1733634
2022-04-22 20:43:01 +10:00

2479 lines
101 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python
# Copyright (c) 2010-2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hmac
import unittest
import itertools
import hashlib
import six
import time
from six.moves import urllib
from uuid import uuid4
from swift.common.http import is_success
from swift.common.swob import normalize_etag
from swift.common.utils import json, MD5_OF_EMPTY_STRING, md5
from swift.common.middleware.slo import SloGetContext
from test.functional import check_response, retry, requires_acls, \
cluster_info, SkipTest
from test.functional.tests import Base, TestFileComparisonEnv, Utils, BaseEnv
from test.functional.test_slo import TestSloEnv
from test.functional.test_dlo import TestDloEnv
from test.functional.test_tempurl import TestContainerTempurlEnv, \
TestTempurlEnv
from test.functional.swift_test_client import ResponseError
import test.functional as tf
from test.unit import group_by_byte
TARGET_BODY = b'target body'
def setUpModule():
tf.setup_package()
if 'symlink' not in cluster_info:
raise SkipTest("Symlinks not enabled")
def tearDownModule():
tf.teardown_package()
class TestSymlinkEnv(BaseEnv):
link_cont = uuid4().hex
tgt_cont = uuid4().hex
tgt_obj = uuid4().hex
@classmethod
def setUp(cls):
if tf.skip or tf.skip2:
raise SkipTest
cls._create_container(cls.tgt_cont) # use_account=1
cls._create_container(cls.link_cont) # use_account=1
# container in account 2
cls._create_container(cls.link_cont, use_account=2)
cls._create_tgt_object()
@classmethod
def containers(cls):
return (cls.link_cont, cls.tgt_cont)
@classmethod
def target_content_location(cls, override_obj=None, override_account=None):
account = override_account or tf.parsed[0].path.split('/', 2)[2]
return '/v1/%s/%s/%s' % (account, cls.tgt_cont,
override_obj or cls.tgt_obj)
@classmethod
def _make_request(cls, url, token, parsed, conn, method,
container, obj='', headers=None, body=b'',
query_args=None):
headers = headers or {}
headers.update({'X-Auth-Token': token})
path = '%s/%s/%s' % (parsed.path, container, obj) if obj \
else '%s/%s' % (parsed.path, container)
if query_args:
path += '?%s' % query_args
conn.request(method, path, body, headers)
resp = check_response(conn)
# to read the buffer and keep it in the attribute, call resp.content
resp.content
return resp
@classmethod
def _create_container(cls, name, headers=None, use_account=1):
headers = headers or {}
resp = retry(cls._make_request, method='PUT', container=name,
headers=headers, use_account=use_account)
if resp.status not in (201, 202):
raise ResponseError(resp)
return name
@classmethod
def _create_tgt_object(cls, body=TARGET_BODY):
resp = retry(cls._make_request, method='PUT',
headers={'Content-Type': 'application/target'},
container=cls.tgt_cont, obj=cls.tgt_obj,
body=body)
if resp.status != 201:
raise ResponseError(resp)
# sanity: successful put response has content-length 0
cls.tgt_length = str(len(body))
cls.tgt_etag = resp.getheader('etag')
resp = retry(cls._make_request, method='GET',
container=cls.tgt_cont, obj=cls.tgt_obj)
if resp.status != 200 and resp.content != body:
raise ResponseError(resp)
@classmethod
def tearDown(cls):
delete_containers = [
(use_account, containers) for use_account, containers in
enumerate([cls.containers(), [cls.link_cont]], 1)]
# delete objects inside container
for use_account, containers in delete_containers:
if use_account == 2 and tf.skip2:
continue
for container in containers:
while True:
cont = container
resp = retry(cls._make_request, method='GET',
container=cont, query_args='format=json',
use_account=use_account)
if resp.status == 404:
break
if not is_success(resp.status):
raise ResponseError(resp)
objs = json.loads(resp.content)
if not objs:
break
for obj in objs:
resp = retry(cls._make_request, method='DELETE',
container=container, obj=obj['name'],
use_account=use_account)
if resp.status not in (204, 404):
raise ResponseError(resp)
# delete the containers
for use_account, containers in delete_containers:
if use_account == 2 and tf.skip2:
continue
for container in containers:
resp = retry(cls._make_request, method='DELETE',
container=container,
use_account=use_account)
if resp.status not in (204, 404):
raise ResponseError(resp)
class TestSymlink(Base):
env = TestSymlinkEnv
@classmethod
def setUpClass(cls):
# To skip env setup for class setup, instead setUp the env for each
# test method
pass
def setUp(self):
self.env.setUp()
def object_name_generator():
while True:
yield uuid4().hex
self.obj_name_gen = object_name_generator()
self._account_name = None
def tearDown(self):
self.env.tearDown()
@property
def account_name(self):
if not self._account_name:
self._account_name = tf.parsed[0].path.split('/', 2)[2]
return self._account_name
def _make_request(self, url, token, parsed, conn, method,
container, obj='', headers=None, body=b'',
query_args=None, allow_redirects=True):
headers = headers or {}
headers.update({'X-Auth-Token': token})
path = '%s/%s/%s' % (parsed.path, container, obj) if obj \
else '%s/%s' % (parsed.path, container)
if query_args:
path += '?%s' % query_args
conn.requests_args['allow_redirects'] = allow_redirects
conn.request(method, path, body, headers)
resp = check_response(conn)
# to read the buffer and keep it in the attribute, call resp.content
resp.content
return resp
def _make_request_with_symlink_get(self, url, token, parsed, conn, method,
container, obj, headers=None, body=b''):
resp = self._make_request(
url, token, parsed, conn, method, container, obj, headers, body,
query_args='symlink=get')
return resp
def _test_put_symlink(self, link_cont, link_obj, tgt_cont, tgt_obj):
headers = {'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj)}
resp = retry(self._make_request, method='PUT',
container=link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 201)
def _test_put_symlink_with_etag(self, link_cont, link_obj, tgt_cont,
tgt_obj, etag, headers=None):
headers = headers or {}
headers.update({'X-Symlink-Target': '%s/%s' % (tgt_cont, tgt_obj),
'X-Symlink-Target-Etag': etag})
resp = retry(self._make_request, method='PUT',
container=link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 201, resp.content)
def _test_get_as_target_object(
self, link_cont, link_obj, expected_content_location,
use_account=1):
resp = retry(
self._make_request, method='GET',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200, resp.content)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'),
str(self.env.tgt_length))
self.assertEqual(resp.getheader('etag'), self.env.tgt_etag)
self.assertIn('Content-Location', resp.headers)
self.assertEqual(expected_content_location,
resp.getheader('content-location'))
return resp
def _test_head_as_target_object(self, link_cont, link_obj, use_account=1):
resp = retry(
self._make_request, method='HEAD',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200)
def _assertLinkObject(self, link_cont, link_obj, use_account=1):
# HEAD on link_obj itself
resp = retry(
self._make_request_with_symlink_get, method='HEAD',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200)
self.assertTrue(resp.getheader('x-symlink-target'))
# GET on link_obj itself
resp = retry(
self._make_request_with_symlink_get, method='GET',
container=link_cont, obj=link_obj, use_account=use_account)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, b'')
self.assertEqual(resp.getheader('content-length'), str(0))
self.assertTrue(resp.getheader('x-symlink-target'))
def _assertSymlink(self, link_cont, link_obj,
expected_content_location=None, use_account=1):
expected_content_location = \
expected_content_location or self.env.target_content_location()
# sanity: HEAD/GET on link_obj
self._assertLinkObject(link_cont, link_obj, use_account)
# HEAD target object via symlink
self._test_head_as_target_object(
link_cont=link_cont, link_obj=link_obj, use_account=use_account)
# GET target object via symlink
self._test_get_as_target_object(
link_cont=link_cont, link_obj=link_obj, use_account=use_account,
expected_content_location=expected_content_location)
def test_symlink_with_encoded_target_name(self):
# makes sure to test encoded characters as symlink target
target_obj = 'dealde%2Fl04 011e%204c8df/flash.png'
link_obj = uuid4().hex
# create target using unnormalized path
resp = retry(
self._make_request, method='PUT', container=self.env.tgt_cont,
obj=target_obj, body=TARGET_BODY)
self.assertEqual(resp.status, 201)
# you can get it using either name
resp = retry(
self._make_request, method='GET', container=self.env.tgt_cont,
obj=target_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
normalized_quoted_obj = 'dealde/l04%20011e%204c8df/flash.png'
self.assertEqual(normalized_quoted_obj, urllib.parse.quote(
urllib.parse.unquote(target_obj)))
resp = retry(
self._make_request, method='GET', container=self.env.tgt_cont,
obj=normalized_quoted_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
# create a symlink using the un-normalized target path
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=target_obj)
# and it's normalized
self._assertSymlink(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(
normalized_quoted_obj))
# create a symlink using the normalized target path
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=normalized_quoted_obj)
# and it's ALSO normalized
self._assertSymlink(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(
normalized_quoted_obj))
def test_symlink_put_head_get(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
self._assertSymlink(self.env.link_cont, link_obj)
def test_symlink_with_etag_put_head_get(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
self._assertSymlink(self.env.link_cont, link_obj)
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': self.env.tgt_etag})
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': 'not-the-etag'})
self.assertEqual(resp.status, 412)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_static_symlink_with_bad_etag_put_head_get(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# overwrite tgt object
self.env._create_tgt_object(body='updated target body')
resp = retry(
self._make_request, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
# but we still know where it points
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# uses a mechanism entirely divorced from if-match
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': self.env.tgt_etag})
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers={'If-Match': 'not-the-etag'})
self.assertEqual(resp.status, 409)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
resp = retry(
self._make_request, method='DELETE',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
# not-found-ness trumps if-match-ness
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 404)
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_dynamic_link_to_static_link(self):
static_link_obj = uuid4().hex
# PUT static_link to tgt_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
symlink_obj = uuid4().hex
# PUT symlink to static_link
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.link_cont,
tgt_obj=static_link_obj)
self._test_get_as_target_object(
link_cont=self.env.link_cont, link_obj=symlink_obj,
expected_content_location=self.env.target_content_location())
def test_static_link_to_dynamic_link(self):
symlink_obj = uuid4().hex
# PUT symlink to tgt_obj
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
static_link_obj = uuid4().hex
# PUT a static_link to the symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
self._test_get_as_target_object(
link_cont=self.env.link_cont, link_obj=static_link_obj,
expected_content_location=self.env.target_content_location())
def test_static_link_to_nowhere(self):
missing_obj = uuid4().hex
static_link_obj = uuid4().hex
# PUT a static_link to the missing name
headers = {
'X-Symlink-Target': '%s/%s' % (self.env.link_cont, missing_obj),
'X-Symlink-Target-Etag': MD5_OF_EMPTY_STRING}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=static_link_obj,
headers=headers)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content, b'X-Symlink-Target does not exist')
def test_static_link_to_broken_symlink(self):
symlink_obj = uuid4().hex
# PUT symlink to tgt_obj
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=symlink_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
static_link_obj = uuid4().hex
# PUT a static_link to the symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
# break the symlink
resp = retry(
self._make_request, method='DELETE',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
self.assertEqual(resp.status // 100, 2)
# sanity
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=symlink_obj)
self.assertEqual(resp.status, 404)
# static_link is broken too!
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=static_link_obj)
self.assertEqual(resp.status, 404)
# interestingly you may create a static_link to a broken symlink
broken_static_link_obj = uuid4().hex
# PUT a static_link to the broken symlink
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=broken_static_link_obj,
tgt_cont=self.env.link_cont,
tgt_obj=symlink_obj,
etag=MD5_OF_EMPTY_STRING)
def test_symlink_get_ranged(self):
link_obj = uuid4().hex
# PUT symlink
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
headers = {'Range': 'bytes=7-10'}
resp = retry(self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 206)
self.assertEqual(resp.content, b'body')
def test_create_symlink_before_target(self):
link_obj = uuid4().hex
target_obj = uuid4().hex
# PUT link_obj before target object is written
# PUT, GET, HEAD (on symlink) should all work ok without target object
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont, tgt_obj=target_obj)
# Try to GET target via symlink.
# 404 will be returned with Content-Location of target path.
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=1)
self.assertEqual(resp.status, 404)
self.assertIn('Content-Location', resp.headers)
self.assertEqual(self.env.target_content_location(target_obj),
resp.getheader('content-location'))
# HEAD on target object via symlink should return a 404 since target
# object has not yet been written
resp = retry(
self._make_request, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 404)
# GET on target object directly
resp = retry(
self._make_request, method='GET',
container=self.env.tgt_cont, obj=target_obj)
self.assertEqual(resp.status, 404)
# Now let's write target object and symlink will be able to return
# object
resp = retry(
self._make_request, method='PUT', container=self.env.tgt_cont,
obj=target_obj, body=TARGET_BODY)
self.assertEqual(resp.status, 201)
# successful put response has content-length 0
target_length = str(len(TARGET_BODY))
target_etag = resp.getheader('etag')
# sanity: HEAD/GET on link_obj itself
self._assertLinkObject(self.env.link_cont, link_obj)
# HEAD target object via symlink
self._test_head_as_target_object(
link_cont=self.env.link_cont, link_obj=link_obj)
# GET target object via symlink
resp = retry(self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'), str(target_length))
self.assertEqual(resp.getheader('etag'), target_etag)
self.assertIn('Content-Location', resp.headers)
self.assertEqual(self.env.target_content_location(target_obj),
resp.getheader('content-location'))
def test_symlink_chain(self):
# Testing to symlink chain like symlink -> symlink -> target.
symloop_max = cluster_info['symlink']['symloop_max']
# create symlink chain in a container. To simplify,
# use target container for all objects (symlinks and target) here
previous = self.env.tgt_obj
container = self.env.tgt_cont
for link_obj in itertools.islice(self.obj_name_gen, symloop_max):
# PUT link_obj point to tgt_obj
self._test_put_symlink(
link_cont=container, link_obj=link_obj,
tgt_cont=container, tgt_obj=previous)
# set corrent link_obj to previous
previous = link_obj
# the last link is valid for symloop_max constraint
max_chain_link = link_obj
self._assertSymlink(link_cont=container, link_obj=max_chain_link)
# PUT a new link_obj points to the max_chain_link
# that will result in 409 error on the HEAD/GET.
too_many_chain_link = next(self.obj_name_gen)
self._test_put_symlink(
link_cont=container, link_obj=too_many_chain_link,
tgt_cont=container, tgt_obj=max_chain_link)
# try to HEAD to target object via too_many_chain_link
resp = retry(self._make_request, method='HEAD',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content, b'')
# try to GET to target object via too_many_chain_link
resp = retry(self._make_request, method='GET',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(
resp.content,
b'Too many levels of symbolic links, maximum allowed is %d' %
symloop_max)
# However, HEAD/GET to the (just) link is still ok
self._assertLinkObject(container, too_many_chain_link)
def test_symlink_chain_with_etag(self):
# Testing to symlink chain like symlink -> symlink -> target.
symloop_max = cluster_info['symlink']['symloop_max']
# create symlink chain in a container. To simplify,
# use target container for all objects (symlinks and target) here
previous = self.env.tgt_obj
container = self.env.tgt_cont
for link_obj in itertools.islice(self.obj_name_gen, symloop_max):
# PUT link_obj point to tgt_obj
self._test_put_symlink_with_etag(link_cont=container,
link_obj=link_obj,
tgt_cont=container,
tgt_obj=previous,
etag=self.env.tgt_etag)
# set current link_obj to previous
previous = link_obj
# the last link is valid for symloop_max constraint
max_chain_link = link_obj
self._assertSymlink(link_cont=container, link_obj=max_chain_link)
# chained etag validation works as long as the target symlink works
headers = {'X-Symlink-Target': '%s/%s' % (container, max_chain_link),
'X-Symlink-Target-Etag': 'not-the-real-etag'}
resp = retry(self._make_request, method='PUT',
container=container, obj=uuid4().hex,
headers=headers)
self.assertEqual(resp.status, 409)
# PUT a new link_obj pointing to the max_chain_link can validate the
# ETag but will result in 409 error on the HEAD/GET.
too_many_chain_link = next(self.obj_name_gen)
self._test_put_symlink_with_etag(
link_cont=container, link_obj=too_many_chain_link,
tgt_cont=container, tgt_obj=max_chain_link,
etag=self.env.tgt_etag)
# try to HEAD to target object via too_many_chain_link
resp = retry(self._make_request, method='HEAD',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content, b'')
# try to GET to target object via too_many_chain_link
resp = retry(self._make_request, method='GET',
container=container,
obj=too_many_chain_link)
self.assertEqual(resp.status, 409)
self.assertEqual(
resp.content,
b'Too many levels of symbolic links, maximum allowed is %d' %
symloop_max)
# However, HEAD/GET to the (just) link is still ok
self._assertLinkObject(container, too_many_chain_link)
def test_symlink_and_slo_manifest_chain(self):
if 'slo' not in cluster_info:
raise SkipTest
symloop_max = cluster_info['symlink']['symloop_max']
# create symlink chain in a container. To simplify,
# use target container for all objects (symlinks and target) here
previous = self.env.tgt_obj
container = self.env.tgt_cont
# make symlink and slo manifest chain
# e.g. slo -> symlink -> symlink -> slo -> symlink -> symlink -> target
for _ in range(SloGetContext.max_slo_recursion_depth or 1):
for link_obj in itertools.islice(self.obj_name_gen, symloop_max):
# PUT link_obj point to previous object
self._test_put_symlink(
link_cont=container, link_obj=link_obj,
tgt_cont=container, tgt_obj=previous)
# next link will point to this link
previous = link_obj
else:
# PUT a manifest with single segment to the symlink
manifest_obj = next(self.obj_name_gen)
manifest = json.dumps(
[{'path': '/%s/%s' % (container, link_obj)}])
resp = retry(self._make_request, method='PUT',
container=container, obj=manifest_obj,
body=manifest,
query_args='multipart-manifest=put')
self.assertEqual(resp.status, 201) # sanity
previous = manifest_obj
# From the last manifest to the final target obj length is
# symloop_max * max_slo_recursion_depth
max_recursion_manifest = previous
# Check GET to max_recursion_manifest returns valid target object
resp = retry(
self._make_request, method='GET', container=container,
obj=max_recursion_manifest)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'),
str(self.env.tgt_length))
# N.B. since the last manifest is slo so it will remove
# content-location info from the response header
self.assertNotIn('Content-Location', resp.headers)
# sanity: one more link to the slo can work still
one_more_link = next(self.obj_name_gen)
self._test_put_symlink(
link_cont=container, link_obj=one_more_link,
tgt_cont=container, tgt_obj=max_recursion_manifest)
resp = retry(
self._make_request, method='GET', container=container,
obj=one_more_link)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
self.assertEqual(resp.getheader('content-length'),
str(self.env.tgt_length))
self.assertIn('Content-Location', resp.headers)
self.assertIn('%s/%s' % (container, max_recursion_manifest),
resp.getheader('content-location'))
# PUT a new slo manifest point to the max_recursion_manifest
# Symlink and slo manifest chain from the new manifest to the final
# target has (max_slo_recursion_depth + 1) manifests.
too_many_recursion_manifest = next(self.obj_name_gen)
manifest = json.dumps(
[{'path': '/%s/%s' % (container, max_recursion_manifest)}])
resp = retry(self._make_request, method='PUT',
container=container, obj=too_many_recursion_manifest,
body=manifest.encode('ascii'),
query_args='multipart-manifest=put')
self.assertEqual(resp.status, 201) # sanity
# Check GET to too_many_recursion_mani returns 409 error
resp = retry(self._make_request, method='GET',
container=container, obj=too_many_recursion_manifest)
self.assertEqual(resp.status, 409)
# N.B. This error message is from slo middleware that uses default.
self.assertEqual(
resp.content,
b'<html><h1>Conflict</h1><p>There was a conflict when trying to'
b' complete your request.</p></html>')
def test_symlink_put_missing_target_container(self):
link_obj = uuid4().hex
# set only object, no container in the prefix
headers = {'X-Symlink-Target': self.env.tgt_obj}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 412)
self.assertEqual(resp.content,
b'X-Symlink-Target header must be of the form'
b' <container name>/<object name>')
def test_symlink_put_non_zero_length(self):
link_obj = uuid4().hex
headers = {'X-Symlink-Target':
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj)}
resp = retry(
self._make_request, method='PUT', container=self.env.link_cont,
obj=link_obj, body=b'non-zero-length', headers=headers)
self.assertEqual(resp.status, 400)
self.assertEqual(resp.content,
b'Symlink requests require a zero byte body')
def test_symlink_target_itself(self):
link_obj = uuid4().hex
headers = {
'X-Symlink-Target': '%s/%s' % (self.env.link_cont, link_obj)}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 400)
self.assertEqual(resp.content, b'Symlink cannot target itself')
def test_symlink_target_each_other(self):
symloop_max = cluster_info['symlink']['symloop_max']
link_obj1 = uuid4().hex
link_obj2 = uuid4().hex
# PUT two links which targets each other
self._test_put_symlink(
link_cont=self.env.link_cont, link_obj=link_obj1,
tgt_cont=self.env.link_cont, tgt_obj=link_obj2)
self._test_put_symlink(
link_cont=self.env.link_cont, link_obj=link_obj2,
tgt_cont=self.env.link_cont, tgt_obj=link_obj1)
for obj in (link_obj1, link_obj2):
# sanity: HEAD/GET on the link itself is ok
self._assertLinkObject(self.env.link_cont, obj)
for obj in (link_obj1, link_obj2):
resp = retry(self._make_request, method='HEAD',
container=self.env.link_cont, obj=obj)
self.assertEqual(resp.status, 409)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont, obj=obj)
self.assertEqual(resp.status, 409)
self.assertEqual(
resp.content,
b'Too many levels of symbolic links, maximum allowed is %d' %
symloop_max)
def test_symlink_put_copy_from(self):
link_obj1 = uuid4().hex
link_obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_src = '%s/%s' % (self.env.link_cont, link_obj1)
# copy symlink
headers = {'X-Copy-From': copy_src}
resp = retry(self._make_request_with_symlink_get,
method='PUT',
container=self.env.link_cont, obj=link_obj2,
headers=headers)
self.assertEqual(resp.status, 201)
self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2)
@requires_acls
def test_symlink_put_copy_from_cross_account(self):
link_obj1 = uuid4().hex
link_obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_src = '%s/%s' % (self.env.link_cont, link_obj1)
perm_two = tf.swift_test_perm[1]
# add X-Content-Read to account 1 link_cont and tgt_cont
# permit account 2 to read account 1 link_cont to perform copy_src
# and tgt_cont so that link_obj2 can refer to tgt_object
# this ACL allows the copy to succeed
headers = {'X-Container-Read': perm_two}
resp = retry(
self._make_request, method='POST',
container=self.env.link_cont, headers=headers)
self.assertEqual(resp.status, 204)
# this ACL allows link_obj in account 2 to target object in account 1
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=headers)
self.assertEqual(resp.status, 204)
# copy symlink itself to a different account w/o
# X-Symlink-Target-Account. This operation will result in copying
# symlink to the account 2 container that points to the
# container/object in the account 2.
# (the container/object is not prepared)
headers = {'X-Copy-From-Account': self.account_name,
'X-Copy-From': copy_src}
resp = retry(self._make_request_with_symlink_get, method='PUT',
container=self.env.link_cont, obj=link_obj2,
headers=headers, use_account=2)
self.assertEqual(resp.status, 201)
# sanity: HEAD/GET on link_obj itself
self._assertLinkObject(self.env.link_cont, link_obj2, use_account=2)
account_two = tf.parsed[1].path.split('/', 2)[2]
# no target object in the account 2
for method in ('HEAD', 'GET'):
resp = retry(
self._make_request, method=method,
container=self.env.link_cont, obj=link_obj2, use_account=2)
self.assertEqual(resp.status, 404)
self.assertIn('content-location', resp.headers)
self.assertEqual(
self.env.target_content_location(override_account=account_two),
resp.getheader('content-location'))
# copy symlink itself to a different account with target account
# the target path will be in account 1
# the target path will have an object
headers = {'X-Symlink-target-Account': self.account_name,
'X-Copy-From-Account': self.account_name,
'X-Copy-From': copy_src}
resp = retry(
self._make_request_with_symlink_get, method='PUT',
container=self.env.link_cont, obj=link_obj2,
headers=headers, use_account=2)
self.assertEqual(resp.status, 201)
self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2,
use_account=2)
def test_symlink_copy_from_target(self):
link_obj1 = uuid4().hex
obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_src = '%s/%s' % (self.env.link_cont, link_obj1)
# issuing a COPY request to a symlink w/o symlink=get, should copy
# the target object, not the symlink itself
headers = {'X-Copy-From': copy_src}
resp = retry(self._make_request, method='PUT',
container=self.env.tgt_cont, obj=obj2,
headers=headers)
self.assertEqual(resp.status, 201)
# HEAD to the copied object
resp = retry(self._make_request, method='HEAD',
container=self.env.tgt_cont, obj=obj2)
self.assertEqual(200, resp.status)
self.assertNotIn('Content-Location', resp.headers)
# GET to the copied object
resp = retry(self._make_request, method='GET',
container=self.env.tgt_cont, obj=obj2)
# But... this is a raw object (not a symlink)
self.assertEqual(200, resp.status)
self.assertNotIn('Content-Location', resp.headers)
self.assertEqual(TARGET_BODY, resp.content)
def test_symlink_copy(self):
link_obj1 = uuid4().hex
link_obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_dst = '%s/%s' % (self.env.link_cont, link_obj2)
# copy symlink
headers = {'Destination': copy_dst}
resp = retry(
self._make_request_with_symlink_get, method='COPY',
container=self.env.link_cont, obj=link_obj1, headers=headers)
self.assertEqual(resp.status, 201)
self._assertSymlink(link_cont=self.env.link_cont, link_obj=link_obj2)
def test_symlink_copy_target(self):
link_obj1 = uuid4().hex
obj2 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj1,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
copy_dst = '%s/%s' % (self.env.tgt_cont, obj2)
# copy target object
headers = {'Destination': copy_dst}
resp = retry(self._make_request, method='COPY',
container=self.env.link_cont, obj=link_obj1,
headers=headers)
self.assertEqual(resp.status, 201)
# HEAD to target object via symlink
resp = retry(self._make_request, method='HEAD',
container=self.env.tgt_cont, obj=obj2)
self.assertEqual(resp.status, 200)
self.assertNotIn('Content-Location', resp.headers)
# GET to the copied object that should be a raw object (not symlink)
resp = retry(self._make_request, method='GET',
container=self.env.tgt_cont, obj=obj2)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
self.assertNotIn('Content-Location', resp.headers)
def test_post_symlink(self):
link_obj = uuid4().hex
value1 = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
# POSTing to a symlink is not allowed and should return a 307
headers = {'X-Object-Meta-Alpha': 'apple'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
# we are using account 0 in this test
expected_location_hdr = "%s/%s/%s" % (
tf.parsed[0].path, self.env.tgt_cont, self.env.tgt_obj)
self.assertEqual(resp.getheader('Location'), expected_location_hdr)
# Read header from symlink itself. The metadata is applied to symlink
resp = retry(self._make_request_with_symlink_get, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('X-Object-Meta-Alpha'), 'apple')
# Post the target object directly
headers = {'x-object-meta-test': value1}
resp = retry(
self._make_request, method='POST', container=self.env.tgt_cont,
obj=self.env.tgt_obj, headers=headers)
self.assertEqual(resp.status, 202)
resp = retry(self._make_request, method='GET',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('X-Object-Meta-Test'), value1)
# Read header from target object via symlink, should exist now.
resp = retry(
self._make_request, method='GET', container=self.env.link_cont,
obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.getheader('X-Object-Meta-Test'), value1)
# sanity: no X-Object-Meta-Alpha exists in the response header
self.assertNotIn('X-Object-Meta-Alpha', resp.headers)
def test_post_to_broken_dynamic_symlink(self):
# create a symlink to nowhere
link_obj = '%s-the-link' % uuid4().hex
tgt_obj = '%s-no-where' % uuid4().hex
headers = {'X-Symlink-Target': '%s/%s' % (self.env.tgt_cont, tgt_obj)}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 201)
# it's a real link!
self._assertLinkObject(self.env.link_cont, link_obj)
# ... it's just broken
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 404)
target_path = '/v1/%s/%s/%s' % (
self.account_name, self.env.tgt_cont, tgt_obj)
self.assertEqual(target_path, resp.headers['Content-Location'])
# we'll redirect with the Location header to the (invalid) target
headers = {'X-Object-Meta-Alpha': 'apple'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
self.assertEqual(target_path, resp.headers['Location'])
# and of course metadata *is* applied to the link
resp = retry(
self._make_request_with_symlink_get, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
self.assertTrue(resp.getheader('X-Object-Meta-Alpha'), 'apple')
def test_post_to_broken_static_symlink(self):
link_obj = uuid4().hex
# PUT link_obj
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# overwrite tgt object
old_tgt_etag = normalize_etag(self.env.tgt_etag)
self.env._create_tgt_object(body='updated target body')
# sanity
resp = retry(
self._make_request, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 409)
# but POST will still 307
headers = {'X-Object-Meta-Alpha': 'apple'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
target_path = '/v1/%s/%s/%s' % (
self.account_name, self.env.tgt_cont, self.env.tgt_obj)
self.assertEqual(target_path, resp.headers['Location'])
# but we give you the Etag just like... FYI?
self.assertEqual(old_tgt_etag, resp.headers['X-Symlink-Target-Etag'])
def test_post_with_symlink_header(self):
# POSTing to a symlink is not allowed and should return a 307
# updating the symlink target with a POST should always fail
headers = {'X-Symlink-Target': 'container/new_target'}
resp = retry(
self._make_request, method='POST', container=self.env.tgt_cont,
obj=self.env.tgt_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 400)
self.assertEqual(resp.content,
b'A PUT request is required to set a symlink target')
def test_overwrite_symlink(self):
link_obj = uuid4().hex
new_tgt_obj = "new_target_object_name"
new_tgt = '%s/%s' % (self.env.tgt_cont, new_tgt_obj)
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
# sanity
self._assertSymlink(self.env.link_cont, link_obj)
# Overwrite symlink with PUT
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=new_tgt_obj)
# head symlink to check X-Symlink-Target header
resp = retry(self._make_request_with_symlink_get, method='HEAD',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 200)
# target should remain with old target
self.assertEqual(resp.getheader('X-Symlink-Target'), new_tgt)
def test_delete_symlink(self):
link_obj = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
resp = retry(self._make_request, method='DELETE',
container=self.env.link_cont, obj=link_obj)
self.assertEqual(resp.status, 204)
# make sure target object was not deleted and is still reachable
resp = retry(self._make_request, method='GET',
container=self.env.tgt_cont, obj=self.env.tgt_obj)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.content, TARGET_BODY)
@requires_acls
def test_symlink_put_target_account(self):
if tf.skip or tf.skip2:
raise SkipTest
link_obj = uuid4().hex
# create symlink in account 2
# pointing to account 1
headers = {'X-Symlink-Target-Account': self.account_name,
'X-Symlink-Target':
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj)}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers, use_account=2)
self.assertEqual(resp.status, 201)
perm_two = tf.swift_test_perm[1]
# sanity test:
# it should be ok to get the symlink itself, but not the target object
# because the read acl has not been configured yet
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 403)
# still know where it's pointing
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# add X-Content-Read to account 1 tgt_cont
# permit account 2 to read account 1 tgt_cont
# add acl to allow reading from source
headers = {'X-Container-Read': perm_two}
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=headers)
self.assertEqual(resp.status, 204)
# GET on link_obj itself
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
# GET to target object via symlink
resp = self._test_get_as_target_object(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(),
use_account=2)
@requires_acls
def test_symlink_with_etag_put_target_account(self):
if tf.skip or tf.skip2:
raise SkipTest
link_obj = uuid4().hex
# try to create a symlink in account 2 pointing to account 1
symlink_headers = {
'X-Symlink-Target-Account': self.account_name,
'X-Symlink-Target':
'%s/%s' % (self.env.tgt_cont, self.env.tgt_obj),
'X-Symlink-Target-Etag': self.env.tgt_etag}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=symlink_headers, use_account=2)
# since we don't have read access to verify the object we get the
# permissions error
self.assertEqual(resp.status, 403)
perm_two = tf.swift_test_perm[1]
# add X-Content-Read to account 1 tgt_cont
# permit account 2 to read account 1 tgt_cont
# add acl to allow reading from source
acl_headers = {'X-Container-Read': perm_two}
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=acl_headers)
self.assertEqual(resp.status, 204)
# now we can create the symlink
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=symlink_headers, use_account=2)
self.assertEqual(resp.status, 201)
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
# GET to target object via symlink
resp = self._test_get_as_target_object(
self.env.link_cont, link_obj,
expected_content_location=self.env.target_content_location(),
use_account=2)
# Overwrite target
resp = retry(self._make_request, method='PUT',
container=self.env.tgt_cont, obj=self.env.tgt_obj,
body='some other content')
self.assertEqual(resp.status, 201)
# link is now broken
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 409)
# but we still know where it points
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
# sanity test, remove permissions
headers = {'X-Remove-Container-Read': 'remove'}
resp = retry(self._make_request, method='POST',
container=self.env.tgt_cont, headers=headers)
self.assertEqual(resp.status, 204)
# it should be ok to get the symlink itself, but not the target object
# because the read acl has been revoked
self._assertLinkObject(self.env.link_cont, link_obj, use_account=2)
resp = retry(
self._make_request, method='GET',
container=self.env.link_cont, obj=link_obj, use_account=2)
self.assertEqual(resp.status, 403)
# Still know where it is, though
self.assertEqual(resp.getheader('content-location'),
self.env.target_content_location())
def test_symlink_invalid_etag(self):
link_obj = uuid4().hex
headers = {'X-Symlink-Target': '%s/%s' % (self.env.tgt_cont,
self.env.tgt_obj),
'X-Symlink-Target-Etag': 'not-the-real-etag'}
resp = retry(self._make_request, method='PUT',
container=self.env.link_cont, obj=link_obj,
headers=headers)
self.assertEqual(resp.status, 409)
self.assertEqual(resp.content,
b"Object Etag 'ab706c400731332bffa67ed4bc15dcac' "
b"does not match X-Symlink-Target-Etag header "
b"'not-the-real-etag'")
def test_symlink_object_listing(self):
link_obj = uuid4().hex
self._test_put_symlink(link_cont=self.env.link_cont, link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj)
# sanity
self._assertSymlink(self.env.link_cont, link_obj)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont,
query_args='format=json')
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
obj_info = object_list[0]
self.assertIn('symlink_path', obj_info)
self.assertEqual(self.env.target_content_location(),
obj_info['symlink_path'])
self.assertNotIn('symlink_etag', obj_info)
def test_static_link_object_listing(self):
link_obj = uuid4().hex
self._test_put_symlink_with_etag(link_cont=self.env.link_cont,
link_obj=link_obj,
tgt_cont=self.env.tgt_cont,
tgt_obj=self.env.tgt_obj,
etag=self.env.tgt_etag)
# sanity
self._assertSymlink(self.env.link_cont, link_obj)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont,
query_args='format=json')
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
self.assertIn('symlink_path', object_list[0])
self.assertEqual(self.env.target_content_location(),
object_list[0]['symlink_path'])
obj_info = object_list[0]
self.assertIn('symlink_etag', obj_info)
self.assertEqual(normalize_etag(self.env.tgt_etag),
obj_info['symlink_etag'])
self.assertEqual(int(self.env.tgt_length),
obj_info['symlink_bytes'])
self.assertEqual(obj_info['content_type'], 'application/target')
# POSTing to a static_link can change the listing Content-Type
headers = {'Content-Type': 'application/foo'}
resp = retry(
self._make_request, method='POST', container=self.env.link_cont,
obj=link_obj, headers=headers, allow_redirects=False)
self.assertEqual(resp.status, 307)
resp = retry(self._make_request, method='GET',
container=self.env.link_cont,
query_args='format=json')
self.assertEqual(resp.status, 200)
object_list = json.loads(resp.content)
self.assertEqual(len(object_list), 1)
obj_info = object_list[0]
self.assertEqual(obj_info['content_type'], 'application/foo')
class TestCrossPolicySymlinkEnv(TestSymlinkEnv):
multiple_policies_enabled = None
@classmethod
def setUp(cls):
if tf.skip or tf.skip2:
raise SkipTest
if cls.multiple_policies_enabled is None:
try:
cls.policies = tf.FunctionalStoragePolicyCollection.from_info()
except AssertionError:
pass
if cls.policies and len(cls.policies) > 1:
cls.multiple_policies_enabled = True
else:
cls.multiple_policies_enabled = False
return
link_policy = cls.policies.select()
tgt_policy = cls.policies.exclude(name=link_policy['name']).select()
link_header = {'X-Storage-Policy': link_policy['name']}
tgt_header = {'X-Storage-Policy': tgt_policy['name']}
cls._create_container(cls.link_cont, headers=link_header)
cls._create_container(cls.tgt_cont, headers=tgt_header)
# container in account 2
cls._create_container(cls.link_cont, headers=link_header,
use_account=2)
cls._create_tgt_object()
class TestCrossPolicySymlink(TestSymlink):
env = TestCrossPolicySymlinkEnv
def setUp(self):
super(TestCrossPolicySymlink, self).setUp()
if self.env.multiple_policies_enabled is False:
raise SkipTest('Cross policy test requires multiple policies')
elif self.env.multiple_policies_enabled is not True:
# just some sanity checking
raise Exception("Expected multiple_policies_enabled "
"to be True/False, got %r" % (
self.env.multiple_policies_enabled,))
def tearDown(self):
self.env.tearDown()
class TestSymlinkSlo(Base):
"""
Just some sanity testing of SLO + symlinks.
It is basically a copy of SLO tests in test_slo, but the tested object is
a symlink to the manifest (instead of the manifest itself)
"""
env = TestSloEnv
def setUp(self):
super(TestSymlinkSlo, self).setUp()
if self.env.slo_enabled is False:
raise SkipTest("SLO not enabled")
elif self.env.slo_enabled is not True:
# just some sanity checking
raise Exception(
"Expected slo_enabled to be True/False, got %r" %
(self.env.slo_enabled,))
self.file_symlink = self.env.container.file(uuid4().hex)
self.account_name = self.env.container.conn.storage_path.rsplit(
'/', 1)[-1]
def test_symlink_target_slo_manifest(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde')})
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(self.file_symlink.read()))
manifest_body = self.file_symlink.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == self.file_symlink.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.assertEqual(obj_info, {
'name': self.file_symlink.name,
'content_type': 'application/octet-stream',
'hash': 'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'symlink_path': '/v1/%s/%s/manifest-abcde' % (
self.account_name, self.env.container.name),
})
def test_static_link_target_slo_manifest(self):
manifest_info = self.env.container2.file(
"manifest-abcde").info(parms={
'multipart-manifest': 'get'})
manifest_etag = manifest_info['etag']
self.file_symlink.write(hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': manifest_etag,
})
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(self.file_symlink.read()))
manifest_body = self.file_symlink.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
# check listing
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == self.file_symlink.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.maxDiff = None
slo_info = self.env.container2.file("manifest-abcde").info()
self.assertEqual(obj_info, {
'name': self.file_symlink.name,
'content_type': 'application/octet-stream',
'hash': u'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'slo_etag': slo_info['etag'],
'symlink_path': '/v1/%s/%s/manifest-abcde' % (
self.account_name, self.env.container2.name),
'symlink_bytes': 4 * 2 ** 20 + 1,
'symlink_etag': normalize_etag(manifest_etag),
})
def test_static_link_target_slo_manifest_wrong_etag(self):
# try the slo "etag"
slo_etag = self.env.container2.file(
"manifest-abcde").info()['etag']
self.assertRaises(ResponseError, self.file_symlink.write, hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': slo_etag,
})
self.assert_status(409) # quotes OK, but doesn't match
# try the slo etag w/o the quotes
slo_etag = slo_etag.strip('"')
self.assertRaises(ResponseError, self.file_symlink.write, hdrs={
'X-Symlink-Target': '%s/%s' % (
self.env.container2.name, 'manifest-abcde'),
'X-Symlink-Target-Etag': slo_etag,
})
self.assert_status(409) # that still doesn't match
def test_static_link_target_symlink_to_slo_manifest(self):
# write symlink
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde')})
# write static_link
file_static_link = self.env.container.file(uuid4().hex)
file_static_link.write(hdrs={
'X-Symlink-Target': '%s/%s' % (
self.file_symlink.container, self.file_symlink.name),
'X-Symlink-Target-Etag': MD5_OF_EMPTY_STRING,
})
# validate reads
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(file_static_link.read()))
manifest_body = file_static_link.read(parms={
'multipart-manifest': 'get'})
self.assertEqual(
[seg['hash'] for seg in json.loads(manifest_body)],
[self.env.seg_info['seg_%s' % c]['etag'] for c in 'abcde'])
# check listing
for obj_info in self.env.container.files(parms={'format': 'json'}):
if obj_info['name'] == file_static_link.name:
break
else:
self.fail('Unable to find file_symlink in listing.')
obj_info.pop('last_modified')
self.maxDiff = None
self.assertEqual(obj_info, {
'name': file_static_link.name,
'content_type': 'application/octet-stream',
'hash': 'd41d8cd98f00b204e9800998ecf8427e',
'bytes': 0,
'symlink_path': u'/v1/%s/%s/%s' % (
self.account_name, self.file_symlink.container,
self.file_symlink.name),
# the only time bytes/etag aren't the target object are when they
# validate through another static_link
'symlink_bytes': 0,
'symlink_etag': MD5_OF_EMPTY_STRING,
})
def test_symlink_target_slo_nested_manifest(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde-submanifest')})
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
(b'c', 1024 * 1024),
(b'd', 1024 * 1024),
(b'e', 1),
], group_by_byte(self.file_symlink.read()))
def test_slo_get_ranged_manifest(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'ranged-manifest')})
self.assertEqual([
(b'c', 1),
(b'd', 1024 * 1024),
(b'e', 1),
(b'a', 512 * 1024),
(b'b', 512 * 1024),
(b'c', 1),
(b'd', 1),
], group_by_byte(self.file_symlink.read()))
def test_slo_ranged_get(self):
self.file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manifest-abcde')})
file_contents = self.file_symlink.read(size=1024 * 1024 + 2,
offset=1024 * 1024 - 1)
self.assertEqual([
(b'a', 1),
(b'b', 1024 * 1024),
(b'c', 1),
], group_by_byte(file_contents))
class TestSymlinkSloEnv(TestSloEnv):
@classmethod
def create_links_to_segments(cls, container):
seg_info = {}
for letter in ('a', 'b'):
seg_name = "linkto_seg_%s" % letter
file_item = container.file(seg_name)
sym_hdr = {'X-Symlink-Target': '%s/seg_%s' % (container.name,
letter)}
file_item.write(hdrs=sym_hdr)
seg_info[seg_name] = {
'path': '/%s/%s' % (container.name, seg_name)}
return seg_info
@classmethod
def setUp(cls):
super(TestSymlinkSloEnv, cls).setUp()
cls.link_seg_info = cls.create_links_to_segments(cls.container)
file_item = cls.container.file("manifest-linkto-ab")
file_item.write(
json.dumps([cls.link_seg_info['linkto_seg_a'],
cls.link_seg_info['linkto_seg_b']]).encode('ascii'),
parms={'multipart-manifest': 'put'})
class TestSymlinkToSloSegments(Base):
"""
This test class will contain various tests where the segments of the SLO
manifest are symlinks to the actual segments. Again the tests are basicaly
a copy/paste of the tests in test_slo, only the manifest has been modified
to contain symlinks as the segments.
"""
env = TestSymlinkSloEnv
def setUp(self):
super(TestSymlinkToSloSegments, self).setUp()
if self.env.slo_enabled is False:
raise SkipTest("SLO not enabled")
elif self.env.slo_enabled is not True:
# just some sanity checking
raise Exception(
"Expected slo_enabled to be True/False, got %r" %
(self.env.slo_enabled,))
def test_slo_get_simple_manifest_with_links(self):
file_item = self.env.container.file("manifest-linkto-ab")
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
], group_by_byte(file_item.read()))
def test_slo_container_listing(self):
# the listing object size should equal the sum of the size of the
# segments, not the size of the manifest body
file_item = self.env.container.file(Utils.create_name())
file_item.write(
json.dumps([
self.env.link_seg_info['linkto_seg_a']]).encode('ascii'),
parms={'multipart-manifest': 'put'})
# The container listing has the etag of the actual manifest object
# contents which we get using multipart-manifest=get. New enough swift
# also exposes the etag that we get when NOT using
# multipart-manifest=get. Verify that both remain consistent when the
# object is updated with a POST.
file_item.initialize()
slo_etag = file_item.etag
file_item.initialize(parms={'multipart-manifest': 'get'})
manifest_etag = file_item.etag
listing = self.env.container.files(parms={'format': 'json'})
for f_dict in listing:
if f_dict['name'] == file_item.name:
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual('application/octet-stream',
f_dict['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
self.fail('Failed to find manifest file in container listing')
# now POST updated content-type file
file_item.content_type = 'image/jpeg'
file_item.sync_metadata({'X-Object-Meta-Test': 'blah'})
file_item.initialize()
self.assertEqual('image/jpeg', file_item.content_type) # sanity
# verify that the container listing is consistent with the file
listing = self.env.container.files(parms={'format': 'json'})
for f_dict in listing:
if f_dict['name'] == file_item.name:
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual(file_item.content_type,
f_dict['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
self.fail('Failed to find manifest file in container listing')
# now POST with no change to content-type
file_item.sync_metadata({'X-Object-Meta-Test': 'blah'},
cfg={'no_content_type': True})
file_item.initialize()
self.assertEqual('image/jpeg', file_item.content_type) # sanity
# verify that the container listing is consistent with the file
listing = self.env.container.files(parms={'format': 'json'})
for f_dict in listing:
if f_dict['name'] == file_item.name:
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual(file_item.content_type,
f_dict['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
self.fail('Failed to find manifest file in container listing')
def test_slo_etag_is_hash_of_etags(self):
expected_hash = md5(usedforsecurity=False)
expected_hash.update((
md5(b'a' * 1024 * 1024, usedforsecurity=False)
.hexdigest().encode('ascii')))
expected_hash.update((
md5(b'b' * 1024 * 1024, usedforsecurity=False)
.hexdigest().encode('ascii')))
expected_etag = expected_hash.hexdigest()
file_item = self.env.container.file('manifest-linkto-ab')
self.assertEqual('"%s"' % expected_etag, file_item.info()['etag'])
def test_slo_copy(self):
file_item = self.env.container.file("manifest-linkto-ab")
file_item.copy(self.env.container.name, "copied-abcde")
copied = self.env.container.file("copied-abcde")
self.assertEqual([
(b'a', 1024 * 1024),
(b'b', 1024 * 1024),
], group_by_byte(copied.read(parms={'multipart-manifest': 'get'})))
def test_slo_copy_the_manifest(self):
# first just perform some tests of the contents of the manifest itself
source = self.env.container.file("manifest-linkto-ab")
source_contents = source.read(parms={'multipart-manifest': 'get'})
source_json = json.loads(source_contents)
manifest_etag = md5(source_contents, usedforsecurity=False).hexdigest()
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
manifest_etag = '"%s"' % manifest_etag
source.initialize()
slo_etag = source.etag
self.assertEqual('application/octet-stream', source.content_type)
source.initialize(parms={'multipart-manifest': 'get'})
self.assertEqual(manifest_etag, source.etag)
self.assertEqual('application/json; charset=utf-8',
source.content_type)
# now, copy the manifest
self.assertTrue(source.copy(self.env.container.name,
"copied-ab-manifest-only",
parms={'multipart-manifest': 'get'}))
copied = self.env.container.file("copied-ab-manifest-only")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
try:
copied_json = json.loads(copied_contents)
except ValueError:
self.fail("COPY didn't copy the manifest (invalid json on GET)")
# make sure content of copied manifest is the same as original man.
self.assertEqual(source_json, copied_json)
copied.initialize()
self.assertEqual(copied.etag, slo_etag)
self.assertEqual('application/octet-stream', copied.content_type)
copied.initialize(parms={'multipart-manifest': 'get'})
self.assertEqual(source_contents, copied_contents)
self.assertEqual(copied.etag, manifest_etag)
self.assertEqual('application/json; charset=utf-8',
copied.content_type)
# verify the listing metadata
listing = self.env.container.files(parms={'format': 'json'})
names = {}
for f_dict in listing:
if f_dict['name'] in ('manifest-linkto-ab',
'copied-ab-manifest-only'):
names[f_dict['name']] = f_dict
self.assertIn('manifest-linkto-ab', names)
actual = names['manifest-linkto-ab']
self.assertEqual(2 * 1024 * 1024, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % actual['hash'])
else:
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
self.assertIn('copied-ab-manifest-only', names)
actual = names['copied-ab-manifest-only']
self.assertEqual(2 * 1024 * 1024, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % actual['hash'])
else:
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
class TestSymlinkDlo(Base):
env = TestDloEnv
def test_get_manifest(self):
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man1')})
self.assertEqual([
(b'a', 10),
(b'b', 10),
(b'c', 10),
(b'd', 10),
(b'e', 10),
], group_by_byte(file_symlink.read()))
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man2')})
self.assertEqual([
(b'A', 10),
(b'B', 10),
(b'C', 10),
(b'D', 10),
(b'E', 10),
], group_by_byte(file_symlink.read()))
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'manall')})
self.assertEqual([
(b'a', 10),
(b'b', 10),
(b'c', 10),
(b'd', 10),
(b'e', 10),
(b'A', 10),
(b'B', 10),
(b'C', 10),
(b'D', 10),
(b'E', 10),
], group_by_byte(file_symlink.read()))
def test_get_manifest_document_itself(self):
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man1')})
file_contents = file_symlink.read(parms={'multipart-manifest': 'get'})
self.assertEqual(file_contents, b"man1-contents")
self.assertEqual(file_symlink.info()['x_object_manifest'],
"%s/%s/seg_lower" %
(self.env.container.name, self.env.segment_prefix))
def test_get_range(self):
link_obj = uuid4().hex + "_symlink"
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man1')})
self.assertEqual([
(b'a', 2),
(b'b', 10),
(b'c', 10),
(b'd', 3),
], group_by_byte(file_symlink.read(size=25, offset=8)))
file_contents = file_symlink.read(size=1, offset=47)
self.assertEqual(file_contents, b"e")
def test_get_range_out_of_range(self):
link_obj = uuid4().hex
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
'man1')})
self.assertRaises(ResponseError, file_symlink.read, size=7, offset=50)
self.assert_status(416)
class TestSymlinkTargetObjectComparisonEnv(TestFileComparisonEnv):
@classmethod
def setUp(cls):
super(TestSymlinkTargetObjectComparisonEnv, cls).setUp()
cls.parms = None
cls.expect_empty_etag = False
cls.expect_body = True
class TestSymlinkComparisonEnv(TestFileComparisonEnv):
@classmethod
def setUp(cls):
super(TestSymlinkComparisonEnv, cls).setUp()
cls.parms = {'symlink': 'get'}
cls.expect_empty_etag = True
cls.expect_body = False
class TestSymlinkTargetObjectComparison(Base):
env = TestSymlinkTargetObjectComparisonEnv
def setUp(self):
super(TestSymlinkTargetObjectComparison, self).setUp()
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
file_item.name)})
def testIfMatch(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Match': md5}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-Match': 'bogus'}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
def testIfMatchMultipleEtags(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Match': '"bogus1", "%s", "bogus2"' % md5}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-Match': '"bogus1", "bogus2", "bogus3"'}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
def testIfNoneMatch(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-None-Match': 'bogus'}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-None-Match': md5}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfNoneMatchMultipleEtags(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-None-Match': '"bogus1", "bogus2", "bogus3"'}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-None-Match':
'"bogus1", "bogus2", "%s"' % md5}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfModifiedSince(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Modified-Since': self.env.time_old_f1}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms))
hdrs = {'If-Modified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfUnmodifiedSince(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Unmodified-Since': self.env.time_new}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms))
hdrs = {'If-Unmodified-Since': self.env.time_old_f2}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
def testIfMatchAndUnmodified(self):
for file_item in self.env.files:
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
md5 = MD5_OF_EMPTY_STRING if self.env.expect_empty_etag else \
file_item.md5
hdrs = {'If-Match': md5,
'If-Unmodified-Since': self.env.time_new}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
if self.env.expect_body:
self.assertTrue(body)
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-Match': 'bogus',
'If-Unmodified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
hdrs = {'If-Match': md5,
'If-Unmodified-Since': self.env.time_old_f3}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_etag(md5)
def testLastModified(self):
file_item = self.env.container.file(Utils.create_name())
file_item.content_type = Utils.create_name()
resp = file_item.write_random_return_resp(self.env.file_size)
put_last_modified = resp.getheader('last-modified')
md5 = file_item.md5
# create symlink
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
file_symlink.write(hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
file_item.name)})
info = file_symlink.info()
self.assertIn('last_modified', info)
last_modified = info['last_modified']
self.assertEqual(put_last_modified, info['last_modified'])
hdrs = {'If-Modified-Since': last_modified}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
hdrs = {'If-Unmodified-Since': last_modified}
self.assertTrue(file_symlink.read(hdrs=hdrs))
class TestSymlinkComparison(TestSymlinkTargetObjectComparison):
env = TestSymlinkComparisonEnv
def setUp(self):
super(TestSymlinkComparison, self).setUp()
def testLastModified(self):
file_item = self.env.container.file(Utils.create_name())
file_item.content_type = Utils.create_name()
resp = file_item.write_random_return_resp(self.env.file_size)
put_target_last_modified = resp.getheader('last-modified')
md5 = MD5_OF_EMPTY_STRING
# get different last-modified between file and symlink
time.sleep(1)
# create symlink
link_obj = file_item.name + '_symlink'
file_symlink = self.env.container.file(link_obj)
resp = file_symlink.write(return_resp=True,
hdrs={'X-Symlink-Target':
'%s/%s' % (self.env.container.name,
file_item.name)})
put_sym_last_modified = resp.getheader('last-modified')
info = file_symlink.info(parms=self.env.parms)
self.assertIn('last_modified', info)
last_modified = info['last_modified']
self.assertEqual(put_sym_last_modified, info['last_modified'])
hdrs = {'If-Modified-Since': put_target_last_modified}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
hdrs = {'If-Modified-Since': last_modified}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
hdrs = {'If-Unmodified-Since': last_modified}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_etag(md5)
class TestSymlinkAccountTempurl(Base):
env = TestTempurlEnv
digest_name = 'sha256'
def setUp(self):
super(TestSymlinkAccountTempurl, self).setUp()
if self.env.tempurl_enabled is False:
raise SkipTest("TempURL not enabled")
elif self.env.tempurl_enabled is not True:
# just some sanity checking
raise Exception(
"Expected tempurl_enabled to be True/False, got %r" %
(self.env.tempurl_enabled,))
if self.digest_name not in cluster_info['tempurl'].get(
'allowed_digests', ['sha1']):
raise SkipTest("tempurl does not support %s signatures" %
self.digest_name)
self.digest = getattr(hashlib, self.digest_name)
self.expires = int(time.time()) + 86400
self.obj_tempurl_parms = self.tempurl_parms(
'GET', self.expires, self.env.conn.make_path(self.env.obj.path),
self.env.tempurl_key)
def tempurl_parms(self, method, expires, path, key):
path = urllib.parse.unquote(path)
if not six.PY2:
method = method.encode('utf8')
path = path.encode('utf8')
key = key.encode('utf8')
sig = hmac.new(
key,
b'%s\n%d\n%s' % (method, expires, path),
self.digest).hexdigest()
return {'temp_url_sig': sig, 'temp_url_expires': str(expires)}
def test_PUT_symlink(self):
new_sym = self.env.container.file(Utils.create_name())
# give out a signature which allows a PUT to new_obj
expires = int(time.time()) + 86400
put_parms = self.tempurl_parms(
'PUT', expires, self.env.conn.make_path(new_sym.path),
self.env.tempurl_key)
# try to create symlink object
try:
new_sym.write(
b'', {'x-symlink-target': 'cont/foo'}, parms=put_parms,
cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 400)
else:
self.fail('request did not error')
def test_GET_symlink_inside_container(self):
tgt_obj = self.env.container.file(Utils.create_name())
sym = self.env.container.file(Utils.create_name())
tgt_obj.write(b"target object body")
sym.write(
b'',
{'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)})
expires = int(time.time()) + 86400
get_parms = self.tempurl_parms(
'GET', expires, self.env.conn.make_path(sym.path),
self.env.tempurl_key)
contents = sym.read(parms=get_parms, cfg={'no_auth_token': True})
self.assert_status([200])
self.assertEqual(contents, b"target object body")
def test_GET_symlink_outside_container(self):
tgt_obj = self.env.container.file(Utils.create_name())
tgt_obj.write(b"target object body")
container2 = self.env.account.container(Utils.create_name())
container2.create()
sym = container2.file(Utils.create_name())
sym.write(
b'',
{'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)})
expires = int(time.time()) + 86400
get_parms = self.tempurl_parms(
'GET', expires, self.env.conn.make_path(sym.path),
self.env.tempurl_key)
# cross container tempurl works fine for account tempurl key
contents = sym.read(parms=get_parms, cfg={'no_auth_token': True})
self.assert_status([200])
self.assertEqual(contents, b"target object body")
class TestSymlinkContainerTempurl(Base):
env = TestContainerTempurlEnv
digest_name = 'sha256'
def setUp(self):
super(TestSymlinkContainerTempurl, self).setUp()
if self.env.tempurl_enabled is False:
raise SkipTest("TempURL not enabled")
elif self.env.tempurl_enabled is not True:
# just some sanity checking
raise Exception(
"Expected tempurl_enabled to be True/False, got %r" %
(self.env.tempurl_enabled,))
if self.digest_name not in cluster_info['tempurl'].get(
'allowed_digests', ['sha1']):
raise SkipTest("tempurl does not support %s signatures" %
self.digest_name)
self.digest = getattr(hashlib, self.digest_name)
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'GET', expires, self.env.conn.make_path(self.env.obj.path),
self.env.tempurl_key)
self.obj_tempurl_parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
def tempurl_sig(self, method, expires, path, key):
path = urllib.parse.unquote(path)
if not six.PY2:
method = method.encode('utf8')
path = path.encode('utf8')
key = key.encode('utf8')
return hmac.new(
key,
b'%s\n%d\n%s' % (method, expires, path),
self.digest).hexdigest()
def test_PUT_symlink(self):
new_sym = self.env.container.file(Utils.create_name())
# give out a signature which allows a PUT to new_obj
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'PUT', expires, self.env.conn.make_path(new_sym.path),
self.env.tempurl_key)
put_parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
# try to create symlink object, should fail
try:
new_sym.write(
b'', {'x-symlink-target': 'cont/foo'}, parms=put_parms,
cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 400)
else:
self.fail('request did not error')
def test_GET_symlink_inside_container(self):
tgt_obj = self.env.container.file(Utils.create_name())
sym = self.env.container.file(Utils.create_name())
tgt_obj.write(b"target object body")
sym.write(
b'',
{'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)})
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'GET', expires, self.env.conn.make_path(sym.path),
self.env.tempurl_key)
parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
contents = sym.read(parms=parms, cfg={'no_auth_token': True})
self.assert_status([200])
self.assertEqual(contents, b"target object body")
def test_GET_symlink_outside_container(self):
tgt_obj = self.env.container.file(Utils.create_name())
tgt_obj.write(b"target object body")
container2 = self.env.account.container(Utils.create_name())
container2.create()
sym = container2.file(Utils.create_name())
sym.write(
b'',
{'x-symlink-target': '%s/%s' % (self.env.container.name, tgt_obj)})
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'GET', expires, self.env.conn.make_path(sym.path),
self.env.tempurl_key)
parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
# cross container tempurl does not work for container tempurl key
try:
sym.read(parms=parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 401)
else:
self.fail('request did not error')
try:
sym.info(parms=parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 401)
else:
self.fail('request did not error')
if __name__ == '__main__':
unittest.main()
View Git Blame Copy Permalink