From aa1bfb3e67218e905db1163b2b83f72f33b912a1 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Thu, 15 Jan 2015 12:31:55 +0530 Subject: [PATCH] Rebase to Swift 2.2.1 release NOTE: The previous rebase was to Swift 2.1.0 and this rebase is to Swift 2.2.1 (first release in kilo series). There was a Swift 2.2.0 (last release in juno series) release in between. Change-Id: Ibce2e299935e165db89a91a6fe8c4c5c027db098 Signed-off-by: Prashanth Pai --- requirements.txt | 4 + swiftonfile/swift/__init__.py | 2 +- test-requirements.txt | 4 + test/functional/__init__.py | 142 ++++---- test/functional/swift_test_client.py | 30 +- test/functional/test_account.py | 53 +++ test/functional/test_container.py | 196 ++++++++++- test/functional/test_object.py | 154 ++++++++- test/functional/tests.py | 344 ++++++++++++++++++- test/unit/obj/test_server.py | 30 -- test/unit/{test_swift.py => test_sof_pkg.py} | 0 tox.ini | 2 +- 12 files changed, 855 insertions(+), 106 deletions(-) delete mode 100644 test/unit/obj/test_server.py rename test/unit/{test_swift.py => test_sof_pkg.py} (100%) diff --git a/requirements.txt b/requirements.txt index 1e3b339..96cb4fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + dnspython>=1.9.4 eventlet>=0.9.15 greenlet>=0.3.1 diff --git a/swiftonfile/swift/__init__.py b/swiftonfile/swift/__init__.py index 9f18b56..7253670 100644 --- a/swiftonfile/swift/__init__.py +++ b/swiftonfile/swift/__init__.py @@ -43,6 +43,6 @@ class PkgInfo(object): # Change the Package version here -_pkginfo = PkgInfo('2.1.0', '0', 'swiftonfile', False) +_pkginfo = PkgInfo('2.2.1', '0', 'swiftonfile', False) __version__ = _pkginfo.pretty_version __canonical_version__ = _pkginfo.canonical_version diff --git a/test-requirements.txt b/test-requirements.txt index a5a7118..8c617ba 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,7 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + # Hacking already pins down pep8, pyflakes and flake8 hacking>=0.8.0,<0.9 coverage diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 1d66169..fd460d4 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mock import os import sys import pickle @@ -30,6 +31,7 @@ from contextlib import closing from gzip import GzipFile from shutil import rmtree from tempfile import mkdtemp +from swift.common.middleware.memcache import MemcacheMiddleware from test import get_config from test.functional.swift_test_client import Account, Connection, \ @@ -40,15 +42,12 @@ from test.functional.swift_test_client import Account, Connection, \ from test.unit import debug_logger, FakeMemcache from swift.common import constraints, utils, ring, storage_policy -from swift.common.wsgi import monkey_patch_mimetools -from swift.common.middleware import catch_errors, gatekeeper, healthcheck, \ - proxy_logging, container_sync, bulk, tempurl, slo, dlo, ratelimit, \ - tempauth, container_quotas, account_quotas +from swift.common.ring import Ring +from swift.common.wsgi import monkey_patch_mimetools, loadapp from swift.common.utils import config_true_value -from swift.proxy import server as proxy_server from swift.account import server as account_server from swift.container import server as container_server -from swift.obj import server as object_server +from swift.obj import server as object_server, mem_server as mem_object_server import swift.proxy.controllers.obj # In order to get the proper blocking behavior of sockets without using @@ -83,10 +82,13 @@ normalized_urls = None # If no config was read, we will fall back to old school env vars swift_test_auth_version = None swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') -swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] -swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] -swift_test_tenant = ['', '', ''] -swift_test_perm = ['', '', ''] +swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None, ''] +swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None, ''] +swift_test_tenant = ['', '', '', ''] +swift_test_perm = ['', '', '', ''] +swift_test_domain = ['', '', '', ''] +swift_test_user_id = ['', '', '', ''] +swift_test_tenant_id = ['', '', '', ''] skip, skip2, skip3 = False, False, False @@ -100,25 +102,16 @@ in_process = False _testdir = _test_servers = _test_sockets = _test_coros = None -class FakeMemcacheMiddleware(object): +class FakeMemcacheMiddleware(MemcacheMiddleware): """ - Caching middleware that fakes out caching in swift. + Caching middleware that fakes out caching in swift if memcached + does not appear to be running. """ def __init__(self, app, conf): - self.app = app + super(FakeMemcacheMiddleware, self).__init__(app, conf) self.memcache = FakeMemcache() - def __call__(self, env, start_response): - env['swift.cache'] = self.memcache - return self.app(env, start_response) - - -def fake_memcache_filter_factory(conf): - def filter_app(app): - return FakeMemcacheMiddleware(app, conf) - return filter_app - # swift.conf contents for in-process functional test runs functests_swift_conf = ''' @@ -133,6 +126,16 @@ max_file_size = %d def in_process_setup(the_object_server=object_server): print >>sys.stderr, 'IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS' + print >>sys.stderr, 'Using object_server: %s' % the_object_server.__name__ + _dir = os.path.normpath(os.path.join(os.path.abspath(__file__), + os.pardir, os.pardir, os.pardir)) + proxy_conf = os.path.join(_dir, 'etc', 'proxy-server.conf-sample') + if os.path.exists(proxy_conf): + print >>sys.stderr, 'Using proxy-server config from %s' % proxy_conf + + else: + print >>sys.stderr, 'Failed to find conf file %s' % proxy_conf + return monkey_patch_mimetools() @@ -159,7 +162,9 @@ def in_process_setup(the_object_server=object_server): if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration - config.update(constraints.EFFECTIVE_CONSTRAINTS) + _c = dict((k, str(v)) + for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) + config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest @@ -180,12 +185,9 @@ def in_process_setup(the_object_server=object_server): 'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', - 'client_timeout': 4, + 'client_timeout': '4', 'allow_account_management': 'true', 'account_autocreate': 'true', - 'allowed_headers': - 'content-disposition, content-encoding, x-delete-at,' - ' x-object-manifest, x-static-large-object', 'allow_versions': 'True', # Below are values used by the functional test framework, as well as # by the various in-process swift servers @@ -257,7 +259,6 @@ def in_process_setup(the_object_server=object_server): # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 - prosrv = proxy_server.Application(config, logger=debug_logger('proxy')) acc1srv = account_server.AccountController( config, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( @@ -270,35 +271,16 @@ def in_process_setup(the_object_server=object_server): config, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( config, logger=debug_logger('obj2')) - global _test_servers - _test_servers = \ - (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv) - pipeline = [ - catch_errors.filter_factory, - gatekeeper.filter_factory, - healthcheck.filter_factory, - proxy_logging.filter_factory, - fake_memcache_filter_factory, - container_sync.filter_factory, - bulk.filter_factory, - tempurl.filter_factory, - slo.filter_factory, - dlo.filter_factory, - ratelimit.filter_factory, - tempauth.filter_factory, - container_quotas.filter_factory, - account_quotas.filter_factory, - proxy_logging.filter_factory, - ] - app = prosrv - import mock - for filter_factory in reversed(pipeline): - app_filter = filter_factory(config) - with mock.patch('swift.common.utils') as mock_utils: - mock_utils.get_logger.return_value = None - app = app_filter(app) - app.logger = prosrv.logger + logger = debug_logger('proxy') + + def get_logger(name, *args, **kwargs): + return logger + + with mock.patch('swift.common.utils.get_logger', get_logger): + with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', + FakeMemcacheMiddleware): + app = loadapp(proxy_conf, global_conf=config) nl = utils.NullLogger() prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl) @@ -315,7 +297,8 @@ def in_process_setup(the_object_server=object_server): # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) - partition, nodes = prosrv.account_ring.get_nodes(act) + account_ring = Ring(_testdir, ring_name='account') + partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. @@ -348,7 +331,13 @@ def get_cluster_info(): # test.conf data pass else: - eff_constraints.update(cluster_info.get('swift', {})) + try: + eff_constraints.update(cluster_info['swift']) + except KeyError: + # Most likely the swift cluster has "expose_info = false" set + # in its proxy-server.conf file, so we'll just do the best we + # can. + print >>sys.stderr, "** Swift Cluster not exposing /info **" # Finally, we'll allow any constraint present in the swift-constraints # section of test.conf to override everything. Note that only those @@ -402,7 +391,10 @@ def setup_package(): config.update(get_config('func_test')) if in_process: - in_process_setup() + in_mem_obj_env = os.environ.get('SWIFT_TEST_IN_MEMORY_OBJ') + in_mem_obj = utils.config_true_value(in_mem_obj_env) + in_process_setup(the_object_server=( + mem_object_server if in_mem_obj else object_server)) global web_front_end web_front_end = config.get('web_front_end', 'integral') @@ -422,6 +414,7 @@ def setup_package(): global swift_test_key global swift_test_tenant global swift_test_perm + global swift_test_domain if config: swift_test_auth_version = str(config.get('auth_version', '1')) @@ -478,8 +471,13 @@ def setup_package(): swift_test_user[2] = config['username3'] swift_test_tenant[2] = config['account'] swift_test_key[2] = config['password3'] + if 'username4' in config: + swift_test_user[3] = config['username4'] + swift_test_tenant[3] = config['account4'] + swift_test_key[3] = config['password4'] + swift_test_domain[3] = config['domain4'] - for _ in range(3): + for _ in range(4): swift_test_perm[_] = swift_test_tenant[_] + ':' \ + swift_test_user[_] @@ -501,6 +499,15 @@ def setup_package(): print >>sys.stderr, \ 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + global skip_if_not_v3 + skip_if_not_v3 = (swift_test_auth_version != '3' + or not all([not skip, + swift_test_user[3], + swift_test_key[3]])) + if not skip and skip_if_not_v3: + print >>sys.stderr, \ + 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3' + get_cluster_info() @@ -539,10 +546,10 @@ class InternalServerError(Exception): pass -url = [None, None, None] -token = [None, None, None] -parsed = [None, None, None] -conn = [None, None, None] +url = [None, None, None, None] +token = [None, None, None, None] +parsed = [None, None, None, None] +conn = [None, None, None, None] def connection(url): @@ -569,7 +576,8 @@ def retry(func, *args, **kwargs): # access our own account by default url_account = kwargs.pop('url_account', use_account + 1) - 1 - + os_options = {'user_domain_name': swift_test_domain[use_account], + 'project_domain_name': swift_test_domain[use_account]} while attempts <= retries: attempts += 1 try: @@ -580,7 +588,7 @@ def retry(func, *args, **kwargs): snet=False, tenant_name=swift_test_tenant[use_account], auth_version=swift_test_auth_version, - os_options={}) + os_options=os_options) parsed[use_account] = conn[use_account] = None if not parsed[use_account] or not conn[use_account]: parsed[use_account], conn[use_account] = \ diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 2c35520..941dfbb 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -174,8 +174,10 @@ class Connection(object): # unicode and this would cause troubles when doing # no_safe_quote query. self.storage_url = str('/%s/%s' % (x[3], x[4])) - + self.account_name = str(x[4]) + self.auth_user = auth_user self.storage_token = storage_token + self.user_acl = '%s:%s' % (self.account, self.username) self.http_connect() return self.storage_url, self.storage_token @@ -664,6 +666,32 @@ class File(Base): return self.conn.make_request('COPY', self.path, hdrs=headers, parms=parms) == 201 + def copy_account(self, dest_account, dest_cont, dest_file, + hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + if 'destination' in cfg: + headers = {'Destination': cfg['destination']} + elif cfg.get('no_destination'): + headers = {} + else: + headers = {'Destination-Account': dest_account, + 'Destination': '%s/%s' % (dest_cont, dest_file)} + headers.update(hdrs) + + if 'Destination-Account' in headers: + headers['Destination-Account'] = \ + urllib.quote(headers['Destination-Account']) + if 'Destination' in headers: + headers['Destination'] = urllib.quote(headers['Destination']) + + return self.conn.make_request('COPY', self.path, hdrs=headers, + parms=parms) == 201 + def delete(self, hdrs=None, parms=None): if hdrs is None: hdrs = {} diff --git a/test/functional/test_account.py b/test/functional/test_account.py index 0b0ccff..eee42e5 100755 --- a/test/functional/test_account.py +++ b/test/functional/test_account.py @@ -777,6 +777,21 @@ class TestAccount(unittest.TestCase): resp.read() self.assertEqual(resp.status, 400) + def test_bad_metadata2(self): + if tf.skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + + # TODO: Find the test that adds these and remove them. + headers = {'x-remove-account-meta-temp-url-key': 'remove', + 'x-remove-account-meta-temp-url-key-2': 'remove'} + resp = retry(post, headers) + headers = {} for x in xrange(self.max_meta_count): headers['X-Account-Meta-%d' % x] = 'v' @@ -790,6 +805,16 @@ class TestAccount(unittest.TestCase): resp.read() self.assertEqual(resp.status, 400) + def test_bad_metadata3(self): + if tf.skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + headers = {} header_value = 'k' * self.max_meta_value_length size = 0 @@ -812,5 +837,33 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status, 400) +class TestAccountInNonDefaultDomain(unittest.TestCase): + def setUp(self): + if tf.skip or tf.skip2 or tf.skip_if_not_v3: + raise SkipTest('AUTH VERSION 3 SPECIFIC TEST') + + def test_project_domain_id_header(self): + # make sure account exists (assumes account auto create) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(post, use_account=4) + resp.read() + self.assertEqual(resp.status, 204) + + # account in non-default domain should have a project domain id + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(head, use_account=4) + resp.read() + self.assertEqual(resp.status, 204) + self.assertTrue('X-Account-Project-Domain-Id' in resp.headers) + + if __name__ == '__main__': unittest.main() diff --git a/test/functional/test_container.py b/test/functional/test_container.py index 969b9bf..ba561b5 100755 --- a/test/functional/test_container.py +++ b/test/functional/test_container.py @@ -404,6 +404,16 @@ class TestContainer(unittest.TestCase): resp.read() self.assertEqual(resp.status, 400) + def test_POST_bad_metadata2(self): + if tf.skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) + headers = {} for x in xrange(self.max_meta_count): headers['X-Container-Meta-%d' % x] = 'v' @@ -417,6 +427,16 @@ class TestContainer(unittest.TestCase): resp.read() self.assertEqual(resp.status, 400) + def test_POST_bad_metadata3(self): + if tf.skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) + headers = {} header_value = 'k' * self.max_meta_value_length size = 0 @@ -1419,7 +1439,7 @@ class TestContainer(unittest.TestCase): self.assertEquals(headers.get('x-storage-policy'), policy['name']) - # and test recreate with-out specifiying Storage Policy + # and test recreate with-out specifying Storage Policy resp = retry(put) resp.read() self.assertEqual(resp.status, 202) @@ -1514,5 +1534,179 @@ class TestContainer(unittest.TestCase): policy['name']) +class BaseTestContainerACLs(unittest.TestCase): + # subclasses can change the account in which container + # is created/deleted by setUp/tearDown + account = 1 + + def _get_account(self, url, token, parsed, conn): + return parsed.path + + def _get_tenant_id(self, url, token, parsed, conn): + account = parsed.path + return account.replace('/v1/AUTH_', '', 1) + + def setUp(self): + if tf.skip or tf.skip2 or tf.skip_if_not_v3: + raise SkipTest('AUTH VERSION 3 SPECIFIC TEST') + self.name = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 201) + + def tearDown(self): + if tf.skip or tf.skip2 or tf.skip_if_not_v3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name + '?format=json', + '', {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '/'.join([parsed.path, self.name, obj['name']]), '', + {'X-Auth-Token': token}) + return check_response(conn) + + while True: + resp = retry(get, use_account=self.account) + body = resp.read() + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, obj, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 204) + + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(delete, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 204) + + def _assert_cross_account_acl_granted(self, granted, grantee_account, acl): + ''' + Check whether a given container ACL is granted when a user specified + by account_b attempts to access a container. + ''' + # Obtain the first account's string + first_account = retry(self._get_account, use_account=self.account) + + # Ensure we can't access the container with the grantee account + def get2(url, token, parsed, conn): + conn.request('GET', first_account + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(get2, use_account=grantee_account) + resp.read() + self.assertEqual(resp.status, 403) + + def put2(url, token, parsed, conn): + conn.request('PUT', first_account + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put2, use_account=grantee_account) + resp.read() + self.assertEqual(resp.status, 403) + + # Post ACL to the container + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': acl, + 'X-Container-Write': acl}) + return check_response(conn) + + resp = retry(post, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 204) + + # Check access to container from grantee account with ACL in place + resp = retry(get2, use_account=grantee_account) + resp.read() + expected = 204 if granted else 403 + self.assertEqual(resp.status, expected) + + resp = retry(put2, use_account=grantee_account) + resp.read() + expected = 201 if granted else 403 + self.assertEqual(resp.status, expected) + + # Make the container private again + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': '', + 'X-Container-Write': ''}) + return check_response(conn) + + resp = retry(post, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 204) + + # Ensure we can't access the container with the grantee account again + resp = retry(get2, use_account=grantee_account) + resp.read() + self.assertEqual(resp.status, 403) + + resp = retry(put2, use_account=grantee_account) + resp.read() + self.assertEqual(resp.status, 403) + + +class TestContainerACLsAccount1(BaseTestContainerACLs): + def test_cross_account_acl_names_with_user_in_non_default_domain(self): + # names in acls are disallowed when grantee is in a non-default domain + acl = '%s:%s' % (tf.swift_test_tenant[3], tf.swift_test_user[3]) + self._assert_cross_account_acl_granted(False, 4, acl) + + def test_cross_account_acl_ids_with_user_in_non_default_domain(self): + # ids are allowed in acls when grantee is in a non-default domain + tenant_id = retry(self._get_tenant_id, use_account=4) + acl = '%s:%s' % (tenant_id, '*') + self._assert_cross_account_acl_granted(True, 4, acl) + + def test_cross_account_acl_names_in_default_domain(self): + # names are allowed in acls when grantee and project are in + # the default domain + acl = '%s:%s' % (tf.swift_test_tenant[1], tf.swift_test_user[1]) + self._assert_cross_account_acl_granted(True, 2, acl) + + def test_cross_account_acl_ids_in_default_domain(self): + # ids are allowed in acls when grantee and project are in + # the default domain + tenant_id = retry(self._get_tenant_id, use_account=2) + acl = '%s:%s' % (tenant_id, '*') + self._assert_cross_account_acl_granted(True, 2, acl) + + +class TestContainerACLsAccount4(BaseTestContainerACLs): + account = 4 + + def test_cross_account_acl_names_with_project_in_non_default_domain(self): + # names in acls are disallowed when project is in a non-default domain + acl = '%s:%s' % (tf.swift_test_tenant[0], tf.swift_test_user[0]) + self._assert_cross_account_acl_granted(False, 1, acl) + + def test_cross_account_acl_ids_with_project_in_non_default_domain(self): + # ids are allowed in acls when project is in a non-default domain + tenant_id = retry(self._get_tenant_id, use_account=1) + acl = '%s:%s' % (tenant_id, '*') + self._assert_cross_account_acl_granted(True, 1, acl) + + if __name__ == '__main__': unittest.main() diff --git a/test/functional/test_object.py b/test/functional/test_object.py index 6b29800..e74a7f6 100755 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -35,6 +35,7 @@ class TestObject(unittest.TestCase): self.containers = [] self._create_container(self.container) + self._create_container(self.container, use_account=2) self.obj = uuid4().hex @@ -47,7 +48,7 @@ class TestObject(unittest.TestCase): resp.read() self.assertEqual(resp.status, 201) - def _create_container(self, name=None, headers=None): + def _create_container(self, name=None, headers=None, use_account=1): if not name: name = uuid4().hex self.containers.append(name) @@ -58,7 +59,7 @@ class TestObject(unittest.TestCase): conn.request('PUT', parsed.path + '/' + name, '', new_headers) return check_response(conn) - resp = retry(put, name) + resp = retry(put, name, use_account=use_account) resp.read() self.assertEqual(resp.status, 201) return name @@ -133,6 +134,45 @@ class TestObject(unittest.TestCase): resp.read() self.assertEquals(resp.status, 400) + def test_non_integer_x_delete_after(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'non_integer_x_delete_after'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-After': '*'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'Non-integer X-Delete-After') + + def test_non_integer_x_delete_at(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'non_integer_x_delete_at'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': '*'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'Non-integer X-Delete-At') + + def test_x_delete_at_in_the_past(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'x_delete_at_in_the_past'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': '0'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'X-Delete-At in past') + def test_copy_object(self): if tf.skip: raise SkipTest @@ -207,6 +247,116 @@ class TestObject(unittest.TestCase): resp.read() self.assertEqual(resp.status, 204) + def test_copy_between_accounts(self): + if tf.skip: + raise SkipTest + + source = '%s/%s' % (self.container, self.obj) + dest = '%s/%s' % (self.container, 'test_copy') + + # get contents of source + def get_source(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, source), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_source) + source_contents = resp.read() + self.assertEqual(resp.status, 200) + self.assertEqual(source_contents, 'test') + + acct = tf.parsed[0].path.split('/', 2)[2] + + # copy source to dest with X-Copy-From-Account + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From-Account': acct, + 'X-Copy-From': source}) + return check_response(conn) + # try to put, will not succeed + # user does not have permissions to read from source + resp = retry(put, use_account=2) + self.assertEqual(resp.status, 403) + + # add acl to allow reading from source + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token, + 'X-Container-Read': tf.swift_test_perm[1]}) + return check_response(conn) + resp = retry(post) + self.assertEqual(resp.status, 204) + + # retry previous put, now should succeed + resp = retry(put, use_account=2) + self.assertEqual(resp.status, 201) + + # contents of dest should be the same as source + def get_dest(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, dest), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_dest, use_account=2) + dest_contents = resp.read() + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents) + + # delete the copy + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete, use_account=2) + resp.read() + self.assertEqual(resp.status, 204) + # verify dest does not exist + resp = retry(get_dest, use_account=2) + resp.read() + self.assertEqual(resp.status, 404) + + acct_dest = tf.parsed[1].path.split('/', 2)[2] + + # copy source to dest with COPY + def copy(url, token, parsed, conn): + conn.request('COPY', '%s/%s' % (parsed.path, source), '', + {'X-Auth-Token': token, + 'Destination-Account': acct_dest, + 'Destination': dest}) + return check_response(conn) + # try to copy, will not succeed + # user does not have permissions to write to destination + resp = retry(copy) + resp.read() + self.assertEqual(resp.status, 403) + + # add acl to allow write to destination + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token, + 'X-Container-Write': tf.swift_test_perm[0]}) + return check_response(conn) + resp = retry(post, use_account=2) + self.assertEqual(resp.status, 204) + + # now copy will succeed + resp = retry(copy) + resp.read() + self.assertEqual(resp.status, 201) + + # contents of dest should be the same as source + resp = retry(get_dest, use_account=2) + dest_contents = resp.read() + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents) + + # delete the copy + resp = retry(delete, use_account=2) + resp.read() + self.assertEqual(resp.status, 204) + def test_public_object(self): if tf.skip: raise SkipTest diff --git a/test/functional/tests.py b/test/functional/tests.py index 924f193..d0c415d 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -25,6 +25,7 @@ import time import unittest import urllib import uuid +from copy import deepcopy import eventlet from nose import SkipTest @@ -269,6 +270,8 @@ class TestAccount(Base): containers) def testQuotedWWWAuthenticateHeader(self): + # check that the www-authenticate header value with the swift realm + # is correctly quoted. conn = Connection(tf.config) conn.authenticate() inserted_html = 'Hello World' @@ -277,9 +280,16 @@ class TestAccount(Base): quoted_hax = urllib.quote(hax) conn.connection.request('GET', '/v1/' + quoted_hax, None, {}) resp = conn.connection.getresponse() - resp_headers = resp.getheaders() - expected = ('www-authenticate', 'Swift realm="%s"' % quoted_hax) - self.assert_(expected in resp_headers) + resp_headers = dict(resp.getheaders()) + self.assertTrue('www-authenticate' in resp_headers, + 'www-authenticate not found in %s' % resp_headers) + actual = resp_headers['www-authenticate'] + expected = 'Swift realm="%s"' % quoted_hax + # other middleware e.g. auth_token may also set www-authenticate + # headers in which case actual values will be a comma separated list. + # check that expected value is among the actual values + self.assertTrue(expected in actual, + '%s not found in %s' % (expected, actual)) class TestAccountUTF8(Base2, TestAccount): @@ -794,9 +804,22 @@ class TestFileEnv(object): def setUp(cls): cls.conn = Connection(tf.config) cls.conn.authenticate() + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + # creating another account and connection + # for account to account copy tests + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + cls.account = Account(cls.conn, tf.config.get('account', tf.config['username'])) cls.account.delete_containers() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): @@ -850,6 +873,62 @@ class TestFile(Base): self.assert_(file_item.initialize()) self.assert_(metadata == file_item.metadata) + def testCopyAccount(self): + # makes sure to test encoded characters + source_filename = 'dealde%2Fl04 011e%204c8df/flash.png' + file_item = self.env.container.file(source_filename) + + metadata = {Utils.create_ascii_name(): Utils.create_name()} + + data = file_item.write_random() + file_item.sync_metadata(metadata) + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + acct = self.env.conn.account_name + # copy both from within and across containers + for cont in (self.env.container, dest_cont): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file_item = self.env.container.file(source_filename) + file_item.copy_account(acct, + '%s%s' % (prefix, cont), + dest_filename) + + self.assert_(dest_filename in cont.files()) + + file_item = cont.file(dest_filename) + + self.assert_(data == file_item.read()) + self.assert_(file_item.initialize()) + self.assert_(metadata == file_item.metadata) + + dest_cont = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + + acct = self.env.conn2.account_name + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file_item = self.env.container.file(source_filename) + file_item.copy_account(acct, + '%s%s' % (prefix, dest_cont), + dest_filename) + + self.assert_(dest_filename in dest_cont.files()) + + file_item = dest_cont.file(dest_filename) + + self.assert_(data == file_item.read()) + self.assert_(file_item.initialize()) + self.assert_(metadata == file_item.metadata) + def testCopy404s(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) @@ -888,6 +967,77 @@ class TestFile(Base): '%s%s' % (prefix, Utils.create_name()), Utils.create_name())) + def testCopyAccount404s(self): + acct = self.env.conn.account_name + acct2 = self.env.conn2.account_name + source_filename = Utils.create_name() + file_item = self.env.container.file(source_filename) + file_item.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create(hdrs={ + 'X-Container-Read': self.env.conn2.user_acl + })) + dest_cont2 = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont2.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl, + 'X-Container-Read': self.env.conn.user_acl + })) + + for acct, cont in ((acct, dest_cont), (acct2, dest_cont2)): + for prefix in ('', '/'): + # invalid source container + source_cont = self.env.account.container(Utils.create_name()) + file_item = source_cont.file(source_filename) + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, self.env.container), + Utils.create_name())) + if acct == acct2: + # there is no such source container + # and foreign user can have no permission to read it + self.assert_status(403) + else: + self.assert_status(404) + + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, cont), + Utils.create_name())) + self.assert_status(404) + + # invalid source object + file_item = self.env.container.file(Utils.create_name()) + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, self.env.container), + Utils.create_name())) + if acct == acct2: + # there is no such object + # and foreign user can have no permission to read it + self.assert_status(403) + else: + self.assert_status(404) + + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, cont), + Utils.create_name())) + self.assert_status(404) + + # invalid destination container + file_item = self.env.container.file(source_filename) + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, Utils.create_name()), + Utils.create_name())) + if acct == acct2: + # there is no such destination container + # and foreign user can have no permission to write there + self.assert_status(403) + else: + self.assert_status(404) + def testCopyNoDestinationHeader(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) @@ -942,6 +1092,49 @@ class TestFile(Base): self.assert_(file_item.initialize()) self.assert_(metadata == file_item.metadata) + def testCopyFromAccountHeader(self): + acct = self.env.conn.account_name + src_cont = self.env.account.container(Utils.create_name()) + self.assert_(src_cont.create(hdrs={ + 'X-Container-Read': self.env.conn2.user_acl + })) + source_filename = Utils.create_name() + file_item = src_cont.file(source_filename) + + metadata = {} + for i in range(1): + metadata[Utils.create_ascii_name()] = Utils.create_name() + file_item.metadata = metadata + + data = file_item.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + dest_cont2 = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont2.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + + for cont in (src_cont, dest_cont, dest_cont2): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file_item = cont.file(dest_filename) + file_item.write(hdrs={'X-Copy-From-Account': acct, + 'X-Copy-From': '%s%s/%s' % ( + prefix, + src_cont.name, + source_filename)}) + + self.assert_(dest_filename in cont.files()) + + file_item = cont.file(dest_filename) + + self.assert_(data == file_item.read()) + self.assert_(file_item.initialize()) + self.assert_(metadata == file_item.metadata) + def testCopyFromHeader404s(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) @@ -973,6 +1166,52 @@ class TestFile(Base): self.env.container.name, source_filename)}) self.assert_status(404) + def testCopyFromAccountHeader404s(self): + acct = self.env.conn2.account_name + src_cont = self.env.account2.container(Utils.create_name()) + self.assert_(src_cont.create(hdrs={ + 'X-Container-Read': self.env.conn.user_acl + })) + source_filename = Utils.create_name() + file_item = src_cont.file(source_filename) + file_item.write_random() + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + for prefix in ('', '/'): + # invalid source container + file_item = dest_cont.file(Utils.create_name()) + self.assertRaises(ResponseError, file_item.write, + hdrs={'X-Copy-From-Account': acct, + 'X-Copy-From': '%s%s/%s' % + (prefix, + Utils.create_name(), + source_filename)}) + # looks like cached responses leak "not found" + # to un-authorized users, not going to fix it now, but... + self.assert_status([403, 404]) + + # invalid source object + file_item = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file_item.write, + hdrs={'X-Copy-From-Account': acct, + 'X-Copy-From': '%s%s/%s' % + (prefix, + src_cont, + Utils.create_name())}) + self.assert_status(404) + + # invalid destination container + dest_cont = self.env.account.container(Utils.create_name()) + file_item = dest_cont.file(Utils.create_name()) + self.assertRaises(ResponseError, file_item.write, + hdrs={'X-Copy-From-Account': acct, + 'X-Copy-From': '%s%s/%s' % + (prefix, + src_cont, + source_filename)}) + self.assert_status(404) + def testNameLimit(self): raise SkipTest('SOF constraints middleware enforces constraints.') @@ -1196,6 +1435,16 @@ class TestFile(Base): cfg={'no_content_length': True}) self.assert_status(400) + # no content-length + self.assertRaises(ResponseError, file_item.write_random, file_length, + cfg={'no_content_length': True}) + self.assert_status(411) + + self.assertRaises(ResponseError, file_item.write_random, file_length, + hdrs={'transfer-encoding': 'gzip,chunked'}, + cfg={'no_content_length': True}) + self.assert_status(501) + # bad request types #for req in ('LICK', 'GETorHEAD_base', 'container_info', # 'best_response'): @@ -1598,6 +1847,30 @@ class TestDlo(Base): file_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") + def test_copy_account(self): + # dlo use same account and same container only + acct = self.env.conn.account_name + # Adding a new segment, copying the manifest, and then deleting the + # segment proves that the new object is really the concatenated + # segments and not just a manifest. + f_segment = self.env.container.file("%s/seg_lowerf" % + (self.env.segment_prefix)) + f_segment.write('ffffffffff') + try: + man1_item = self.env.container.file('man1') + man1_item.copy_account(acct, + self.env.container.name, + "copied-man1") + finally: + # try not to leave this around for other tests to stumble over + f_segment.delete() + + file_item = self.env.container.file('copied-man1') + file_contents = file_item.read() + self.assertEqual( + file_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") + def test_copy_manifest(self): # Copying the manifest should result in another manifest try: @@ -1794,6 +2067,14 @@ class TestSloEnv(object): def setUp(cls): cls.conn = Connection(tf.config) cls.conn.authenticate() + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() if cls.slo_enabled is None: cls.slo_enabled = 'slo' in cluster_info @@ -1976,6 +2257,29 @@ class TestSlo(Base): copied_contents = copied.read(parms={'multipart-manifest': 'get'}) self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + def test_slo_copy_account(self): + acct = self.env.conn.account_name + # same account copy + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, self.env.container.name, "copied-abcde") + + copied = self.env.container.file("copied-abcde") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + + # copy to different account + acct = self.env.conn2.account_name + dest_cont = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, dest_cont, "copied-abcde") + + copied = dest_cont.file("copied-abcde") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + def test_slo_copy_the_manifest(self): file_item = self.env.container.file("manifest-abcde") file_item.copy(self.env.container.name, "copied-abcde-manifest-only", @@ -1988,6 +2292,40 @@ class TestSlo(Base): except ValueError: self.fail("COPY didn't copy the manifest (invalid json on GET)") + def test_slo_copy_the_manifest_account(self): + acct = self.env.conn.account_name + # same account + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, + self.env.container.name, + "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + + copied = self.env.container.file("copied-abcde-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + + # different account + acct = self.env.conn2.account_name + dest_cont = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + file_item.copy_account(acct, + dest_cont, + "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + + copied = dest_cont.file("copied-abcde-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + def test_slo_get_the_manifest(self): manifest = self.env.container.file("manifest-abcde") got_body = manifest.read(parms={'multipart-manifest': 'get'}) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py deleted file mode 100644 index f9737dd..0000000 --- a/test/unit/obj/test_server.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Tests for swiftonfile.swift.obj.server subclass """ - -import unittest -from nose import SkipTest - -import swiftonfile.swift.obj.server as server - - -class TestObjServer(unittest.TestCase): - """ - Tests for object server subclass. - """ - - def test_constructor(self): - raise SkipTest diff --git a/test/unit/test_swift.py b/test/unit/test_sof_pkg.py similarity index 100% rename from test/unit/test_swift.py rename to test/unit/test_sof_pkg.py diff --git a/tox.ini b/tox.ini index 60f94f2..764b903 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ deps = # Note: pip supports installing from git repos. # https://pip.pypa.io/en/latest/reference/pip_install.html#git # Example: git+https://github.com/openstack/swift.git@2.0.0 - https://launchpad.net/swift/juno/2.1.0/+download/swift-2.1.0.tar.gz + https://launchpad.net/swift/kilo/2.2.1/+download/swift-2.2.1.tar.gz -r{toxinidir}/test-requirements.txt changedir = {toxinidir}/test/unit commands = nosetests -v {posargs}