Make tempurl functional tests clean up account keys

Addresses a TODO in test/functional/test_account.py where
an account metadata test was having to clean up tempurl keys
in the account metadata that were left by another test in
a different module. This cleanup is necessary because tests
in test_account.py fail if there is any pre-existing
account metadata.

This patch:

* makes the tempurl tests clean up their keys from account
  metadata.

* makes the test_account.py:TestAccount class remove any
  pre-existing metadata before attempting any tests and
  replacing that metadata when all the tests in that class
  have completed. This is more robust than the existing code
  which only removes any tempurl keys that might be in the
  account - now you could have x-account-meta-foo = bar in
  the test account and test_account.py will still pass.

* consolidates some common setup code currently repeated for
  many of the functional test classes into into a BaseEnv class.

Change-Id: I874a9e23dfcdd1caa934945b46089f11b9f6de65
This commit is contained in:
Alistair Coles 2016-07-25 13:50:24 +01:00 committed by Tim Burke
parent c0640f8710
commit 9f30c5d31e
7 changed files with 165 additions and 203 deletions

View File

@ -482,8 +482,11 @@ class Account(Base):
fields = [['object_count', 'x-account-object-count'],
['container_count', 'x-account-container-count'],
['bytes_used', 'x-account-bytes-used']]
optional_fields = [
['temp-url-key', 'x-account-meta-temp-url-key'],
['temp-url-key-2', 'x-account-meta-temp-url-key-2']]
return self.header_fields(fields)
return self.header_fields(fields, optional_fields=optional_fields)
@property
def path(self):

View File

@ -38,6 +38,54 @@ def tearDownModule():
class TestAccount(unittest2.TestCase):
existing_metadata = None
@classmethod
def get_meta(cls):
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(head)
resp.read()
return dict((k, v) for k, v in resp.getheaders() if
k.lower().startswith('x-account-meta'))
@classmethod
def clear_meta(cls, remove_metadata_keys):
def post(url, token, parsed, conn, hdr_keys):
headers = {'X-Auth-Token': token}
headers.update((k, '') for k in hdr_keys)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
for i in range(0, len(remove_metadata_keys), 90):
batch = remove_metadata_keys[i:i + 90]
resp = retry(post, batch)
resp.read()
@classmethod
def set_meta(cls, metadata):
def post(url, token, parsed, conn, meta_hdrs):
headers = {'X-Auth-Token': token}
headers.update(meta_hdrs)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
if not metadata:
return
resp = retry(post, metadata)
resp.read()
@classmethod
def setUpClass(cls):
# remove and stash any existing account user metadata before tests
cls.existing_metadata = cls.get_meta()
cls.clear_meta(cls.existing_metadata.keys())
@classmethod
def tearDownClass(cls):
# replace any stashed account user metadata
cls.set_meta(cls.existing_metadata)
def setUp(self):
self.max_meta_count = load_constraint('max_meta_count')
@ -45,35 +93,10 @@ class TestAccount(unittest2.TestCase):
self.max_meta_overall_size = load_constraint('max_meta_overall_size')
self.max_meta_value_length = load_constraint('max_meta_value_length')
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(head)
self.existing_metadata = set([
k for k, v in resp.getheaders() if
k.lower().startswith('x-account-meta')])
def tearDown(self):
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(head)
resp.read()
new_metadata = set(
[k for k, v in resp.getheaders() if
k.lower().startswith('x-account-meta')])
def clear_meta(url, token, parsed, conn, remove_metadata_keys):
headers = {'X-Auth-Token': token}
headers.update((k, '') for k in remove_metadata_keys)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
extra_metadata = list(self.existing_metadata ^ new_metadata)
for i in range(0, len(extra_metadata), 90):
batch = extra_metadata[i:i + 90]
resp = retry(clear_meta, batch)
resp.read()
self.assertEqual(resp.status // 100, 2)
# clean up any account user metadata created by test
new_metadata = self.get_meta().keys()
self.clear_meta(new_metadata)
def test_metadata(self):
if tf.skip:
@ -794,11 +817,6 @@ class TestAccount(unittest2.TestCase):
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
# TODO: Find the test that adds these and remove them.
headers = {'x-remove-account-meta-temp-url-key': 'remove',
'x-remove-account-meta-temp-url-key-2': 'remove'}
resp = retry(post, headers)
headers = {}
for x in range(self.max_meta_count):
headers['X-Account-Meta-%d' % x] = 'v'

View File

@ -15,9 +15,8 @@
# limitations under the License.
import test.functional as tf
from test.functional.tests import Utils, Base, Base2
from test.functional.swift_test_client import Account, Connection, \
ResponseError
from test.functional.tests import Utils, Base, Base2, BaseEnv
from test.functional.swift_test_client import Connection, ResponseError
def setUpModule():
@ -28,22 +27,16 @@ def tearDownModule():
tf.teardown_package()
class TestDloEnv(object):
class TestDloEnv(BaseEnv):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
super(TestDloEnv, cls).setUp()
config2 = tf.config.copy()
config2['username'] = tf.config['username3']
config2['password'] = tf.config['password3']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
cls.container2 = cls.account.container(Utils.create_name())
@ -92,7 +85,6 @@ class TestDloEnv(object):
class TestDlo(Base):
env = TestDloEnv
set_up = False
def test_get_manifest(self):
file_item = self.env.container.file('man1')
@ -394,4 +386,4 @@ class TestDlo(Base):
class TestDloUTF8(Base2, TestDlo):
set_up = False
pass

View File

@ -23,9 +23,8 @@ from unittest2 import SkipTest
import test.functional as tf
from test.functional import cluster_info
from test.functional.tests import Utils, Base, Base2
from test.functional.swift_test_client import Account, Connection, \
ResponseError
from test.functional.tests import Utils, Base, Base2, BaseEnv
from test.functional.swift_test_client import Connection, ResponseError
def setUpModule():
@ -36,7 +35,7 @@ def tearDownModule():
tf.teardown_package()
class TestSloEnv(object):
class TestSloEnv(BaseEnv):
slo_enabled = None # tri-state: None initially, then True/False
@classmethod
@ -58,8 +57,13 @@ class TestSloEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.slo_enabled is None:
cls.slo_enabled = 'slo' in cluster_info
if not cls.slo_enabled:
return
super(TestSloEnv, cls).setUp()
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
@ -74,15 +78,6 @@ class TestSloEnv(object):
cls.conn3 = Connection(config3)
cls.conn3.authenticate()
if cls.slo_enabled is None:
cls.slo_enabled = 'slo' in cluster_info
if not cls.slo_enabled:
return
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
cls.container2 = cls.account.container(Utils.create_name())
@ -214,7 +209,6 @@ class TestSloEnv(object):
class TestSlo(Base):
env = TestSloEnv
set_up = False
def setUp(self):
super(TestSlo, self).setUp()
@ -963,4 +957,4 @@ class TestSlo(Base):
class TestSloUTF8(Base2, TestSlo):
set_up = False
pass

View File

@ -24,7 +24,7 @@ from unittest2 import SkipTest
import test.functional as tf
from test.functional import cluster_info
from test.functional.tests import Utils, Base, Base2
from test.functional.tests import Utils, Base, Base2, BaseEnv
from test.functional import requires_acls
from test.functional.swift_test_client import Account, Connection, \
ResponseError
@ -38,25 +38,38 @@ def tearDownModule():
tf.teardown_package()
class TestTempurlEnv(object):
class TestTempurlBaseEnv(BaseEnv):
original_account_meta = None
@classmethod
def setUp(cls):
super(TestTempurlBaseEnv, cls).setUp()
cls.original_account_meta = cls.account.info()
@classmethod
def tearDown(cls):
if cls.original_account_meta:
# restore any tempurl keys that the tests may have overwritten
cls.account.update_metadata(
dict((k, cls.original_account_meta.get(k, ''))
for k in ('temp-url-key', 'temp-url-key-2',)))
class TestTempurlEnv(TestTempurlBaseEnv):
tempurl_enabled = None # tri-state: None initially, then True/False
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.tempurl_enabled is None:
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
super(TestTempurlEnv, cls).setUp()
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
cls.account = Account(
cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.account.update_metadata({
'temp-url-key': cls.tempurl_key,
'temp-url-key-2': cls.tempurl_key2
@ -74,7 +87,6 @@ class TestTempurlEnv(object):
class TestTempurl(Base):
env = TestTempurlEnv
set_up = False
def setUp(self):
super(TestTempurl, self).setUp()
@ -376,29 +388,24 @@ class TestTempURLPrefix(TestTempurl):
class TestTempurlUTF8(Base2, TestTempurl):
set_up = False
pass
class TestContainerTempurlEnv(object):
class TestContainerTempurlEnv(BaseEnv):
tempurl_enabled = None # tri-state: None initially, then True/False
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.tempurl_enabled is None:
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
super(TestContainerTempurlEnv, cls).setUp()
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
cls.account = Account(
cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
# creating another account and connection
# for ACL tests
config2 = deepcopy(tf.config)
@ -426,7 +433,6 @@ class TestContainerTempurlEnv(object):
class TestContainerTempurl(Base):
env = TestContainerTempurlEnv
set_up = False
def setUp(self):
super(TestContainerTempurl, self).setUp()
@ -647,25 +653,20 @@ class TestContainerTempurl(Base):
class TestContainerTempurlUTF8(Base2, TestContainerTempurl):
set_up = False
pass
class TestSloTempurlEnv(object):
class TestSloTempurlEnv(TestTempurlBaseEnv):
enabled = None # tri-state: None initially, then True/False
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
super(TestSloTempurlEnv, cls).setUp()
if cls.enabled is None:
cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info
cls.tempurl_key = Utils.create_name()
cls.account = Account(
cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.account.update_metadata({'temp-url-key': cls.tempurl_key})
cls.manifest_container = cls.account.container(Utils.create_name())
@ -698,7 +699,6 @@ class TestSloTempurlEnv(object):
class TestSloTempurl(Base):
env = TestSloTempurlEnv
set_up = False
def setUp(self):
super(TestSloTempurl, self).setUp()
@ -734,4 +734,4 @@ class TestSloTempurl(Base):
class TestSloTempurlUTF8(Base2, TestSloTempurl):
set_up = False
pass

View File

@ -16,11 +16,12 @@
import json
import time
import unittest2
from unittest2 import SkipTest
import test.functional as tf
from copy import deepcopy
from test.functional.tests import Base, Base2, Utils
from test.functional.tests import Base, Base2, BaseEnv, Utils
from test.functional import cluster_info
from test.functional.swift_test_client import Account, Connection, \
ResponseError
@ -34,18 +35,14 @@ def tearDownModule():
tf.teardown_package()
class TestObjectVersioningEnv(object):
class TestObjectVersioningEnv(BaseEnv):
versioning_enabled = None # tri-state: None initially, then True/False
location_header_key = 'X-Versions-Location'
account2 = None
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.storage_url, cls.storage_token = cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
super(TestObjectVersioningEnv, cls).setUp()
# Second connection for ACL tests
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
@ -96,22 +93,23 @@ class TestObjectVersioningEnv(object):
@classmethod
def tearDown(cls):
cls.account.delete_containers()
cls.account2.delete_containers()
if cls.account:
cls.account.delete_containers()
if cls.account2:
cls.account2.delete_containers()
class TestCrossPolicyObjectVersioningEnv(object):
class TestCrossPolicyObjectVersioningEnv(BaseEnv):
# tri-state: None initially, then True/False
versioning_enabled = None
multiple_policies_enabled = None
policies = None
location_header_key = 'X-Versions-Location'
account2 = None
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
super(TestCrossPolicyObjectVersioningEnv, cls).setUp()
if cls.multiple_policies_enabled is None:
try:
cls.policies = tf.FunctionalStoragePolicyCollection.from_info()
@ -131,9 +129,6 @@ class TestCrossPolicyObjectVersioningEnv(object):
policy = cls.policies.select()
version_policy = cls.policies.exclude(name=policy['name']).select()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
# Second connection for ACL tests
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
@ -185,8 +180,10 @@ class TestCrossPolicyObjectVersioningEnv(object):
@classmethod
def tearDown(cls):
cls.account.delete_containers()
cls.account2.delete_containers()
if cls.account:
cls.account.delete_containers()
if cls.account2:
cls.account2.delete_containers()
class TestObjectVersioningHistoryModeEnv(TestObjectVersioningEnv):
@ -195,7 +192,6 @@ class TestObjectVersioningHistoryModeEnv(TestObjectVersioningEnv):
class TestObjectVersioning(Base):
env = TestObjectVersioningEnv
set_up = False
def setUp(self):
super(TestObjectVersioning, self).setUp()
@ -487,7 +483,6 @@ class TestObjectVersioning(Base):
class TestObjectVersioningUTF8(Base2, TestObjectVersioning):
set_up = False
def tearDown(self):
self._tear_down_files()
@ -496,7 +491,6 @@ class TestObjectVersioningUTF8(Base2, TestObjectVersioning):
class TestCrossPolicyObjectVersioning(TestObjectVersioning):
env = TestCrossPolicyObjectVersioningEnv
set_up = False
def setUp(self):
super(TestCrossPolicyObjectVersioning, self).setUp()
@ -511,7 +505,6 @@ class TestCrossPolicyObjectVersioning(TestObjectVersioning):
class TestObjectVersioningHistoryMode(TestObjectVersioning):
env = TestObjectVersioningHistoryModeEnv
set_up = False
# those override tests includes assertions for delete versioned objects
# behaviors different from default object versioning using
@ -695,7 +688,7 @@ class TestObjectVersioningHistoryMode(TestObjectVersioning):
self.assertEqual(expected, prev_version.read())
class TestSloWithVersioning(Base):
class TestSloWithVersioning(unittest2.TestCase):
def setUp(self):
if 'slo' not in cluster_info:

View File

@ -67,12 +67,33 @@ class Utils(object):
create_name = create_ascii_name
class BaseEnv(object):
account = conn = None
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
@classmethod
def tearDown(cls):
pass
class Base(unittest2.TestCase):
def setUp(self):
cls = type(self)
if not cls.set_up:
cls.env.setUp()
cls.set_up = True
# subclasses may override env class
env = BaseEnv
@classmethod
def setUpClass(cls):
cls.env.setUp()
@classmethod
def tearDownClass(cls):
cls.env.tearDown()
def assert_body(self, body):
response_body = self.env.conn.response.read()
@ -105,15 +126,10 @@ class Base2(object):
Utils.create_name = Utils.create_ascii_name
class TestAccountEnv(object):
class TestAccountEnv(BaseEnv):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
super(TestAccountEnv, cls).setUp()
cls.containers = []
for i in range(10):
cont = cls.account.container(Utils.create_name())
@ -125,16 +141,14 @@ class TestAccountEnv(object):
class TestAccountDev(Base):
env = TestAccountEnv
set_up = False
class TestAccountDevUTF8(Base2, TestAccountDev):
set_up = False
pass
class TestAccount(Base):
env = TestAccountEnv
set_up = False
def testNoAuthToken(self):
self.assertRaises(ResponseError, self.env.account.info,
@ -352,23 +366,10 @@ class TestAccount(Base):
class TestAccountUTF8(Base2, TestAccount):
set_up = False
class TestAccountNoContainersEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
pass
class TestAccountNoContainers(Base):
env = TestAccountNoContainersEnv
set_up = False
def testGetRequest(self):
for format_type in [None, 'json', 'xml']:
self.assertFalse(self.env.account.containers(
@ -381,18 +382,13 @@ class TestAccountNoContainers(Base):
class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):
set_up = False
pass
class TestAccountSortingEnv(object):
class TestAccountSortingEnv(BaseEnv):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
super(TestAccountSortingEnv, cls).setUp()
postfix = Utils.create_name()
cls.cont_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz')
cls.cont_items = ['%s%s' % (x, postfix) for x in cls.cont_items]
@ -405,7 +401,6 @@ class TestAccountSortingEnv(object):
class TestAccountSorting(Base):
env = TestAccountSortingEnv
set_up = False
def testAccountContainerListSorting(self):
# name (byte order) sorting.
@ -470,15 +465,10 @@ class TestAccountSorting(Base):
self.assertEqual([], cont_listing)
class TestContainerEnv(object):
class TestContainerEnv(BaseEnv):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
super(TestContainerEnv, cls).setUp()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
@ -494,16 +484,14 @@ class TestContainerEnv(object):
class TestContainerDev(Base):
env = TestContainerEnv
set_up = False
class TestContainerDevUTF8(Base2, TestContainerDev):
set_up = False
pass
class TestContainer(Base):
env = TestContainerEnv
set_up = False
def testContainerNameLimit(self):
limit = load_constraint('max_container_name_length')
@ -889,18 +877,13 @@ class TestContainer(Base):
class TestContainerUTF8(Base2, TestContainer):
set_up = False
pass
class TestContainerSortingEnv(object):
class TestContainerSortingEnv(BaseEnv):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
super(TestContainerSortingEnv, cls).setUp()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
@ -916,7 +899,6 @@ class TestContainerSortingEnv(object):
class TestContainerSorting(Base):
env = TestContainerSortingEnv
set_up = False
def testContainerFileListSortingReversed(self):
file_list = list(sorted(self.env.file_items))
@ -1004,15 +986,10 @@ class TestContainerSorting(Base):
self.assertEqual(file_list, cont_files)
class TestContainerPathsEnv(object):
class TestContainerPathsEnv(BaseEnv):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
super(TestContainerPathsEnv, cls).setUp()
cls.file_size = 8
cls.container = cls.account.container(Utils.create_name())
@ -1082,7 +1059,6 @@ class TestContainerPathsEnv(object):
class TestContainerPaths(Base):
env = TestContainerPathsEnv
set_up = False
def testTraverseContainer(self):
found_files = []
@ -1183,13 +1159,10 @@ class TestContainerPaths(Base):
['dir1/subdir with spaces/file B'])
class TestFileEnv(object):
class TestFileEnv(BaseEnv):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
super(TestFileEnv, cls).setUp()
# creating another account and connection
# for account to account copy tests
config2 = deepcopy(tf.config)
@ -1199,9 +1172,6 @@ class TestFileEnv(object):
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
cls.account2 = cls.conn2.get_account()
cls.account2.delete_containers()
@ -1223,16 +1193,14 @@ class TestFileEnv(object):
class TestFileDev(Base):
env = TestFileEnv
set_up = False
class TestFileDevUTF8(Base2, TestFileDev):
set_up = False
pass
class TestFile(Base):
env = TestFileEnv
set_up = False
def testCopy(self):
# makes sure to test encoded characters
@ -2505,18 +2473,13 @@ class TestFile(Base):
class TestFileUTF8(Base2, TestFile):
set_up = False
pass
class TestFileComparisonEnv(object):
class TestFileComparisonEnv(BaseEnv):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
super(TestFileComparisonEnv, cls).setUp()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
@ -2542,7 +2505,6 @@ class TestFileComparisonEnv(object):
class TestFileComparison(Base):
env = TestFileComparisonEnv
set_up = False
def testIfMatch(self):
for file_item in self.env.files:
@ -2662,7 +2624,7 @@ class TestFileComparison(Base):
class TestFileComparisonUTF8(Base2, TestFileComparison):
set_up = False
pass
class TestServiceToken(unittest2.TestCase):