openstack/swift
Code Issues Proposed changes

Fixes probe tests with non-zero default storage policy

Browse Source 
Add headers param to direct_client.direct_get_object, which is used in
probetests to passthrough the X-Storage-Policy-Index header.

DocImpact
Implements: blueprint storage-policies
Change-Id: I19adbbcefbc086c8467bd904a275d55cde596412
This commit is contained in:
Yuan Zhou 2014-04-15 16:06:22 +08:00 committed by Clay Gerrard
parent fbcfb83566
commit ad2a9cefe5
10 changed files with 120 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
test/probe/common.py
View File

@ -19,12 +19,14 @@ from subprocess import Popen, PIPE
import sys
from time import sleep, time
from collections import defaultdict
from nose import SkipTest
from swiftclient import get_auth, head_account
from swift.common.ring import Ring
from swift.common.utils import readconf
from swift.common.manager import Manager
from swift.common.storage_policy import POLICIES
from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC
@ -136,13 +138,16 @@ def kill_nonprimary_server(primary_nodes, port2server, pids):
return port
def get_ring(server, force_validate=None):
ring = Ring('/etc/swift/%s.ring.gz' % server)
def get_ring(ring_name, server=None, force_validate=None):
if not server:
server = ring_name
ring = Ring('/etc/swift', ring_name=ring_name)
if not VALIDATE_RSYNC and not force_validate:
return ring
# easy sanity checks
assert 3 == ring.replica_count, '%s has %s replicas instead of 3' % (
ring.serialized_path, ring.replica_count)
if ring.replica_count != 3:
print 'WARNING: %s has %s replicas instead of 3' % (
ring.serialized_path, ring.replica_count)
assert 4 == len(ring.devs), '%s has %s devices instead of 4' % (
ring.serialized_path, len(ring.devs))
# map server to config by port
@ -197,7 +202,8 @@ def reset_environment():
try:
account_ring = get_ring('account')
container_ring = get_ring('container')
object_ring = get_ring('object')
policy = POLICIES.default
object_ring = get_ring(policy.ring_name, 'object')
Manager(['main']).start(wait=False)
port2server = {}
for server, port in [('account', 6002), ('container', 6001),
@ -218,15 +224,14 @@ def reset_environment():
try:
raise
except AssertionError as e:
print >>sys.stderr, 'ERROR: %s' % e
os._exit(1)
raise SkipTest(e)
finally:
try:
kill_servers(port2server, pids)
except Exception:
pass
return pids, port2server, account_ring, container_ring, object_ring, url, \
token, account, config_dict
return pids, port2server, account_ring, container_ring, object_ring, \
policy, url, token, account, config_dict
def get_to_final_state():
@ -242,6 +247,15 @@ def get_to_final_state():
if __name__ == "__main__":
for server in ('account', 'container', 'object'):
get_ring(server, force_validate=True)
for server in ('account', 'container'):
try:
get_ring(server, force_validate=True)
except AssertionError as err:
sys.exit('%s ERROR: %s' % (server, err))
print '%s OK' % server
for policy in POLICIES:
try:
get_ring(policy.ring_name, server='object', force_validate=True)
except AssertionError as err:
sys.exit('object ERROR (%s): %s' % (policy.name, err))
print 'object OK (%s)' % policy.name

2
test/probe/test_account_failures.py
View File

@ -28,7 +28,7 @@ class TestAccountFailures(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):

2
test/probe/test_account_get_fake_responses_match.py
View File

@ -27,7 +27,7 @@ class TestAccountGetFakeResponsesMatch(unittest.TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
self.url, self.token = get_auth(
'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin')

2
test/probe/test_container_failures.py
View File

@ -44,7 +44,7 @@ class TestContainerFailures(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):

2
test/probe/test_container_merge_policy_index.py
View File

@ -179,7 +179,7 @@ class TestContainerMergePolicyIndex(unittest.TestCase):
if len(POLICIES) < 2:
raise SkipTest()
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()

27
test/probe/test_empty_device_handoff.py
View File

@ -24,6 +24,8 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.storage_policy import POLICY_INDEX
from swift.obj.diskfile import get_data_dir
from swift.common.exceptions import ClientException
from test.probe.common import kill_server, kill_servers, reset_environment,\
start_server
@ -35,7 +37,7 @@ class TestEmptyDevice(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
@ -52,7 +54,7 @@ class TestEmptyDevice(TestCase):
def test_main(self):
# Create container
# Kill one container/obj primary server
# Delete the "objects" directory on the primary server
# Delete the default data directory for objects on the primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
@ -76,7 +78,8 @@ class TestEmptyDevice(TestCase):
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
obj_dir = '%s/objects' % self._get_objects_dir(onode)
obj_dir = '%s/%s' % (self._get_objects_dir(onode),
get_data_dir(self.policy.idx))
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
client.put_object(self.url, self.token, container, obj, 'VERIFY')
@ -98,7 +101,8 @@ class TestEmptyDevice(TestCase):
# let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
another_onode, opart, self.account, container, obj,
headers={POLICY_INDEX: self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
@ -128,8 +132,9 @@ class TestEmptyDevice(TestCase):
self.assertFalse(os.path.exists(obj_dir))
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
@ -150,15 +155,17 @@ class TestEmptyDevice(TestCase):
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)

2
test/probe/test_object_async_update.py
View File

@ -29,7 +29,7 @@ class TestObjectAsyncUpdate(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):

46
test/probe/test_object_failures.py
View File

@ -23,9 +23,10 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.storage_policy import POLICY_INDEX
from swift.common.exceptions import ClientException
from swift.common.utils import hash_path, readconf
from swift.obj.diskfile import write_metadata, read_metadata
from swift.obj.diskfile import write_metadata, read_metadata, get_data_dir
from test.probe.common import kill_servers, reset_environment
@ -53,7 +54,7 @@ class TestObjectFailures(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
@ -72,9 +73,9 @@ class TestObjectFailures(TestCase):
hash_str = hash_path(self.account, container, obj)
obj_server_conf = readconf(self.configs['object-server'][node_id])
devices = obj_server_conf['app:object-server']['devices']
obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices,
device, opart,
hash_str[-3:], hash_str)
obj_dir = '%s/%s/%s/%s/%s/%s/' % (devices, device,
get_data_dir(self.policy.idx),
opart, hash_str[-3:], hash_str)
data_file = get_data_file_path(obj_dir)
return onode, opart, data_file
@ -88,11 +89,13 @@ class TestObjectFailures(TestCase):
write_metadata(data_file, metadata)
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj)[-1]
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})[-1]
self.assertEquals(odata, 'VERIFY')
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
raise Exception("Did not quarantine object")
except ClientException as err:
self.assertEquals(err.http_status, 404)
@ -106,16 +109,21 @@ class TestObjectFailures(TestCase):
metadata = read_metadata(data_file)
metadata['ETag'] = 'badetag'
write_metadata(data_file, metadata)
base_headers = {POLICY_INDEX: self.policy.idx}
for header, result in [({'Range': 'bytes=0-2'}, 'RAN'),
({'Range': 'bytes=1-11'}, 'ANGE'),
({'Range': 'bytes=0-11'}, 'RANGE')]:
req_headers = base_headers.copy()
req_headers.update(header)
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers=header)[-1]
onode, opart, self.account, container, obj,
headers=req_headers)[-1]
self.assertEquals(odata, result)
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
raise Exception("Did not quarantine object")
except ClientException as err:
self.assertEquals(err.http_status, 404)
@ -130,9 +138,9 @@ class TestObjectFailures(TestCase):
with open(data_file, 'w') as fpointer:
write_metadata(fpointer, metadata)
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj, conn_timeout=1,
response_timeout=1)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, conn_timeout=1,
response_timeout=1, headers={POLICY_INDEX: self.policy.idx})
raise Exception("Did not quarantine object")
except ClientException as err:
self.assertEquals(err.http_status, 404)
@ -147,9 +155,9 @@ class TestObjectFailures(TestCase):
with open(data_file, 'w') as fpointer:
write_metadata(fpointer, metadata)
try:
direct_client.direct_head_object(onode, opart, self.account,
container, obj, conn_timeout=1,
response_timeout=1)
direct_client.direct_head_object(
onode, opart, self.account, container, obj, conn_timeout=1,
response_timeout=1, headers={POLICY_INDEX: self.policy.idx})
raise Exception("Did not quarantine object")
except ClientException as err:
self.assertEquals(err.http_status, 404)
@ -164,10 +172,12 @@ class TestObjectFailures(TestCase):
with open(data_file, 'w') as fpointer:
write_metadata(fpointer, metadata)
try:
headers = {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two',
POLICY_INDEX: self.policy.idx}
direct_client.direct_post_object(
onode, opart, self.account,
container, obj,
{'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'},
headers=headers,
conn_timeout=1,
response_timeout=1)
raise Exception("Did not quarantine object")

41
test/probe/test_object_handoff.py
View File

@ -20,6 +20,7 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.storage_policy import POLICY_INDEX
from swift.common.exceptions import ClientException
from swift.common.manager import Manager
from test.probe.common import kill_server, kill_servers, reset_environment, \
@ -30,7 +31,7 @@ class TestObjectHandoff(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
@ -90,7 +91,8 @@ class TestObjectHandoff(TestCase):
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
another_onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
@ -109,8 +111,9 @@ class TestObjectHandoff(TestCase):
start_server(onode['port'], self.port2server, self.pids)
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
@ -128,21 +131,31 @@ class TestObjectHandoff(TestCase):
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
kill_server(onode['port'], self.port2server, self.pids)
client.delete_object(self.url, self.token, container, obj)
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
exc = None
try:
client.head_object(self.url, self.token, container, obj)
@ -162,8 +175,9 @@ class TestObjectHandoff(TestCase):
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
direct_client.direct_get_object(onode, opart, self.account, container,
obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
# Run the extra server last so it'll remove its extra partition
for node in onodes:
try:
@ -176,8 +190,9 @@ class TestObjectHandoff(TestCase):
Manager(['object-replicator']).once(number=another_node_id)
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)

28
test/probe/test_replication_servers_working.py
View File

@ -21,6 +21,8 @@ import time
import shutil
from swiftclient import client
from swift.common.storage_policy import POLICIES
from swift.obj.diskfile import get_data_dir
from test.probe.common import kill_servers, reset_environment
from swift.common.utils import readconf
@ -80,7 +82,7 @@ class TestReplicatorFunctions(TestCase):
Reset all environment and start all servers.
"""
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
@ -100,6 +102,7 @@ class TestReplicatorFunctions(TestCase):
# Delete file "hashes.pkl".
# Check, that all files were replicated.
path_list = []
data_dir = get_data_dir(POLICIES.default.idx)
# Figure out where the devices are
for node_id in range(1, 5):
conf = readconf(self.configs['object-server'][node_id])
@ -124,7 +127,10 @@ class TestReplicatorFunctions(TestCase):
for files in files_list[num]:
if not files.endswith('.pending'):
test_node_files_list.append(files)
test_node_dir_list = dir_list[num]
test_node_dir_list = []
for d in dir_list[num]:
if not d.startswith('tmp'):
test_node_dir_list.append(d)
# Run all replicators
try:
Manager(['object-replicator', 'container-replicator',
@ -155,24 +161,24 @@ class TestReplicatorFunctions(TestCase):
time.sleep(1)
# Check behavior by deleting hashes.pkl file
for directory in os.listdir(os.path.join(test_node, 'objects')):
for directory in os.listdir(os.path.join(test_node, data_dir)):
for input_dir in os.listdir(os.path.join(
test_node, 'objects', directory)):
test_node, data_dir, directory)):
if os.path.isdir(os.path.join(
test_node, 'objects', directory, input_dir)):
test_node, data_dir, directory, input_dir)):
shutil.rmtree(os.path.join(
test_node, 'objects', directory, input_dir))
test_node, data_dir, directory, input_dir))
# We will keep trying these tests until they pass for up to 60s
begin = time.time()
while True:
try:
for directory in os.listdir(os.path.join(
test_node, 'objects')):
test_node, data_dir)):
for input_dir in os.listdir(os.path.join(
test_node, 'objects', directory)):
test_node, data_dir, directory)):
self.assertFalse(os.path.isdir(
os.path.join(test_node, 'objects',
os.path.join(test_node, data_dir,
directory, '/', input_dir)))
break
except Exception:
@ -180,9 +186,9 @@ class TestReplicatorFunctions(TestCase):
raise
time.sleep(1)
for directory in os.listdir(os.path.join(test_node, 'objects')):
for directory in os.listdir(os.path.join(test_node, data_dir)):
os.remove(os.path.join(
test_node, 'objects', directory, 'hashes.pkl'))
test_node, data_dir, directory, 'hashes.pkl'))
# We will keep trying these tests until they pass for up to 60s
begin = time.time()