Fixes probe tests with non-zero default storage policy

Add headers param to direct_client.direct_get_object, which is used in
probetests to passthrough the X-Storage-Policy-Index header.

DocImpact
Implements: blueprint storage-policies
Change-Id: I19adbbcefbc086c8467bd904a275d55cde596412

test/probe/common.py

@@ -19,12 +19,14 @@ from subprocess import Popen, PIPE
 import sys
 from time import sleep, time
 from collections import defaultdict
+from nose import SkipTest
 
 from swiftclient import get_auth, head_account
 
 from swift.common.ring import Ring
 from swift.common.utils import readconf
 from swift.common.manager import Manager
+from swift.common.storage_policy import POLICIES
 
 from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC
 
@@ -136,13 +138,16 @@ def kill_nonprimary_server(primary_nodes, port2server, pids):
             return port
 
 
-def get_ring(server, force_validate=None):
+def get_ring(ring_name, server=None, force_validate=None):
-    ring = Ring('/etc/swift/%s.ring.gz' % server)
+    if not server:
+        server = ring_name
+    ring = Ring('/etc/swift', ring_name=ring_name)
     if not VALIDATE_RSYNC and not force_validate:
         return ring
     # easy sanity checks
-    assert 3 == ring.replica_count, '%s has %s replicas instead of 3' % (
+    if ring.replica_count != 3:
-        ring.serialized_path, ring.replica_count)
+        print 'WARNING: %s has %s replicas instead of 3' % (
+            ring.serialized_path, ring.replica_count)
     assert 4 == len(ring.devs), '%s has %s devices instead of 4' % (
         ring.serialized_path, len(ring.devs))
     # map server to config by port
@@ -197,7 +202,8 @@ def reset_environment():
     try:
         account_ring = get_ring('account')
         container_ring = get_ring('container')
-        object_ring = get_ring('object')
+        policy = POLICIES.default
+        object_ring = get_ring(policy.ring_name, 'object')
         Manager(['main']).start(wait=False)
         port2server = {}
         for server, port in [('account', 6002), ('container', 6001),
@@ -218,15 +224,14 @@ def reset_environment():
         try:
             raise
         except AssertionError as e:
-            print >>sys.stderr, 'ERROR: %s' % e
+            raise SkipTest(e)
-            os._exit(1)
         finally:
             try:
                 kill_servers(port2server, pids)
             except Exception:
                 pass
-    return pids, port2server, account_ring, container_ring, object_ring, url, \
+    return pids, port2server, account_ring, container_ring, object_ring, \
-        token, account, config_dict
+        policy, url, token, account, config_dict
 
 
 def get_to_final_state():
@@ -242,6 +247,15 @@ def get_to_final_state():
 
 
 if __name__ == "__main__":
-    for server in ('account', 'container', 'object'):
+    for server in ('account', 'container'):
-        get_ring(server, force_validate=True)
+        try:
+            get_ring(server, force_validate=True)
+        except AssertionError as err:
+            sys.exit('%s ERROR: %s' % (server, err))
         print '%s OK' % server
+    for policy in POLICIES:
+        try:
+            get_ring(policy.ring_name, server='object', force_validate=True)
+        except AssertionError as err:
+            sys.exit('object ERROR (%s): %s' % (policy.name, err))
+        print 'object OK (%s)' % policy.name

test/probe/test_account_failures.py

@@ -28,7 +28,7 @@ class TestAccountFailures(TestCase):
 
     def setUp(self):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
 
     def tearDown(self):

test/probe/test_account_get_fake_responses_match.py

@@ -27,7 +27,7 @@ class TestAccountGetFakeResponsesMatch(unittest.TestCase):
 
     def setUp(self):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
         self.url, self.token = get_auth(
             'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin')

test/probe/test_container_failures.py

@@ -44,7 +44,7 @@ class TestContainerFailures(TestCase):
 
     def setUp(self):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
 
     def tearDown(self):

test/probe/test_container_merge_policy_index.py

@@ -179,7 +179,7 @@ class TestContainerMergePolicyIndex(unittest.TestCase):
         if len(POLICIES) < 2:
             raise SkipTest()
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
         self.container_name = 'container-%s' % uuid.uuid4()
         self.object_name = 'object-%s' % uuid.uuid4()

test/probe/test_empty_device_handoff.py

@@ -24,6 +24,8 @@ from uuid import uuid4
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.storage_policy import POLICY_INDEX
+from swift.obj.diskfile import get_data_dir
 from swift.common.exceptions import ClientException
 from test.probe.common import kill_server, kill_servers, reset_environment,\
     start_server
@@ -35,7 +37,7 @@ class TestEmptyDevice(TestCase):
 
     def setUp(self):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
 
     def tearDown(self):
@@ -52,7 +54,7 @@ class TestEmptyDevice(TestCase):
     def test_main(self):
         # Create container
         # Kill one container/obj primary server
-        # Delete the "objects" directory on the primary server
+        # Delete the default data directory for objects on the primary server
         # Create container/obj (goes to two primary servers and one handoff)
         # Kill other two container/obj primary servers
         # Indirectly through proxy assert we can get container/obj
@@ -76,7 +78,8 @@ class TestEmptyDevice(TestCase):
             self.account, container, obj)
         onode = onodes[0]
         kill_server(onode['port'], self.port2server, self.pids)
-        obj_dir = '%s/objects' % self._get_objects_dir(onode)
+        obj_dir = '%s/%s' % (self._get_objects_dir(onode),
+                             get_data_dir(self.policy.idx))
         shutil.rmtree(obj_dir, True)
         self.assertFalse(os.path.exists(obj_dir))
         client.put_object(self.url, self.token, container, obj, 'VERIFY')
@@ -98,7 +101,8 @@ class TestEmptyDevice(TestCase):
             # let's directly verify it.
         another_onode = self.object_ring.get_more_nodes(opart).next()
         odata = direct_client.direct_get_object(
-            another_onode, opart, self.account, container, obj)[-1]
+            another_onode, opart, self.account, container, obj,
+            headers={POLICY_INDEX: self.policy.idx})[-1]
         if odata != 'VERIFY':
             raise Exception('Direct object GET did not return VERIFY, instead '
                             'it returned: %s' % repr(odata))
@@ -128,8 +132,9 @@ class TestEmptyDevice(TestCase):
         self.assertFalse(os.path.exists(obj_dir))
         exc = None
         try:
-            direct_client.direct_get_object(onode, opart, self.account,
+            direct_client.direct_get_object(
-                                            container, obj)
+                onode, opart, self.account, container, obj, headers={
+                    POLICY_INDEX: self.policy.idx})
         except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
@@ -150,15 +155,17 @@ class TestEmptyDevice(TestCase):
         another_num = (another_port_num - 6000) / 10
         Manager(['object-replicator']).once(number=another_num)
 
-        odata = direct_client.direct_get_object(onode, opart, self.account,
+        odata = direct_client.direct_get_object(
-                                                container, obj)[-1]
+            onode, opart, self.account, container, obj, headers={
+                POLICY_INDEX: self.policy.idx})[-1]
         if odata != 'VERIFY':
             raise Exception('Direct object GET did not return VERIFY, instead '
                             'it returned: %s' % repr(odata))
         exc = None
         try:
-            direct_client.direct_get_object(another_onode, opart, self.account,
+            direct_client.direct_get_object(
-                                            container, obj)
+                another_onode, opart, self.account, container, obj, headers={
+                    POLICY_INDEX: self.policy.idx})
         except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)

test/probe/test_object_async_update.py

@@ -29,7 +29,7 @@ class TestObjectAsyncUpdate(TestCase):
 
     def setUp(self):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
 
     def tearDown(self):

test/probe/test_object_failures.py

@@ -23,9 +23,10 @@ from uuid import uuid4
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.storage_policy import POLICY_INDEX
 from swift.common.exceptions import ClientException
 from swift.common.utils import hash_path, readconf
-from swift.obj.diskfile import write_metadata, read_metadata
+from swift.obj.diskfile import write_metadata, read_metadata, get_data_dir
 from test.probe.common import kill_servers, reset_environment
 
 
@@ -53,7 +54,7 @@ class TestObjectFailures(TestCase):
 
     def setUp(self):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
 
     def tearDown(self):
@@ -72,9 +73,9 @@ class TestObjectFailures(TestCase):
         hash_str = hash_path(self.account, container, obj)
         obj_server_conf = readconf(self.configs['object-server'][node_id])
         devices = obj_server_conf['app:object-server']['devices']
-        obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices,
+        obj_dir = '%s/%s/%s/%s/%s/%s/' % (devices, device,
-                                               device, opart,
+                                          get_data_dir(self.policy.idx),
-                                               hash_str[-3:], hash_str)
+                                          opart, hash_str[-3:], hash_str)
         data_file = get_data_file_path(obj_dir)
         return onode, opart, data_file
 
@@ -88,11 +89,13 @@ class TestObjectFailures(TestCase):
         write_metadata(data_file, metadata)
 
         odata = direct_client.direct_get_object(
-            onode, opart, self.account, container, obj)[-1]
+            onode, opart, self.account, container, obj, headers={
+                POLICY_INDEX: self.policy.idx})[-1]
         self.assertEquals(odata, 'VERIFY')
         try:
-            direct_client.direct_get_object(onode, opart, self.account,
+            direct_client.direct_get_object(
-                                            container, obj)
+                onode, opart, self.account, container, obj, headers={
+                    POLICY_INDEX: self.policy.idx})
             raise Exception("Did not quarantine object")
         except ClientException as err:
             self.assertEquals(err.http_status, 404)
@@ -106,16 +109,21 @@ class TestObjectFailures(TestCase):
         metadata = read_metadata(data_file)
         metadata['ETag'] = 'badetag'
         write_metadata(data_file, metadata)
+        base_headers = {POLICY_INDEX: self.policy.idx}
         for header, result in [({'Range': 'bytes=0-2'}, 'RAN'),
                                ({'Range': 'bytes=1-11'}, 'ANGE'),
                                ({'Range': 'bytes=0-11'}, 'RANGE')]:
+            req_headers = base_headers.copy()
+            req_headers.update(header)
             odata = direct_client.direct_get_object(
-                onode, opart, self.account, container, obj, headers=header)[-1]
+                onode, opart, self.account, container, obj,
+                headers=req_headers)[-1]
             self.assertEquals(odata, result)
 
         try:
-            direct_client.direct_get_object(onode, opart, self.account,
+            direct_client.direct_get_object(
-                                            container, obj)
+                onode, opart, self.account, container, obj, headers={
+                    POLICY_INDEX: self.policy.idx})
             raise Exception("Did not quarantine object")
         except ClientException as err:
             self.assertEquals(err.http_status, 404)
@@ -130,9 +138,9 @@ class TestObjectFailures(TestCase):
         with open(data_file, 'w') as fpointer:
             write_metadata(fpointer, metadata)
         try:
-            direct_client.direct_get_object(onode, opart, self.account,
+            direct_client.direct_get_object(
-                                            container, obj, conn_timeout=1,
+                onode, opart, self.account, container, obj, conn_timeout=1,
-                                            response_timeout=1)
+                response_timeout=1, headers={POLICY_INDEX: self.policy.idx})
             raise Exception("Did not quarantine object")
         except ClientException as err:
             self.assertEquals(err.http_status, 404)
@@ -147,9 +155,9 @@ class TestObjectFailures(TestCase):
         with open(data_file, 'w') as fpointer:
             write_metadata(fpointer, metadata)
         try:
-            direct_client.direct_head_object(onode, opart, self.account,
+            direct_client.direct_head_object(
-                                             container, obj, conn_timeout=1,
+                onode, opart, self.account, container, obj, conn_timeout=1,
-                                             response_timeout=1)
+                response_timeout=1, headers={POLICY_INDEX: self.policy.idx})
             raise Exception("Did not quarantine object")
         except ClientException as err:
             self.assertEquals(err.http_status, 404)
@@ -164,10 +172,12 @@ class TestObjectFailures(TestCase):
         with open(data_file, 'w') as fpointer:
             write_metadata(fpointer, metadata)
         try:
+            headers = {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two',
+                       POLICY_INDEX: self.policy.idx}
             direct_client.direct_post_object(
                 onode, opart, self.account,
                 container, obj,
-                {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'},
+                headers=headers,
                 conn_timeout=1,
                 response_timeout=1)
             raise Exception("Did not quarantine object")

test/probe/test_object_handoff.py

@@ -20,6 +20,7 @@ from uuid import uuid4
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.storage_policy import POLICY_INDEX
 from swift.common.exceptions import ClientException
 from swift.common.manager import Manager
 from test.probe.common import kill_server, kill_servers, reset_environment, \
@@ -30,7 +31,7 @@ class TestObjectHandoff(TestCase):
 
     def setUp(self):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
 
     def tearDown(self):
@@ -90,7 +91,8 @@ class TestObjectHandoff(TestCase):
         # directly verify it.
         another_onode = self.object_ring.get_more_nodes(opart).next()
         odata = direct_client.direct_get_object(
-            another_onode, opart, self.account, container, obj)[-1]
+            another_onode, opart, self.account, container, obj, headers={
+                POLICY_INDEX: self.policy.idx})[-1]
         if odata != 'VERIFY':
             raise Exception('Direct object GET did not return VERIFY, instead '
                             'it returned: %s' % repr(odata))
@@ -109,8 +111,9 @@ class TestObjectHandoff(TestCase):
         start_server(onode['port'], self.port2server, self.pids)
         exc = None
         try:
-            direct_client.direct_get_object(onode, opart, self.account,
+            direct_client.direct_get_object(
-                                            container, obj)
+                onode, opart, self.account, container, obj, headers={
+                    POLICY_INDEX: self.policy.idx})
         except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
@@ -128,21 +131,31 @@ class TestObjectHandoff(TestCase):
             another_port_num = another_onode['port']
         another_num = (another_port_num - 6000) / 10
         Manager(['object-replicator']).once(number=another_num)
-        odata = direct_client.direct_get_object(onode, opart, self.account,
+        odata = direct_client.direct_get_object(
-                                                container, obj)[-1]
+            onode, opart, self.account, container, obj, headers={
+                POLICY_INDEX: self.policy.idx})[-1]
         if odata != 'VERIFY':
             raise Exception('Direct object GET did not return VERIFY, instead '
                             'it returned: %s' % repr(odata))
         exc = None
         try:
-            direct_client.direct_get_object(another_onode, opart, self.account,
+            direct_client.direct_get_object(
-                                            container, obj)
+                another_onode, opart, self.account, container, obj, headers={
+                    POLICY_INDEX: self.policy.idx})
         except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
 
         kill_server(onode['port'], self.port2server, self.pids)
-        client.delete_object(self.url, self.token, container, obj)
+        try:
+            client.delete_object(self.url, self.token, container, obj)
+        except client.ClientException as err:
+            if self.object_ring.replica_count > 2:
+                raise
+            # Object DELETE returning 503 for (404, 204)
+            # remove this with fix for
+            # https://bugs.launchpad.net/swift/+bug/1318375
+            self.assertEqual(503, err.http_status)
         exc = None
         try:
             client.head_object(self.url, self.token, container, obj)
@@ -162,8 +175,9 @@ class TestObjectHandoff(TestCase):
                     'Container server %s:%s still knew about object' %
                     (cnode['ip'], cnode['port']))
         start_server(onode['port'], self.port2server, self.pids)
-        direct_client.direct_get_object(onode, opart, self.account, container,
+        direct_client.direct_get_object(
-                                        obj)
+            onode, opart, self.account, container, obj, headers={
+                POLICY_INDEX: self.policy.idx})
         # Run the extra server last so it'll remove its extra partition
         for node in onodes:
             try:
@@ -176,8 +190,9 @@ class TestObjectHandoff(TestCase):
         Manager(['object-replicator']).once(number=another_node_id)
         exc = None
         try:
-            direct_client.direct_get_object(another_onode, opart, self.account,
+            direct_client.direct_get_object(
-                                            container, obj)
+                another_onode, opart, self.account, container, obj, headers={
+                    POLICY_INDEX: self.policy.idx})
         except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)

test/probe/test_replication_servers_working.py

@@ -21,6 +21,8 @@ import time
 import shutil
 
 from swiftclient import client
+from swift.common.storage_policy import POLICIES
+from swift.obj.diskfile import get_data_dir
 
 from test.probe.common import kill_servers, reset_environment
 from swift.common.utils import readconf
@@ -80,7 +82,7 @@ class TestReplicatorFunctions(TestCase):
         Reset all environment and start all servers.
         """
         (self.pids, self.port2server, self.account_ring, self.container_ring,
-         self.object_ring, self.url, self.token,
+         self.object_ring, self.policy, self.url, self.token,
          self.account, self.configs) = reset_environment()
 
     def tearDown(self):
@@ -100,6 +102,7 @@ class TestReplicatorFunctions(TestCase):
         # Delete file "hashes.pkl".
         # Check, that all files were replicated.
         path_list = []
+        data_dir = get_data_dir(POLICIES.default.idx)
         # Figure out where the devices are
         for node_id in range(1, 5):
             conf = readconf(self.configs['object-server'][node_id])
@@ -124,7 +127,10 @@ class TestReplicatorFunctions(TestCase):
         for files in files_list[num]:
             if not files.endswith('.pending'):
                 test_node_files_list.append(files)
-        test_node_dir_list = dir_list[num]
+        test_node_dir_list = []
+        for d in dir_list[num]:
+            if not d.startswith('tmp'):
+                test_node_dir_list.append(d)
         # Run all replicators
         try:
             Manager(['object-replicator', 'container-replicator',
@@ -155,24 +161,24 @@ class TestReplicatorFunctions(TestCase):
                     time.sleep(1)
 
             # Check behavior by deleting hashes.pkl file
-            for directory in os.listdir(os.path.join(test_node, 'objects')):
+            for directory in os.listdir(os.path.join(test_node, data_dir)):
                 for input_dir in os.listdir(os.path.join(
-                        test_node, 'objects', directory)):
+                        test_node, data_dir, directory)):
                     if os.path.isdir(os.path.join(
-                            test_node, 'objects', directory, input_dir)):
+                            test_node, data_dir, directory, input_dir)):
                         shutil.rmtree(os.path.join(
-                            test_node, 'objects', directory, input_dir))
+                            test_node, data_dir, directory, input_dir))
 
             # We will keep trying these tests until they pass for up to 60s
             begin = time.time()
             while True:
                 try:
                     for directory in os.listdir(os.path.join(
-                            test_node, 'objects')):
+                            test_node, data_dir)):
                         for input_dir in os.listdir(os.path.join(
-                                test_node, 'objects', directory)):
+                                test_node, data_dir, directory)):
                             self.assertFalse(os.path.isdir(
-                                os.path.join(test_node, 'objects',
+                                os.path.join(test_node, data_dir,
                                              directory, '/', input_dir)))
                     break
                 except Exception:
@@ -180,9 +186,9 @@ class TestReplicatorFunctions(TestCase):
                         raise
                     time.sleep(1)
 
-            for directory in os.listdir(os.path.join(test_node, 'objects')):
+            for directory in os.listdir(os.path.join(test_node, data_dir)):
                 os.remove(os.path.join(
-                    test_node, 'objects', directory, 'hashes.pkl'))
+                    test_node, data_dir, directory, 'hashes.pkl'))
 
             # We will keep trying these tests until they pass for up to 60s
             begin = time.time()