Merge pull request #36 from morucci/issue-26

Willl test that in live as I am refilling cluster.

swsync/filler.py

@@ -37,6 +37,7 @@ import string
 import StringIO
 
 from swiftclient import client as sclient
+from swiftclient.client import ClientException
 
 import eventlet
 
@@ -186,17 +187,22 @@ def create_objects(cnx, acc, o_amount, fmax, index_containers):
                            map(get_rand_str, ('meta_v_',) * 3)]
             meta = dict(zip(meta_keys, meta_values))
             data = f_object.read()
-            logging.info("Put data for container %s "
-                         "(filename: %s,\tsize: %.3f KB)" %
-                         (container,
-                         object_name.encode('ascii', 'ignore'),
-                         float(len(data))/1024))
-            etag = cnx.put_object(container, object_name,
-                                  data, headers=copy.copy(meta))
             f_object.close()
-            obj_info = {'object_info':
-                        (object_name, etag, len(data)), 'meta': meta}
-            containers_d[container]['objects'].append(obj_info)
+            try:
+                etag = cnx.put_object(container, object_name,
+                                      data, headers=copy.copy(meta))
+                logging.info("Put data for container %s "
+                             "(filename: %s,\tsize: %.3f KB)" %
+                             (container,
+                             object_name.encode('ascii', 'ignore'),
+                             float(len(data))/1024))
+                obj_info = {'object_info':
+                            (object_name, etag, len(data)), 'meta': meta}
+                containers_d[container]['objects'].append(obj_info)
+            except ClientException:
+                logging.warning('Unable to put object %s in container %s' % (
+                                object_name.encode('ascii', 'ignore'),
+                                container.encode('ascii', 'ignore')))
 
 
 def create_containers(cnx, acc, c_amount, index_containers=None):
@@ -214,8 +220,12 @@ def create_containers(cnx, acc, c_amount, index_containers=None):
         meta = dict(zip(meta_keys, meta_values))
         logging.info("Create container %s" %
                      container_name.encode('ascii', 'ignore'))
-        cnx.put_container(container_name, headers=copy.copy(meta))
-        containers_d[container_name] = {'meta': meta, 'objects': []}
+        try:
+            cnx.put_container(container_name, headers=copy.copy(meta))
+            containers_d[container_name] = {'meta': meta, 'objects': []}
+        except ClientException:
+            logging.warning("Unable to create container %s" %
+                            container_name.encode('ascii', 'ignore'))
 
 
 def fill_swift(pool, created_account, c_amount,

tests/units/test_filler.py

@@ -18,13 +18,17 @@
 import eventlet
 import swiftclient
 
-from swsync.utils import get_config
+from fakes import FakeKSClient
+from fakes import FakeKSUser
+from fakes import FakeSWConnection
+
+from tests.units import base
+
 from swsync import filler
-import base as test_base
-from fakes import FakeSWConnection, FakeKSClient, FakeKSUser
+from swsync import utils
 
 
-class TestFiller(test_base.TestCase):
+class TestFiller(base.TestCase):
     def setUp(self):
         super(TestFiller, self).setUp()
         self._stubs()
@@ -34,7 +38,7 @@ class TestFiller(test_base.TestCase):
                        FakeSWConnection)
 
     def get_connection(self, *args):
-        return swiftclient.client.Connection(get_config(
+        return swiftclient.client.Connection(utils.get_config(
                                              'auth', 'keystone_origin'),
                                              'test', 'password',
                                              tenant_name='test')
@@ -55,6 +59,25 @@ class TestFiller(test_base.TestCase):
         meta_amount = len(return_dict_ref['test'].values())
         self.assertEqual(meta_amount, 3)
 
+    def test_create_containers_fail(self):
+        get_containers_created = []
+        return_dict_ref = {}
+        self.attempts = 0
+
+        def put_container(*args, **kwargs):
+            if self.attempts == 0:
+                self.attempts += 1
+                raise swiftclient.client.ClientException('Fake err msg')
+            else:
+                self.attempts += 1
+                get_containers_created.append(args[1])
+
+        self.stubs.Set(FakeSWConnection, 'put_container', put_container)
+        cnx = self.get_connection()
+        filler.create_containers(cnx, 'test', 3, return_dict_ref)
+
+        self.assertEqual(len(get_containers_created), 2)
+
     def test_create_objects(self):
         get_object_created = []
         return_dict_ref = {'test': {'container_a': {'objects': []},
@@ -73,6 +96,27 @@ class TestFiller(test_base.TestCase):
         self.assertEqual(len(objects_ca), 2)
         self.assertEqual(len(objects_cb), 2)
 
+    def test_create_objects_fail(self):
+        get_object_created = []
+        return_dict_ref = {'test': {'container_a': {'objects': []}}}
+        self.attempts = 0
+
+        def put_object(*args, **kwargs):
+            if self.attempts == 0:
+                self.attempts += 1
+                raise swiftclient.client.ClientException('Fake err msg')
+            else:
+                self.attempts += 1
+                get_object_created.append(args[1:])
+
+        self.stubs.Set(FakeSWConnection,
+                       'put_object',
+                       put_object)
+        cnx = self.get_connection()
+        filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref)
+        objects_ca = return_dict_ref['test']['container_a']['objects']
+        self.assertEqual(len(objects_ca), 1)
+
     def test_fill_swift(self):
         self.cont_cnt = 0
         self.obj_cnt = 0
@@ -91,7 +135,7 @@ class TestFiller(test_base.TestCase):
         self.stubs.Set(filler, 'create_objects', create_objects)
         self.stubs.Set(filler, 'create_containers', create_containers)
 
-        concurrency = int(get_config('filler', 'concurrency'))
+        concurrency = int(utils.get_config('filler', 'concurrency'))
         pool = eventlet.GreenPool(concurrency)
 
         created = {('account1', 'account1_id'): ['test', 'test_id', 'role_id'],
@@ -111,7 +155,8 @@ class TestFiller(test_base.TestCase):
         def add_user_role(*args, **kargs):
             self.role_cnt += 1
 
-        co = get_config('auth', 'keystone_origin_admin_credentials').split(':')
+        co = utils.get_config('auth',
+                              'keystone_origin_admin_credentials').split(':')
         tenant_name, username, password = co
         client = FakeKSClient()
         client.roles.add_user_role = add_user_role
@@ -130,7 +175,7 @@ class TestFiller(test_base.TestCase):
 
         self.stubs.Set(filler, 'create_swift_user', create_swift_user)
 
-        concurrency = int(get_config('filler', 'concurrency'))
+        concurrency = int(utils.get_config('filler', 'concurrency'))
         pile = eventlet.GreenPile(concurrency)
         client = FakeKSClient()
         filler.create_swift_account(client, pile, 1, 1, self.ret_index)