Fix replication failure when Swift isn't available

* add Swift token verification in create instance method in Trove API
    when slave_id is given
  * catch ConnectionError exception in verify_swift_auth_token
  * add Swift token verification to guestagent before backup and restore
  * add new exception representing Swift connection error
  * set missing fault information when replication snapshot fails
  * mock verify auth token method in replication and restore unit tests

Closes-Bug: #1395523

Change-Id: I6a21ba2ba890a82875f9b6dae3c6b93bc9fdb4b0
Signed-off-by: Dariusz Krol <d.krol@samsung.com>

trove/backup/models.py

@@ -15,6 +15,7 @@
 """Model classes that form the core of snapshots functionality."""
 
 from oslo_log import log as logging
+from requests.exceptions import ConnectionError
 from sqlalchemy import desc
 from swiftclient.client import ClientException
 
@@ -29,7 +30,6 @@ from trove.db.models import DatabaseModelBase
 from trove.quota.quota import run_with_quotas
 from trove.taskmanager import api
 
-
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
@@ -291,6 +291,8 @@ class Backup(object):
             raise exception.SwiftAuthError(tenant_id=context.tenant)
         except exception.NoServiceEndpoint:
             raise exception.SwiftNotFound(tenant_id=context.tenant)
+        except ConnectionError:
+            raise exception.SwiftConnectionError()
 
 
 def persisted_models():

trove/common/exception.py

@@ -461,6 +461,10 @@ class SwiftNotFound(TroveError):
     message = _("Swift is disabled for tenant %(tenant_id)s.")
 
 
+class SwiftConnectionError(TroveError):
+    message = _("Cannot connect to Swift.")
+
+
 class DatabaseForUserNotInDatabaseListError(TroveError):
     message = _("The request indicates that user %(user)s should have access "
                 "to database %(database)s, but database %(database)s is not "

trove/common/wsgi.py

@@ -361,6 +361,7 @@ class Controller(object):
         webob.exc.HTTPServerError: [
             exception.VolumeCreationFailure,
             exception.UpdateGuestError,
+            exception.SwiftConnectionError,
         ],
         webob.exc.HTTPNotImplemented: [
             exception.VolumeNotSupported,

trove/instance/models.py

@@ -923,6 +923,8 @@ class Instance(BuiltInstance):
                 target_size = flavor.ephemeral  # ephemeral_Storage
 
         if backup_id:
+            Backup.verify_swift_auth_token(context)
+
             call_args['backup_id'] = backup_id
             backup_info = Backup.get_by_id(context, backup_id)
             if not backup_info.is_done_successfuly:
@@ -946,6 +948,8 @@ class Instance(BuiltInstance):
                     datastore2=datastore.name)
 
         if slave_of_id:
+            Backup.verify_swift_auth_token(context)
+
             if databases or users:
                 raise exception.ReplicaCreateWithUsersDatabasesError()
             call_args['replica_of'] = slave_of_id

trove/taskmanager/models.py

@@ -625,6 +625,12 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
                 'replica': self.id
             }
             err = inst_models.InstanceTasks.BUILDING_ERROR_REPLICA
+            e_create_fault = create_log_fmt % create_fmt_content
+            e_create_stack = traceback.format_exc()
+            # we persist fault details to source instance
+            inst_models.save_instance_fault(slave_of_id, e_create_fault,
+                                            e_create_stack)
+
             # if the delete of the 'bad' backup fails, it'll mask the
             # create exception, so we trap it here
             try:

trove/tests/unittests/instance/test_instance_models.py

@@ -196,6 +196,12 @@ class CreateInstanceTest(trove_testtools.TestCase):
         backup_models.DBBackup.check_swift_object_exist = Mock(
             return_value=True)
         self.locality = 'affinity'
+
+        self.swift_verify_patch = patch.object(models.Backup,
+                                               'verify_swift_auth_token')
+        self.addCleanup(self.swift_verify_patch.stop)
+        self.swift_verify_patch.start()
+
         super(CreateInstanceTest, self).setUp()
 
     @patch.object(task_api.API, 'get_client', Mock(return_value=Mock()))
@@ -363,6 +369,12 @@ class TestReplication(trove_testtools.TestCase):
 
         self.safe_nova_client = models.create_nova_client
         models.create_nova_client = nova.fake_create_nova_client
+
+        self.swift_verify_patch = patch.object(models.Backup,
+                                               'verify_swift_auth_token')
+        self.addCleanup(self.swift_verify_patch.stop)
+        self.swift_verify_patch.start()
+
         super(TestReplication, self).setUp()
 
     def tearDown(self):