fix: Taskflow tasks for Update, Delete and Create

- Make sure to convert serialized data into objects
- Look for Certain Messages in the Exception, before retrying for DNS
  Failure eg: Rackspace DNS Exception
- Change maximum timeout interval to 120s (Tested with 6 workers)
- Change Default Exponential Backoff to start at 20s

Change-Id: I701b91241f12cdb8d78b94d63d8d822dbf6fa811

poppy/distributed_task/taskflow/task/create_service_tasks.py

@@ -55,6 +55,7 @@ class CreateProviderServicesTask(task.Task):
                 service_obj)
             responders.append(responder)
             LOG.info('Create service from {0} complete...'.format(provider))
+
         return responders
 
 
@@ -69,13 +70,16 @@ class CreateServiceDNSMappingTask(task.Task):
         dns_responder = dns.create(responders)
         for provider_name in dns_responder:
             if 'error' in dns_responder[provider_name].keys():
-                LOG.info('Creating DNS for {0} failed!'.format(provider_name))
-                raise Exception('DNS Creation Failed')
+                if 'DNS Exception'\
+                        in dns_responder[provider_name]['error_detail']:
+                    msg = 'Create DNS for {0} failed!'.format(provider_name)
+                    LOG.info(msg)
+                    raise Exception(msg)
 
         return dns_responder
 
     def revert(self, responders, retry_sleep_time, **kwargs):
-        LOG.info('Sleeping for {0} minutes and '
+        LOG.info('Sleeping for {0} seconds and '
                  'retrying'.format(retry_sleep_time))
 
 

poppy/distributed_task/taskflow/task/delete_service_tasks.py

@@ -74,13 +74,16 @@ class DeleteServiceDNSMappingTask(task.Task):
             provider_details)
         for provider_name in dns_responder:
             if 'error' in dns_responder[provider_name].keys():
-                LOG.info('Deleting DNS for {0} failed!'.format(provider_name))
-                raise Exception('DNS Deletion Failed')
+                if 'DNS Exception'\
+                        in dns_responder[provider_name]['error_detail']:
+                    msg = 'Deleting DNS for {0} failed!'.format(provider_name)
+                    LOG.info(msg)
+                    raise Exception(msg)
 
         return dns_responder
 
     def revert(self, provider_details, retry_sleep_time, **kwargs):
-        LOG.info('Sleeping for {0} minutes and '
+        LOG.info('Sleeping for {0} seconds and '
                  'retrying'.format(retry_sleep_time))
 
 

poppy/distributed_task/taskflow/task/update_service_tasks.py

@@ -65,17 +65,30 @@ class UpdateServiceDNSMappingTask(task.Task):
         bootstrap_obj = bootstrap.Bootstrap(conf)
         service_controller = bootstrap_obj.manager.services_controller
         dns = service_controller.dns_controller
+        service_obj_json = json.loads(service_obj)
+        service_obj = service.load_from_json(service_obj_json)
+        service_old_json = json.loads(service_old)
+        service_old = service.load_from_json(service_old_json)
         dns_responder = dns.update(service_old, service_obj, responders)
 
         for provider_name in dns_responder:
-            if 'error' in dns_responder[provider_name].keys():
-                LOG.info('Updating DNS for {0} failed!'.format(provider_name))
-                raise Exception('DNS Update Failed')
+            try:
+                if 'error' in dns_responder[provider_name].keys():
+                    if 'DNS Exception'\
+                            in dns_responder[provider_name]['error_detail']:
+                        msg = 'Update DNS for {0}' \
+                              'failed!'.format(provider_name)
+                        LOG.info(msg)
+                        raise Exception(msg)
+            except KeyError:
+                # NOTE(TheSriram): This means the provider updates failed, and
+                # just access_urls were returned
+                pass
 
         return dns_responder
 
     def revert(self, responders, retry_sleep_time, **kwargs):
-        LOG.info('Sleeping for {0} minutes and '
+        LOG.info('Sleeping for {0} seconds and '
                  'retrying'.format(retry_sleep_time))
 
 

poppy/dns/rackspace/services.py

@@ -191,8 +191,9 @@ class ServicesController(base.ServicesBase):
                     error_msg = (error_msg + 'Can not access subdomain . '
                                  'Exception: {0}'.format(e))
                 except Exception as e:
-                    LOG.error('Exception: {0}'.format(e))
-                    error_msg = error_msg + 'Exception: {0}'.format(e)
+                    LOG.error('Rackspace DNS Exception: {0}'.format(e))
+                    error_msg = error_msg + 'Rackspace DNS ' \
+                                            'Exception: {0}'.format(e)
             # format the error message for this provider
             if not error_msg:
                 dns_details[provider_name] = self.responder.deleted({})

poppy/manager/default/services.py

@@ -58,7 +58,7 @@ class DefaultServicesController(base.ServicesController):
                                        group=DNS_GROUP)
         self.dns_conf = self.driver.conf[DNS_GROUP]
         self.default_sleep_time = [0]
-        self.backoff = [(2**i) * 60 for i in range(0, self.dns_conf.retries)]
+        self.backoff = [(2**i) * 20 for i in range(0, self.dns_conf.retries)]
 
         self.time_seconds = self.default_sleep_time + self.backoff
 

tests/etc/api.conf

@@ -19,7 +19,7 @@ flavor = {"flavor1": ["provider_1"], "flavor2": ["provider_2", "provider_3"]}
 [test_configuration]
 provider_validation=False
 status_check_retry_interval=2
-status_check_retry_timeout=30
+status_check_retry_timeout=120
 generate_flavors=False
 default_flavor=standard
 generated_provider=fastly