x
/
poppy
Code Issues Proposed changes

fix: only retry on appropriate dns exceptions per dns driver 

Change-Id: I02b546c4eb7c7069d95906d1458b74e9832fb44f
Closes-Bug: 1425556
This commit is contained in:
Sriram Madapusi Vasudevan 2015-07-09 17:54:02 -04:00
parent 93eaad3e58
commit 6e4070366a
17 changed files with 298 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
poppy/distributed_task/taskflow/task/create_service_tasks.py
View File

@ -20,6 +20,7 @@ from oslo.config import cfg
from taskflow import task
from poppy.distributed_task.taskflow.task import common
from poppy.distributed_task.utils import exc_loader
from poppy.distributed_task.utils import memoized_controllers
from poppy.model.helpers import provider_details
from poppy.openstack.common import log
@ -77,11 +78,21 @@ class CreateServiceDNSMappingTask(task.Task):
dns_responder = dns.create(responders)
for provider_name in dns_responder:
if 'error' in dns_responder[provider_name]:
if 'DNS Exception'\
in dns_responder[provider_name]['error']:
msg = 'Create DNS for {0} failed!'.format(provider_name)
LOG.info(msg)
raise Exception(msg)
msg = 'Create DNS for {0} ' \
'failed!'.format(provider_name)
LOG.info(msg)
if 'error_class' in dns_responder[provider_name]:
exception_repr = \
dns_responder[provider_name]['error_class']
exception_class = exc_loader(exception_repr)
if any([isinstance(exception_class(), exception) for
exception in dns._driver.retry_exceptions]):
LOG.info('Due to {0} Exception, '
'Task {1} will '
'be retried'.format(exception_class,
self.__class__))
raise exception_class(msg)
return dns_responder
@ -134,24 +145,31 @@ class GatherProviderDetailsTask(task.Task):
provider_details_dict = {}
for responder in responders:
for provider_name in responder:
error_class = None
if 'error' in responder[provider_name]:
error_msg = responder[provider_name]['error']
error_info = responder[provider_name]['error_detail']
if 'error_class' in responder[provider_name]:
error_class = \
responder[provider_name]['error_class']
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
error_info=error_info,
status='failed',
error_message=error_msg))
error_message=error_msg,
error_class=error_class))
elif 'error' in dns_responder[provider_name]:
error_msg = dns_responder[provider_name]['error']
error_info = dns_responder[provider_name]['error_detail']
if 'error_class' in dns_responder[provider_name]:
error_class = \
dns_responder[provider_name]['error_class']
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
error_info=error_info,
status='failed',
error_message=error_msg))
error_message=error_msg,
error_class=error_class))
else:
access_urls = dns_responder[provider_name]['access_urls']
if log_responders:

25
poppy/distributed_task/taskflow/task/delete_service_tasks.py
View File

@ -19,6 +19,7 @@ import time
from oslo.config import cfg
from taskflow import task
from poppy.distributed_task.utils import exc_loader
from poppy.distributed_task.utils import memoized_controllers
from poppy.openstack.common import log
from poppy.transport.pecan.models.request import (
@ -72,11 +73,21 @@ class DeleteServiceDNSMappingTask(task.Task):
provider_details)
for provider_name in dns_responder:
if 'error' in dns_responder[provider_name]:
if 'DNS Exception'\
in dns_responder[provider_name]['error']:
msg = 'Deleting DNS for {0} failed!'.format(provider_name)
LOG.info(msg)
raise Exception(msg)
msg = 'Delete DNS for {0} ' \
'failed!'.format(provider_name)
LOG.info(msg)
if 'error_class' in dns_responder[provider_name]:
exception_repr = \
dns_responder[provider_name]['error_class']
exception_class = exc_loader(exception_repr)
if any([isinstance(exception_class(), exception) for
exception in dns._driver.retry_exceptions]):
LOG.info('Due to {0} Exception, '
'Task {1} will '
'be retried'.format(exception_class,
self.__class__))
raise exception_class(msg)
return dns_responder
@ -111,6 +122,10 @@ class GatherProviderDetailsTask(task.Task):
# stores the error info for debugging purposes.
provider_details[provider_name].error_info = (
dns_responder[provider_name].get('error_info'))
if 'error_class' in dns_responder[provider_name]:
# stores the error class for debugging purposes.
provider_details[provider_name].error_class = (
dns_responder[provider_name].get('error_class'))
else:
# delete service successful, remove this provider detail record
del provider_details[provider_name]

31
poppy/distributed_task/taskflow/task/update_service_tasks.py
View File

@ -20,6 +20,7 @@ from oslo.config import cfg
from taskflow import task
from poppy.distributed_task.taskflow.task import common
from poppy.distributed_task.utils import exc_loader
from poppy.distributed_task.utils import memoized_controllers
from poppy.model.helpers import provider_details
from poppy.openstack.common import log
@ -72,12 +73,21 @@ class UpdateServiceDNSMappingTask(task.Task):
for provider_name in dns_responder:
try:
if 'error' in dns_responder[provider_name]:
if 'DNS Exception'\
in dns_responder[provider_name]['error']:
msg = 'Update DNS for {0}' \
'failed!'.format(provider_name)
LOG.info(msg)
raise Exception(msg)
msg = 'Update DNS for {0} ' \
'failed!'.format(provider_name)
LOG.info(msg)
if 'error_class' in dns_responder[provider_name]:
exception_repr = \
dns_responder[provider_name]['error_class']
exception_class = exc_loader(exception_repr)
if any([isinstance(exception_class(), exception) for
exception in dns._driver.retry_exceptions]):
LOG.info('Due to {0} Exception, '
'Task {1} will '
'be retried'.format(exception_class,
self.__class__))
raise exception_class(msg)
except KeyError:
# NOTE(TheSriram): This means the provider updates failed, and
# just access_urls were returned
@ -123,6 +133,7 @@ class GatherProviderDetailsTask(task.Task):
service_obj = service.load_from_json(service_obj_json)
# gather links and status for service from providers
error_flag = False
error_class = None
provider_details_dict = {}
for responder in responders:
for provider_name in responder:
@ -138,12 +149,16 @@ class GatherProviderDetailsTask(task.Task):
error_flag = True
error_msg = dns_responder[provider_name]['error']
error_info = dns_responder[provider_name]['error_detail']
if 'error_class' in dns_responder[provider_name]:
# stores the error class for debugging purposes.
error_class = dns_responder[provider_name].get(
'error_class')
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
error_info=error_info,
status='failed',
error_message=error_msg))
error_message=error_msg,
error_class=error_class))
else:
access_urls = dns_responder[provider_name]['access_urls']
if log_responders:

33
poppy/distributed_task/utils/__init__.py
View File

@ -0,0 +1,33 @@
# Copyright (c) 2015 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import importlib
def exc_loader(exc_class):
"""Exception Loader
Creates of the instance of the specified
exception class given the fully-qualified name.
The module is dynamically imported.
"""
pos = exc_class.rfind('.')
module_name = exc_class[:pos]
class_name = exc_class[pos + 1:]
mod = importlib.import_module(module_name)
return getattr(mod, class_name)

8
poppy/dns/base/driver.py
View File

@ -71,3 +71,11 @@ class DNSDriverBase(object):
:raises NotImplementedError
"""
raise NotImplementedError
@abc.abstractproperty
def retry_exceptions(self):
"""Retry on certain exceptions.
:raises NotImplementedError
"""
raise NotImplementedError

16
poppy/dns/base/helpers.py Normal file
View File

@ -0,0 +1,16 @@
# Copyright (c) 2015 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
retry_exceptions = []

18
poppy/dns/base/responder.py
View File

@ -35,11 +35,27 @@ class Responder(object):
error_detail = traceback.format_exc()
except AttributeError:
error_detail = msg
error_class = None
if 'error_msg' in msg and 'error_class' in msg:
error_msg = msg['error_msg']
error_class = msg['error_class']
else:
error_msg = msg
error_details[provider] = {
'error': msg,
'error': error_msg,
'error_detail': error_detail
}
if error_class:
try:
module_name = error_class.__module__
exc_name = error_class.__name__
error_details[provider]['error_class'] = \
".".join([module_name, exc_name])
except AttributeError:
error_details[provider]['error_class'] = error_class
return error_details
def created(self, dns_details):

10
poppy/dns/default/driver.py
View File

@ -17,6 +17,7 @@
from poppy.dns import base
from poppy.dns.default import controllers
from poppy.dns.default.helpers import retry_exceptions
from poppy.openstack.common import log as logging
LOG = logging.getLogger(__name__)
@ -62,3 +63,12 @@ class DNSProvider(base.Driver):
"""
return controllers.ServicesController(self)
@property
def retry_exceptions(self):
"""Retry on certain exceptions.
:return list
"""
return retry_exceptions

16
poppy/dns/default/helpers.py Normal file
View File

@ -0,0 +1,16 @@
# Copyright (c) 2015 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
retry_exceptions = []

9
poppy/dns/designate/driver.py
View File

@ -17,6 +17,7 @@
from poppy.dns import base
from poppy.dns.designate import controllers
from poppy.dns.designate.helpers import retry_exceptions
from poppy.openstack.common import log as logging
LOG = logging.getLogger(__name__)
@ -37,3 +38,11 @@ class DNSProvider(base.Driver):
@property
def service_controller(self):
return controllers.ServicesController(self)
@property
def retry_exceptions(self):
"""Retry on certain exceptions.
:return list
"""
return retry_exceptions

16
poppy/dns/designate/helpers.py Normal file
View File

@ -0,0 +1,16 @@
# Copyright (c) 2015 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
retry_exceptions = []

9
poppy/dns/rackspace/driver.py
View File

@ -20,6 +20,7 @@ import pyrax
from poppy.dns import base
from poppy.dns.rackspace import controllers
from poppy.dns.rackspace.helpers import retry_exceptions
from poppy.openstack.common import log as logging
@ -113,3 +114,11 @@ class DNSProvider(base.Driver):
"""
return controllers.ServicesController(self)
@property
def retry_exceptions(self):
"""Retry on certain exceptions.
:return list
"""
return retry_exceptions

21
poppy/dns/rackspace/helpers.py Normal file
View File

@ -0,0 +1,21 @@
# Copyright (c) 2015 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pyrax
retry_exceptions = [
pyrax.exc.DNSCallTimedOut,
pyrax.exc.OverLimit
]

50
poppy/dns/rackspace/services.py
View File

@ -220,7 +220,10 @@ class ServicesController(base.ServicesBase):
for provider_name in responder:
if 'error' in responder[provider_name]:
error_msg = responder[provider_name]['error_detail']
return self.responder.failed(providers, error_msg)
error_dict = {
'error_msg': error_msg
}
return self.responder.failed(providers, error_dict)
# gather the provider urls and cname them
links = {}
@ -238,9 +241,13 @@ class ServicesController(base.ServicesBase):
try:
dns_links = self._create_cname_records(links)
except Exception as e:
error_msg = 'Rackspace DNS Exception: {0}'.format(e)
LOG.error(error_msg)
return self.responder.failed(providers, error_msg)
msg = 'Rackspace DNS Exception: {0}'.format(e)
error = {
'error_msg': msg,
'error_class': e.__class__
}
LOG.error(msg)
return self.responder.failed(providers, error)
# gather the CNAMED links
dns_details = {}
@ -282,6 +289,7 @@ class ServicesController(base.ServicesBase):
dns_details = {}
error_msg = ''
error_class = None
for provider_name in provider_details:
access_urls = provider_details[provider_name].access_urls
for access_url in access_urls:
@ -298,17 +306,23 @@ class ServicesController(base.ServicesBase):
'to CDN subdomain {0}'.format(e))
error_msg = (error_msg + 'Can not access subdomain . '
'Exception: {0}'.format(e))
error_class = e.__class__
except Exception as e:
LOG.error('Rackspace DNS Exception: {0}'.format(e))
error_msg = error_msg + 'Rackspace DNS ' \
'Exception: {0}'.format(e)
error_class = e.__class__
# format the error message for this provider
if not error_msg:
dns_details[provider_name] = self.responder.deleted({})
# format the error message
if error_msg:
return self.responder.failed(providers, error_msg)
error = {
'error_msg': error_msg,
'error_class': error_class
}
return self.responder.failed(providers, error)
return dns_details
@ -344,8 +358,13 @@ class ServicesController(base.ServicesBase):
dns_links = self._create_cname_records(links)
except Exception as e:
error_msg = 'Rackspace DNS Exception: {0}'.format(e)
error_class = e.__class__
error = {
'error_msg': error_msg,
'error_class': error_class
}
LOG.error(error_msg)
return self.responder.failed(providers, error_msg)
return self.responder.failed(providers, error)
# gather the CNAMED links for added domains
for responder in responders:
@ -389,6 +408,7 @@ class ServicesController(base.ServicesBase):
# delete the records for deleted domains
error_msg = ''
error_class = None
for provider_name in provider_details:
provider_detail = provider_details[provider_name]
for access_url in provider_detail.access_urls:
@ -409,16 +429,22 @@ class ServicesController(base.ServicesBase):
'subdomain {0}'.format(e))
error_msg = (error_msg + 'Can not access subdomain. '
'Exception: {0}'.format(e))
error_class = e.__class__
except Exception as e:
LOG.error('Exception: {0}'.format(e))
error_msg = error_msg + 'Exception: {0}'.format(e)
error_class = e.__class__
# format the success message for this provider
if not error_msg:
dns_details[provider_name] = self.responder.deleted({})
# format the error message
if error_msg:
return self.responder.failed(providers, error_msg)
error_dict = {
'error_msg': error_msg,
'error_class': error_class
}
return self.responder.failed(providers, error_dict)
return dns_details
@ -487,7 +513,15 @@ class ServicesController(base.ServicesBase):
for provider_name in dns_links:
if 'error' in dns_links[provider_name]:
error_msg = dns_links[provider_name]['error_detail']
return self.responder.failed(providers, error_msg)
error_dict = {
'error_msg': error_msg
}
if 'error_class' in dns_links[provider_name]:
error_dict['error_class'] = \
dns_links[provider_name]['error_class']
return self.responder.failed(providers, error_dict)
# gather the CNAMED links and remove stale links
dns_details = {}

14
poppy/model/helpers/provider_details.py
View File

@ -34,13 +34,14 @@ class ProviderDetail(common.DictSerializableModel):
def __init__(self, provider_service_id=None, access_urls=[],
status=u"deploy_in_progress", name=None, error_info=None,
error_message=None):
error_message=None, error_class=None):
self._provider_service_id = provider_service_id
self._access_urls = access_urls
self._status = status
self._name = name
self._error_info = error_info
self._error_message = error_message
self._error_class = error_class
@property
def provider_service_id(self):
@ -98,6 +99,14 @@ class ProviderDetail(common.DictSerializableModel):
def error_message(self, value):
self._error_message = value
@property
def error_class(self):
return self._error_class
@error_class.setter
def error_class(self, value):
self._error_class = value
def to_dict(self):
result = collections.OrderedDict()
result["id"] = self.provider_service_id
@ -106,7 +115,7 @@ class ProviderDetail(common.DictSerializableModel):
result["name"] = self.name
result["error_info"] = self.error_info
result["error_message"] = self.error_message
result["error_class"] = self.error_class
return result
@classmethod
@ -127,4 +136,5 @@ class ProviderDetail(common.DictSerializableModel):
o.name = dict_obj.get("name", None)
o.error_info = dict_obj.get("error_info", None)
o.error_message = dict_obj.get("error_message", None)
o.error_class = dict_obj.get("error_class", None)
return o

7
tests/unit/dns/rackspace/test_driver.py
View File

@ -20,6 +20,7 @@ from oslo.config import cfg
import pyrax
from poppy.dns.rackspace import driver
from poppy.dns.rackspace.helpers import retry_exceptions
from tests.unit import base
RACKSPACE_OPTIONS = [
@ -103,3 +104,9 @@ class TestDriver(base.TestCase):
def test_service_controller(self, mock_set_credentials):
provider = driver.DNSProvider(self.conf)
self.assertNotEqual(provider.services_controller, None)
@mock.patch('pyrax.set_credentials')
@mock.patch.object(driver, 'RACKSPACE_OPTIONS', new=RACKSPACE_OPTIONS)
def test_retry_exceptions(self, mock_set_credentials):
provider = driver.DNSProvider(self.conf)
self.assertEqual(provider.retry_exceptions, retry_exceptions)

12
tests/unit/dns/rackspace/test_services.py
View File

@ -266,6 +266,9 @@ class TestServicesDelete(base.TestCase):
for provider_name in provider_details:
self.assertIsNotNone(dns_responder[provider_name]['error'])
self.assertIsNotNone(dns_responder[provider_name]['error_detail'])
self.assertIsNotNone(
dns_responder[provider_name]['error_class']
)
def test_delete_with_generic_exception(self):
akamai_access_urls = [
@ -303,6 +306,9 @@ class TestServicesDelete(base.TestCase):
for provider_name in provider_details:
self.assertIsNotNone(dns_responder[provider_name]['error'])
self.assertIsNotNone(dns_responder[provider_name]['error_detail'])
self.assertIsNotNone(
dns_responder[provider_name]['error_class']
)
def test_delete_no_records_found(self):
akamai_access_urls = [
@ -414,6 +420,9 @@ class TestServicesDelete(base.TestCase):
for provider_name in provider_details:
self.assertIsNotNone(dns_responder[provider_name]['error'])
self.assertIsNotNone(dns_responder[provider_name]['error_detail'])
self.assertIsNotNone(
dns_responder[provider_name]['error_class']
)
def test_delete(self):
akamai_access_urls = [
@ -542,6 +551,9 @@ class TestServicesUpdate(base.TestCase):
self.assertIsNotNone(dns_details[provider_name]['error'])
self.assertIsNotNone(
dns_details[provider_name]['error_detail'])
self.assertIsNotNone(
dns_details[provider_name]['error_class']
)
def test_update_remove_domains_provider_error(self):
domains_new = [domain.Domain('test.domain.com'),