From 63e07e159e8b3bea3e8db6ae4356cfa55c10de56 Mon Sep 17 00:00:00 2001 From: rjrjr Date: Tue, 6 Jan 2015 22:53:58 -0700 Subject: [PATCH] Separate Periodic Recovery from Periodic Sync Introduced a new thread to handle periodic recovery. The periodic recovery thread runs more frequently and handles domain create, delete, and update failures. The periodic sync thread runs less frequently and handles synchronizing the zones on the backend servers with the database. Change-Id: I31cd8f77e14c5fcfaaac93076079395b675fa801 Partial-Bug: 1404395 --- designate/pool_manager/__init__.py | 4 +++- designate/pool_manager/service.py | 31 +++++++++++++++++++++++++++-- etc/designate/designate.conf.sample | 3 ++- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/designate/pool_manager/__init__.py b/designate/pool_manager/__init__.py index 58ecb53d..bb41a60c 100644 --- a/designate/pool_manager/__init__.py +++ b/designate/pool_manager/__init__.py @@ -41,7 +41,9 @@ OPTS = [ cfg.IntOpt('poll-delay', default=1, help='The time to wait before sending the first request ' 'to a server'), - cfg.IntOpt('periodic-sync-interval', default=120, + cfg.IntOpt('periodic-recovery-interval', default=120, + help='The time between recovering from failures'), + cfg.IntOpt('periodic-sync-interval', default=300, help='The time between synchronizing the servers with Storage'), cfg.StrOpt('cache-driver', default='sqlalchemy', help='The cache driver to use'), diff --git a/designate/pool_manager/service.py b/designate/pool_manager/service.py index 9cf239c8..a609a1c3 100644 --- a/designate/pool_manager/service.py +++ b/designate/pool_manager/service.py @@ -123,6 +123,10 @@ class Service(service.RPCService): backend_instance = server_backend['backend_instance'] backend_instance.start() + self.thread_group.add_timer( + cfg.CONF['service:pool_manager'].periodic_recovery_interval, + self.periodic_recovery) + self.thread_group.add_timer( cfg.CONF['service:pool_manager'].periodic_sync_interval, self.periodic_sync) @@ -263,16 +267,25 @@ class Service(service.RPCService): self.central_api.update_status( context, domain.id, ERROR_STATUS, error_serial) - def periodic_sync(self): + def periodic_recovery(self): """ :return: """ - LOG.debug("Calling periodic_sync.") + LOG.debug("Calling periodic_recovery.") context = self.admin_context self._periodic_create_domains_that_failed(context) self._periodic_delete_domains_that_failed(context) + self._periodic_update_domains_that_failed(context) + + def periodic_sync(self): + """ + :return: None + """ + LOG.debug("Calling periodic_sync.") + + context = self.admin_context criterion = { 'pool_id': cfg.CONF['service:pool_manager'].pool_id @@ -331,6 +344,20 @@ class Service(service.RPCService): except exceptions.Backend: pass + def _periodic_update_domains_that_failed(self, context): + + update_statuses = self._find_pool_manager_statuses( + context, UPDATE_ACTION, status=ERROR_STATUS) + + for update_status in update_statuses: + domain = self.central_api.get_domain( + context, update_status.domain_id) + server = self._get_server_backend( + update_status.server_id)['server'] + + self._notify_zone_changed(context, domain, server) + self._poll_for_serial_number(context, domain, server) + def _notify_zone_changed(self, context, domain, server): self.mdns_api.notify_zone_changed( context, domain, self._get_destination(server), diff --git a/etc/designate/designate.conf.sample b/etc/designate/designate.conf.sample index 68688d39..2663c0bd 100644 --- a/etc/designate/designate.conf.sample +++ b/etc/designate/designate.conf.sample @@ -130,7 +130,8 @@ debug = False #poll_retry_interval = 2 #poll_max_retries = 3 #poll_delay = 1 -#periodic_sync_interval = 120 +#periodic_recovery_interval = 120 +#periodic_sync_interval = 300 #cache_driver = sqlalchemy ##############