Browse Source

Actually poll for zone deletes

- This implements a few `TODO`s in the code around actually polling
nameservers for zone deletes. I found this necessary for changes
that were truing up missed DELETEs, and wanted to make a poll/
update_status call to verify deletion across a fleet of nameservers
when one was down.
- This originally failed in PowerDNS, because in older versions
that are still in all the distro repos, PDNS responded with a
NOERROR and empty answer section when it got a query for a zone
it knew nothing about. NEAT.
http://blog.powerdns.com/2015/03/02/from-noerror-to-refused/

- NOTE: This increases timeout for functional tests and we maybe
shouldn't do that.

Change-Id: Ied1e5daf4798127e00a06265164759d98fc9ba72
Tim Simmons 3 years ago
parent
commit
defe3a8f29

+ 6
- 10
designate/central/service.py View File

@@ -2292,17 +2292,13 @@ class Service(service.RPCService, service.Service):
2292 2292
         zone, deleted = self._update_zone_or_record_status(
2293 2293
             zone, status, serial)
2294 2294
 
2295
-        LOG.debug('Setting zone %s, serial %s: action %s, status %s'
2296
-                  % (zone.id, zone.serial, zone.action, zone.status))
2297
-        self.storage.update_zone(context, zone)
2298
-
2299
-        # TODO(Ron): Including this to retain the current logic.
2300
-        # We should NOT be deleting zones.  The zone status should be
2301
-        # used to indicate the zone has been deleted and not the deleted
2302
-        # column.  The deleted column is needed for unique constraints.
2295
+        if zone.status != 'DELETED':
2296
+            LOG.debug('Setting zone %s, serial %s: action %s, status %s'
2297
+                      % (zone.id, zone.serial, zone.action, zone.status))
2298
+            self.storage.update_zone(context, zone)
2299
+
2303 2300
         if deleted:
2304
-            # TODO(vinod): Pass a zone to delete_zone rather than id so
2305
-            # that the action, status and serial are updated correctly.
2301
+            LOG.debug('update_status: deleting %s' % zone.name)
2306 2302
             self.storage.delete_zone(context, zone.id)
2307 2303
 
2308 2304
         return zone

+ 14
- 4
designate/mdns/notify.py View File

@@ -118,9 +118,14 @@ class NotifyEndpoint(base.BaseEndpoint):
118 118
         while (True):
119 119
             (response, retry) = self._make_and_send_dns_message(
120 120
                 zone, host, port, timeout, retry_interval, retries)
121
-            if response and response.rcode() in (
122
-                    dns.rcode.NXDOMAIN, dns.rcode.REFUSED, dns.rcode.SERVFAIL):
121
+
122
+            if response and (response.rcode() in (
123
+                    dns.rcode.NXDOMAIN, dns.rcode.REFUSED, dns.rcode.SERVFAIL)
124
+                    or not bool(response.answer)):
123 125
                 status = 'NO_ZONE'
126
+                if zone.serial == 0 and zone.action in ['DELETE', 'NONE']:
127
+                    return (status, 0, retries)
128
+
124 129
             elif response and len(response.answer) == 1 \
125 130
                     and str(response.answer[0].name) == str(zone.name) \
126 131
                     and response.answer[0].rdclass == dns.rdataclass.IN \
@@ -138,12 +143,14 @@ class NotifyEndpoint(base.BaseEndpoint):
138 143
                          {'zone': zone.name, 'host': host,
139 144
                           'port': port, 'es': zone.serial,
140 145
                           'as': actual_serial, 'retries': retries})
146
+
141 147
                 if retries > 0:
142 148
                     # retry again
143 149
                     time.sleep(retry_interval)
144 150
                     continue
145 151
                 else:
146 152
                     break
153
+
147 154
             else:
148 155
                 # Everything looks good at this point. Return SUCCESS.
149 156
                 status = 'SUCCESS'
@@ -211,8 +218,11 @@ class NotifyEndpoint(base.BaseEndpoint):
211 218
                 break
212 219
             # Check that we actually got a NOERROR in the rcode and and an
213 220
             # authoritative answer
214
-            elif response.rcode() in (dns.rcode.NXDOMAIN, dns.rcode.REFUSED,
215
-                                      dns.rcode.SERVFAIL):
221
+            elif (response.rcode() in
222
+                    (dns.rcode.NXDOMAIN, dns.rcode.REFUSED,
223
+                     dns.rcode.SERVFAIL)) or \
224
+                    (response.rcode() == dns.rcode.NOERROR and
225
+                     not bool(response.answer)):
216 226
                 LOG.info(_LI("%(zone)s not found on %(server)s:%(port)d"),
217 227
                          {'zone': zone.name, 'server': host,
218 228
                          'port': port})

+ 34
- 22
designate/pool_manager/service.py View File

@@ -197,7 +197,7 @@ class Service(service.RPCService, coordination.CoordinationMixin,
197 197
     def _get_admin_context_all_tenants(self):
198 198
         return DesignateContext.get_admin_context(all_tenants=True)
199 199
 
200
-    # Periodioc Tasks
200
+    # Periodic Tasks
201 201
     def periodic_recovery(self):
202 202
         """
203 203
         Runs only on the pool leader
@@ -317,7 +317,7 @@ class Service(service.RPCService, coordination.CoordinationMixin,
317 317
         for also_notify in self.pool.also_notifies:
318 318
             self._update_zone_on_also_notify(context, also_notify, zone)
319 319
 
320
-        # Send a NOTIFY to each nameserver
320
+        # Ensure the change has propagated to each nameserver
321 321
         for nameserver in self.pool.nameservers:
322 322
             create_status = self._build_status_object(
323 323
                 nameserver, zone, CREATE_ACTION)
@@ -391,7 +391,7 @@ class Service(service.RPCService, coordination.CoordinationMixin,
391 391
         for also_notify in self.pool.also_notifies:
392 392
             self._update_zone_on_also_notify(context, also_notify, zone)
393 393
 
394
-        # Ensure the change has propogated to each nameserver
394
+        # Ensure the change has propagated to each nameserver
395 395
         for nameserver in self.pool.nameservers:
396 396
             # See if there is already another update in progress
397 397
             try:
@@ -453,25 +453,29 @@ class Service(service.RPCService, coordination.CoordinationMixin,
453 453
             results.append(
454 454
                 self._delete_zone_on_target(context, target, zone))
455 455
 
456
-        # TODO(kiall): We should monitor that the Zone is actually deleted
457
-        #              correctly on each of the nameservers, rather than
458
-        #              assuming a successful delete-on-target is OK as we have
459
-        #              in the past.
460
-        if self._exceed_or_meet_threshold(
456
+        if not self._exceed_or_meet_threshold(
461 457
                 results.count(True), MAXIMUM_THRESHOLD):
462
-            LOG.debug('Consensus reached for deleting zone %(zone)s '
463
-                      'on pool targets' % {'zone': zone.name})
464
-
465
-            self.central_api.update_status(
466
-                context, zone.id, SUCCESS_STATUS, zone.serial)
467
-
468
-        else:
469 458
             LOG.warning(_LW('Consensus not reached for deleting zone %(zone)s'
470
-                         ' on pool targets') % {'zone': zone.name})
471
-
459
+                            ' on pool targets') % {'zone': zone.name})
472 460
             self.central_api.update_status(
473 461
                 context, zone.id, ERROR_STATUS, zone.serial)
474 462
 
463
+        zone.serial = 0
464
+        # Ensure the change has propagated to each nameserver
465
+        for nameserver in self.pool.nameservers:
466
+            # See if there is already another update in progress
467
+            try:
468
+                self.cache.retrieve(context, nameserver.id, zone.id,
469
+                                    DELETE_ACTION)
470
+            except exceptions.PoolManagerStatusNotFound:
471
+                update_status = self._build_status_object(
472
+                    nameserver, zone, DELETE_ACTION)
473
+                self.cache.store(context, update_status)
474
+
475
+            self.mdns_api.poll_for_serial_number(
476
+                context, zone, nameserver, self.timeout,
477
+                self.retry_interval, self.max_retries, self.delay)
478
+
475 479
     def _delete_zone_on_target(self, context, target, zone):
476 480
         """
477 481
         :param context: Security context information.
@@ -542,7 +546,11 @@ class Service(service.RPCService, coordination.CoordinationMixin,
542 546
                 current_status.serial_number = actual_serial
543 547
                 self.cache.store(context, current_status)
544 548
 
549
+            LOG.debug('Attempting to get consensus serial for %s' %
550
+                      zone.name)
545 551
             consensus_serial = self._get_consensus_serial(context, zone)
552
+            LOG.debug('Consensus serial for %s is %s' %
553
+                      (zone.name, consensus_serial))
546 554
 
547 555
             # If there is a valid consensus serial we can still send a success
548 556
             # for that serial.
@@ -568,11 +576,15 @@ class Service(service.RPCService, coordination.CoordinationMixin,
568 576
                     self.central_api.update_status(
569 577
                         context, zone.id, ERROR_STATUS, error_serial)
570 578
 
571
-            if status == NO_ZONE_STATUS and action != DELETE_ACTION:
572
-                LOG.warning(_LW('Zone %(zone)s is not present in some '
573
-                             'targets') % {'zone': zone.name})
574
-                self.central_api.update_status(
575
-                    context, zone.id, NO_ZONE_STATUS, 0)
579
+            if status == NO_ZONE_STATUS:
580
+                if action == DELETE_ACTION:
581
+                    self.central_api.update_status(
582
+                        context, zone.id, NO_ZONE_STATUS, 0)
583
+                else:
584
+                    LOG.warning(_LW('Zone %(zone)s is not present in some '
585
+                                 'targets') % {'zone': zone.name})
586
+                    self.central_api.update_status(
587
+                        context, zone.id, NO_ZONE_STATUS, 0)
576 588
 
577 589
             if consensus_serial == zone.serial and self._is_consensus(
578 590
                     context, zone, action, SUCCESS_STATUS,

+ 0
- 59
designate/tests/test_pool_manager/test_service.py View File

@@ -252,65 +252,6 @@ class PoolManagerServiceNoopTest(PoolManagerTestCase):
252 252
 
253 253
         self.assertFalse(mock_update_status.called)
254 254
 
255
-    @patch.object(impl_fake.FakeBackend, 'delete_zone',
256
-                  side_effect=exceptions.Backend)
257
-    @patch.object(central_rpcapi.CentralAPI, 'update_status')
258
-    def test_delete_zone(self, mock_update_status, _):
259
-        zone = self._build_zone('example.org.', 'DELETE', 'PENDING')
260
-
261
-        self.service.delete_zone(self.admin_context, zone)
262
-
263
-        mock_update_status.assert_called_once_with(
264
-            self.admin_context, zone.id, 'ERROR', zone.serial)
265
-
266
-    @patch.object(impl_fake.FakeBackend, 'delete_zone')
267
-    @patch.object(central_rpcapi.CentralAPI, 'update_status')
268
-    def test_delete_zone_target_both_failure(
269
-            self, mock_update_status, mock_delete_zone):
270
-
271
-        zone = self._build_zone('example.org.', 'DELETE', 'PENDING')
272
-
273
-        mock_delete_zone.side_effect = exceptions.Backend
274
-
275
-        self.service.delete_zone(self.admin_context, zone)
276
-
277
-        mock_update_status.assert_called_once_with(
278
-            self.admin_context, zone.id, 'ERROR', zone.serial)
279
-
280
-    @patch.object(impl_fake.FakeBackend, 'delete_zone')
281
-    @patch.object(central_rpcapi.CentralAPI, 'update_status')
282
-    def test_delete_zone_target_one_failure(
283
-            self, mock_update_status, mock_delete_zone):
284
-
285
-        zone = self._build_zone('example.org.', 'DELETE', 'PENDING')
286
-
287
-        mock_delete_zone.side_effect = [None, exceptions.Backend]
288
-
289
-        self.service.delete_zone(self.admin_context, zone)
290
-
291
-        mock_update_status.assert_called_once_with(
292
-            self.admin_context, zone.id, 'ERROR', zone.serial)
293
-
294
-    @patch.object(impl_fake.FakeBackend, 'delete_zone')
295
-    @patch.object(central_rpcapi.CentralAPI, 'update_status')
296
-    def test_delete_zone_target_one_failure_consensus(
297
-            self, mock_update_status, mock_delete_zone):
298
-
299
-        self.service.stop()
300
-        self.config(
301
-            threshold_percentage=50,
302
-            group='service:pool_manager')
303
-        self.service = self.start_service('pool_manager')
304
-
305
-        zone = self._build_zone('example.org.', 'DELETE', 'PENDING')
306
-
307
-        mock_delete_zone.side_effect = [None, exceptions.Backend]
308
-
309
-        self.service.delete_zone(self.admin_context, zone)
310
-
311
-        mock_update_status.assert_called_once_with(
312
-            self.admin_context, zone.id, 'ERROR', zone.serial)
313
-
314 255
     @patch.object(mdns_rpcapi.MdnsAPI, 'get_serial_number',
315 256
                   side_effect=messaging.MessagingException)
316 257
     @patch.object(central_rpcapi.CentralAPI, 'update_status')

+ 35
- 0
designate/tests/unit/__init__.py View File

@@ -55,3 +55,38 @@ class RoObject(object):
55 55
 
56 56
     def to_dict(self):
57 57
         return self.__dict__
58
+
59
+
60
+class RwObject(object):
61
+    """Object mock: raise exception on __setitem__ or __setattr__
62
+    on any item/attr created after initialization.
63
+    Allows updating existing items/attrs
64
+    """
65
+    def __init__(self, d=None, **kw):
66
+        if d:
67
+            kw.update(d)
68
+        self.__dict__.update(kw)
69
+
70
+    def __getitem__(self, k):
71
+        try:
72
+            return self.__dict__[k]
73
+        except KeyError:
74
+            cn = self.__class__.__name__
75
+            raise NotImplementedError(
76
+                "Attempt to perform __getitem__"
77
+                " %r on %s %r" % (cn, k, self.__dict__)
78
+            )
79
+
80
+    def __setitem__(self, k, v):
81
+        if k in self.__dict__:
82
+            self.__dict__.update({k: v})
83
+            return
84
+
85
+        cn = self.__class__.__name__
86
+        raise NotImplementedError(
87
+            "Attempt to perform __setitem__ or __setattr__"
88
+            " %r on %s %r" % (cn, k, self.__dict__)
89
+        )
90
+
91
+    def __setattr__(self, k, v):
92
+        self.__setitem__(k, v)

+ 3
- 1
designate/tests/unit/test_mdns/test_notify.py View File

@@ -161,6 +161,7 @@ class MdnsNotifyTest(base.BaseTestCase):
161 161
             rcode=Mock(return_value=dns.rcode.NOERROR),
162 162
             # rcode is NOERROR but (flags & dns.flags.AA) gives 0
163 163
             flags=0,
164
+            answer=['answer'],
164 165
         )
165 166
         self.notify._send_dns_message = Mock(return_value=response)
166 167
 
@@ -176,7 +177,8 @@ class MdnsNotifyTest(base.BaseTestCase):
176 177
             rcode=Mock(return_value=dns.rcode.NOERROR),
177 178
             # rcode is NOERROR but flags are not NOERROR
178 179
             flags=123,
179
-            ednsflags=321
180
+            ednsflags=321,
181
+            answer=['answer'],
180 182
         )
181 183
         self.notify._send_dns_message = Mock(return_value=response)
182 184
 

+ 28
- 0
designate/tests/unit/test_pool_manager/test_service.py View File

@@ -29,6 +29,7 @@ from designate import exceptions
29 29
 from designate import objects
30 30
 from designate.pool_manager.service import Service
31 31
 from designate.tests.unit import RoObject
32
+from designate.tests.unit import RwObject
32 33
 import designate.pool_manager.service as pm_module
33 34
 
34 35
 
@@ -180,3 +181,30 @@ class PoolManagerTest(test.BaseTestCase):
180 181
         self.pm.periodic_sync()
181 182
 
182 183
         self.assertEqual(3, self.pm.update_zone.call_count)
184
+
185
+    @patch.object(pm_module.DesignateContext, 'get_admin_context')
186
+    def test_create_zone(self, mock_get_ctx, *mocks):
187
+        z = RwObject(name='a_zone', serial=1)
188
+
189
+        mock_ctx = mock_get_ctx.return_value
190
+        self.pm._exceed_or_meet_threshold = Mock(return_value=True)
191
+
192
+        self.pm.create_zone(mock_ctx, z)
193
+
194
+    @patch.object(pm_module.DesignateContext, 'get_admin_context')
195
+    def test_update_zone(self, mock_get_ctx, *mocks):
196
+        z = RwObject(name='a_zone', serial=1)
197
+
198
+        mock_ctx = mock_get_ctx.return_value
199
+        self.pm._exceed_or_meet_threshold = Mock(return_value=True)
200
+
201
+        self.pm.update_zone(mock_ctx, z)
202
+
203
+    @patch.object(pm_module.DesignateContext, 'get_admin_context')
204
+    def test_delete_zone(self, mock_get_ctx, *mocks):
205
+        z = RwObject(name='a_zone', serial=1)
206
+
207
+        mock_ctx = mock_get_ctx.return_value
208
+        self.pm._exceed_or_meet_threshold = Mock(return_value=True)
209
+
210
+        self.pm.delete_zone(mock_ctx, z)

+ 1
- 1
devstack/gate/run_tempest_tests.sh View File

@@ -29,4 +29,4 @@ TEMPEST_DIR=${TEMPEST_DIR:-/opt/stack/new/tempest}
29 29
 
30 30
 pushd $DESIGNATE_DIR
31 31
 export TEMPEST_CONFIG=$TEMPEST_DIR/etc/tempest.conf
32
-tox -e functional
32
+tox -e functional -- --concurrency 4

+ 1
- 1
functionaltests/common/utils.py View File

@@ -87,7 +87,7 @@ def parameterized(data):
87 87
     return wrapped
88 88
 
89 89
 
90
-def wait_for_condition(condition, interval=1, timeout=40):
90
+def wait_for_condition(condition, interval=5, timeout=45):
91 91
     end_time = time.time() + timeout
92 92
     while time.time() < end_time:
93 93
         result = condition()

Loading…
Cancel
Save