Browse Source

Merge "Maintenance task: Enhance debug logging" into stable/rocky

changes/03/675603/1
Zuul 1 month ago
parent
commit
9fa9b64b92

+ 45
- 5
networking_ovn/common/maintenance.py View File

@@ -21,6 +21,7 @@ from neutron.common import config as n_conf
21 21
 from neutron_lib import context as n_context
22 22
 from neutron_lib import exceptions as n_exc
23 23
 from neutron_lib import worker
24
+from oslo_config import cfg
24 25
 from oslo_log import log
25 26
 from oslo_utils import timeutils
26 27
 
@@ -29,9 +30,12 @@ from networking_ovn.db import maintenance as db_maint
29 30
 from networking_ovn.db import revision as db_rev
30 31
 from networking_ovn import ovn_db_sync
31 32
 
33
+CONF = cfg.CONF
32 34
 LOG = log.getLogger(__name__)
33 35
 
34 36
 DB_CONSISTENCY_CHECK_INTERVAL = 300  # 5 minutes
37
+INCONSISTENCY_TYPE_CREATE_UPDATE = 'create/update'
38
+INCONSISTENCY_TYPE_DELETE = 'delete'
35 39
 
36 40
 
37 41
 class MaintenanceWorker(worker.BaseWorker):
@@ -244,6 +248,30 @@ class DBInconsistenciesPeriodics(object):
244 248
         nb_sync.migrate_to_port_groups(admin_context)
245 249
         raise periodics.NeverAgain()
246 250
 
251
+    def _log_maintenance_inconsistencies(self, create_update_inconsistencies,
252
+                                         delete_inconsistencies):
253
+        if not CONF.debug:
254
+            return
255
+
256
+        def _log(inconsistencies, type_):
257
+            if not inconsistencies:
258
+                return
259
+
260
+            c = {}
261
+            for f in inconsistencies:
262
+                if f.resource_type not in c:
263
+                    c[f.resource_type] = 1
264
+                else:
265
+                    c[f.resource_type] += 1
266
+
267
+            fail_str = ', '.join('{}={}'.format(k, v) for k, v in c.items())
268
+            LOG.debug('Maintenance task: Number of inconsistencies '
269
+                      'found at %(type_)s: %(fail_str)s',
270
+                      {'type_': type_, 'fail_str': fail_str})
271
+
272
+        _log(create_update_inconsistencies, INCONSISTENCY_TYPE_CREATE_UPDATE)
273
+        _log(delete_inconsistencies, INCONSISTENCY_TYPE_DELETE)
274
+
247 275
     @periodics.periodic(spacing=DB_CONSISTENCY_CHECK_INTERVAL,
248 276
                         run_immediately=True)
249 277
     def check_for_inconsistencies(self):
@@ -255,13 +283,22 @@ class DBInconsistenciesPeriodics(object):
255 283
         create_update_inconsistencies = db_maint.get_inconsistent_resources()
256 284
         delete_inconsistencies = db_maint.get_deleted_resources()
257 285
         if not any([create_update_inconsistencies, delete_inconsistencies]):
286
+            LOG.debug('Maintenance task: No inconsistencies found. Skipping')
258 287
             return
288
+
259 289
         LOG.debug('Maintenance task: Synchronizing Neutron '
260 290
                   'and OVN databases')
291
+        self._log_maintenance_inconsistencies(create_update_inconsistencies,
292
+                                              delete_inconsistencies)
261 293
         self._sync_timer.restart()
262 294
 
295
+        dbg_log_msg = ('Maintenance task: Fixing resource %(res_uuid)s '
296
+                       '(type: %(res_type)s) at %(type_)s')
263 297
         # Fix the create/update resources inconsistencies
264 298
         for row in create_update_inconsistencies:
299
+            LOG.debug(dbg_log_msg, {'res_uuid': row.resource_uuid,
300
+                                    'res_type': row.resource_type,
301
+                                    'type_': INCONSISTENCY_TYPE_CREATE_UPDATE})
265 302
             try:
266 303
                 # NOTE(lucasagomes): The way to fix subnets is bit
267 304
                 # different than other resources. A subnet in OVN language
@@ -274,26 +311,29 @@ class DBInconsistenciesPeriodics(object):
274 311
                 else:
275 312
                     self._fix_create_update(row)
276 313
             except Exception:
277
-                LOG.exception('Failed to fix resource %(res_uuid)s '
278
-                              '(type: %(res_type)s)',
314
+                LOG.exception('Maintenance task: Failed to fix resource '
315
+                              '%(res_uuid)s (type: %(res_type)s)',
279 316
                               {'res_uuid': row.resource_uuid,
280 317
                                'res_type': row.resource_type})
281 318
 
282 319
         # Fix the deleted resources inconsistencies
283 320
         for row in delete_inconsistencies:
321
+            LOG.debug(dbg_log_msg, {'res_uuid': row.resource_uuid,
322
+                                    'res_type': row.resource_type,
323
+                                    'type_': INCONSISTENCY_TYPE_DELETE})
284 324
             try:
285 325
                 if row.resource_type == ovn_const.TYPE_SUBNETS:
286 326
                     self._ovn_client.delete_subnet(row.resource_uuid)
287 327
                 else:
288 328
                     self._fix_delete(row)
289 329
             except Exception:
290
-                LOG.exception('Failed to fix deleted resource %(res_uuid)s '
291
-                              '(type: %(res_type)s)',
330
+                LOG.exception('Maintenance task: Failed to fix deleted '
331
+                              'resource %(res_uuid)s (type: %(res_type)s)',
292 332
                               {'res_uuid': row.resource_uuid,
293 333
                                'res_type': row.resource_type})
294 334
 
295 335
         self._sync_timer.stop()
296
-        LOG.info('Maintenance task synchronization finished '
336
+        LOG.info('Maintenance task: Synchronization finished '
297 337
                  '(took %.2f seconds)', self._sync_timer.elapsed())
298 338
 
299 339
     def _create_lrouter_port(self, port):

+ 39
- 0
networking_ovn/tests/unit/common/test_maintenance.py View File

@@ -19,6 +19,7 @@ from futurist import periodics
19 19
 from neutron.tests.unit.plugins.ml2 import test_security_group as test_sg
20 20
 from neutron_lib.db import api as db_api
21 21
 
22
+from networking_ovn.common import config as ovn_config
22 23
 from networking_ovn.common import constants
23 24
 from networking_ovn.common import maintenance
24 25
 from networking_ovn.common import utils
@@ -229,3 +230,41 @@ class TestDBInconsistenciesPeriodics(db_base.DBTestCase,
229 230
         l3_mock.add_router_interface.assert_called_once_with(
230 231
             mock.ANY, port['device_id'], {'port_id': port['id']},
231 232
             may_exist=True)
233
+
234
+    @mock.patch.object(maintenance.LOG, 'debug')
235
+    def test__log_maintenance_inconsistencies(self, mock_log):
236
+        ovn_config.cfg.CONF.set_override('debug', True)
237
+
238
+        # Create fake inconsistencies: 2 networks, 4 subnets and 8 ports
239
+        incst = []
240
+        incst += [mock.Mock(resource_type=constants.TYPE_NETWORKS)] * 2
241
+        incst += [mock.Mock(resource_type=constants.TYPE_SUBNETS)] * 4
242
+        incst += [mock.Mock(resource_type=constants.TYPE_PORTS)] * 8
243
+
244
+        # Create fake inconsistencies for delete: 3 routers and 6 router ports
245
+        incst_del = []
246
+        incst_del += [mock.Mock(resource_type=constants.TYPE_ROUTERS)] * 3
247
+        incst_del += [mock.Mock(resource_type=constants.TYPE_ROUTER_PORTS)] * 6
248
+
249
+        self.periodic._log_maintenance_inconsistencies(incst, incst_del)
250
+
251
+        # Assert LOG.debug was called twice
252
+        self.assertEqual(2, len(mock_log.call_args_list))
253
+
254
+        # Assert the log matches the number of inconsistencies
255
+        fail_str_create_update = mock_log.call_args_list[0][0][1]['fail_str']
256
+        self.assertIn('networks=2', fail_str_create_update)
257
+        self.assertIn('subnets=4', fail_str_create_update)
258
+        self.assertIn('ports=8', fail_str_create_update)
259
+
260
+        fail_str_delete = mock_log.call_args_list[1][0][1]['fail_str']
261
+        self.assertIn('routers=3', fail_str_delete)
262
+        self.assertIn('router_ports=6', fail_str_delete)
263
+
264
+    @mock.patch.object(maintenance.LOG, 'debug')
265
+    def test__log_maintenance_inconsistencies_debug_disabled(self, mock_log):
266
+        ovn_config.cfg.CONF.set_override('debug', False)
267
+
268
+        incst = [mock.Mock(resource_type=constants.TYPE_NETWORKS)] * 2
269
+        self.periodic._log_maintenance_inconsistencies(incst, [])
270
+        self.assertFalse(mock_log.called)

Loading…
Cancel
Save