Browse Source

Merge "Maintenance task: Enhance debug logging" into stable/queens

changes/07/675607/1
Zuul 1 month ago
parent
commit
e541246631

+ 47
- 6
networking_ovn/common/maintenance.py View File

@@ -20,6 +20,7 @@ from futurist import periodics
20 20
 from neutron.common import config as n_conf
21 21
 from neutron_lib import context as n_context
22 22
 from neutron_lib import worker
23
+from oslo_config import cfg
23 24
 from oslo_log import log
24 25
 from oslo_utils import timeutils
25 26
 
@@ -27,9 +28,12 @@ from networking_ovn.common import constants as ovn_const
27 28
 from networking_ovn.db import maintenance as db_maint
28 29
 from networking_ovn.db import revision as db_rev
29 30
 
31
+CONF = cfg.CONF
30 32
 LOG = log.getLogger(__name__)
31 33
 
32 34
 DB_CONSISTENCY_CHECK_INTERVAL = 300  # 5 minutes
35
+INCONSISTENCY_TYPE_CREATE_UPDATE = 'create/update'
36
+INCONSISTENCY_TYPE_DELETE = 'delete'
33 37
 
34 38
 
35 39
 class MaintenanceWorker(worker.BaseWorker):
@@ -203,6 +207,30 @@ class DBInconsistenciesPeriodics(object):
203 207
         else:
204 208
             self._ovn_client.update_subnet(sn_db_obj, n_db_obj)
205 209
 
210
+    def _log_maintenance_inconsistencies(self, create_update_inconsistencies,
211
+                                         delete_inconsistencies):
212
+        if not CONF.debug:
213
+            return
214
+
215
+        def _log(inconsistencies, type_):
216
+            if not inconsistencies:
217
+                return
218
+
219
+            c = {}
220
+            for f in inconsistencies:
221
+                if f.resource_type not in c:
222
+                    c[f.resource_type] = 1
223
+                else:
224
+                    c[f.resource_type] += 1
225
+
226
+            fail_str = ', '.join('{}={}'.format(k, v) for k, v in c.items())
227
+            LOG.debug('Maintenance task: Number of inconsistencies '
228
+                      'found at %(type_)s: %(fail_str)s',
229
+                      {'type_': type_, 'fail_str': fail_str})
230
+
231
+        _log(create_update_inconsistencies, INCONSISTENCY_TYPE_CREATE_UPDATE)
232
+        _log(delete_inconsistencies, INCONSISTENCY_TYPE_DELETE)
233
+
206 234
     @periodics.periodic(spacing=DB_CONSISTENCY_CHECK_INTERVAL,
207 235
                         run_immediately=True)
208 236
     def check_for_inconsistencies(self):
@@ -214,12 +242,22 @@ class DBInconsistenciesPeriodics(object):
214 242
         create_update_inconsistencies = db_maint.get_inconsistent_resources()
215 243
         delete_inconsistencies = db_maint.get_deleted_resources()
216 244
         if not any([create_update_inconsistencies, delete_inconsistencies]):
245
+            LOG.debug('Maintenance task: No inconsistencies found. Skipping')
217 246
             return
218
-        LOG.warning('Inconsistencies found in the database!')
247
+
248
+        LOG.debug('Maintenance task: Synchronizing Neutron '
249
+                  'and OVN databases')
250
+        self._log_maintenance_inconsistencies(create_update_inconsistencies,
251
+                                              delete_inconsistencies)
219 252
         self._sync_timer.restart()
220 253
 
254
+        dbg_log_msg = ('Maintenance task: Fixing resource %(res_uuid)s '
255
+                       '(type: %(res_type)s) at %(type_)s')
221 256
         # Fix the create/update resources inconsistencies
222 257
         for row in create_update_inconsistencies:
258
+            LOG.debug(dbg_log_msg, {'res_uuid': row.resource_uuid,
259
+                                    'res_type': row.resource_type,
260
+                                    'type_': INCONSISTENCY_TYPE_CREATE_UPDATE})
223 261
             try:
224 262
                 # NOTE(lucasagomes): The way to fix subnets is bit
225 263
                 # different than other resources. A subnet in OVN language
@@ -232,25 +270,28 @@ class DBInconsistenciesPeriodics(object):
232 270
                 else:
233 271
                     self._fix_create_update(row)
234 272
             except Exception:
235
-                LOG.exception('Failed to fix resource %(res_uuid)s '
236
-                              '(type: %(res_type)s)',
273
+                LOG.exception('Maintenance task: Failed to fix resource '
274
+                              '%(res_uuid)s (type: %(res_type)s)',
237 275
                               {'res_uuid': row.resource_uuid,
238 276
                                'res_type': row.resource_type})
239 277
 
240 278
         # Fix the deleted resources inconsistencies
241 279
         for row in delete_inconsistencies:
280
+            LOG.debug(dbg_log_msg, {'res_uuid': row.resource_uuid,
281
+                                    'res_type': row.resource_type,
282
+                                    'type_': INCONSISTENCY_TYPE_DELETE})
242 283
             try:
243 284
                 if row.resource_type == ovn_const.TYPE_SUBNETS:
244 285
                     self._ovn_client.delete_subnet(row.resource_uuid)
245 286
                 else:
246 287
                     self._fix_delete(row)
247 288
             except Exception:
248
-                LOG.exception('Failed to fix deleted resource %(res_uuid)s '
249
-                              '(type: %(res_type)s)',
289
+                LOG.exception('Maintenance task: Failed to fix deleted '
290
+                              'resource %(res_uuid)s (type: %(res_type)s)',
250 291
                               {'res_uuid': row.resource_uuid,
251 292
                                'res_type': row.resource_type})
252 293
 
253
-        LOG.info('Maintenance thread synchronization finished '
294
+        LOG.info('Maintenance task: Synchronization finished '
254 295
                  '(took %.2f seconds)', self._sync_timer.elapsed())
255 296
         self._sync_timer.stop()
256 297
 

+ 39
- 0
networking_ovn/tests/unit/common/test_maintenance.py View File

@@ -18,6 +18,7 @@ import mock
18 18
 from neutron.tests.unit.plugins.ml2 import test_security_group as test_sg
19 19
 from neutron_lib.db import api as db_api
20 20
 
21
+from networking_ovn.common import config as ovn_config
21 22
 from networking_ovn.common import constants
22 23
 from networking_ovn.common import maintenance
23 24
 from networking_ovn.common import utils
@@ -176,3 +177,41 @@ class TestDBInconsistenciesPeriodics(db_base.DBTestCase,
176 177
         l3_mock.add_router_interface.assert_called_once_with(
177 178
             mock.ANY, port['device_id'], {'port_id': port['id']},
178 179
             may_exist=True)
180
+
181
+    @mock.patch.object(maintenance.LOG, 'debug')
182
+    def test__log_maintenance_inconsistencies(self, mock_log):
183
+        ovn_config.cfg.CONF.set_override('debug', True)
184
+
185
+        # Create fake inconsistencies: 2 networks, 4 subnets and 8 ports
186
+        incst = []
187
+        incst += [mock.Mock(resource_type=constants.TYPE_NETWORKS)] * 2
188
+        incst += [mock.Mock(resource_type=constants.TYPE_SUBNETS)] * 4
189
+        incst += [mock.Mock(resource_type=constants.TYPE_PORTS)] * 8
190
+
191
+        # Create fake inconsistencies for delete: 3 routers and 6 router ports
192
+        incst_del = []
193
+        incst_del += [mock.Mock(resource_type=constants.TYPE_ROUTERS)] * 3
194
+        incst_del += [mock.Mock(resource_type=constants.TYPE_ROUTER_PORTS)] * 6
195
+
196
+        self.periodic._log_maintenance_inconsistencies(incst, incst_del)
197
+
198
+        # Assert LOG.debug was called twice
199
+        self.assertEqual(2, len(mock_log.call_args_list))
200
+
201
+        # Assert the log matches the number of inconsistencies
202
+        fail_str_create_update = mock_log.call_args_list[0][0][1]['fail_str']
203
+        self.assertIn('networks=2', fail_str_create_update)
204
+        self.assertIn('subnets=4', fail_str_create_update)
205
+        self.assertIn('ports=8', fail_str_create_update)
206
+
207
+        fail_str_delete = mock_log.call_args_list[1][0][1]['fail_str']
208
+        self.assertIn('routers=3', fail_str_delete)
209
+        self.assertIn('router_ports=6', fail_str_delete)
210
+
211
+    @mock.patch.object(maintenance.LOG, 'debug')
212
+    def test__log_maintenance_inconsistencies_debug_disabled(self, mock_log):
213
+        ovn_config.cfg.CONF.set_override('debug', False)
214
+
215
+        incst = [mock.Mock(resource_type=constants.TYPE_NETWORKS)] * 2
216
+        self.periodic._log_maintenance_inconsistencies(incst, [])
217
+        self.assertFalse(mock_log.called)

Loading…
Cancel
Save