Browse Source

Merge "Mitigate OSSN-0075" into stable/queens

stable/queens
Zuul 6 days ago
parent
commit
cc4f74d112
3 changed files with 89 additions and 12 deletions
  1. 22
    10
      glance/cmd/manage.py
  2. 49
    0
      glance/db/sqlalchemy/api.py
  3. 18
    2
      glance/tests/functional/db/base.py

+ 22
- 10
glance/cmd/manage.py View File

@@ -329,24 +329,17 @@ class DbCommands(object):
329 329
         metadata.db_export_metadefs(db_api.get_engine(),
330 330
                                     path)
331 331
 
332
-    @args('--age_in_days', type=int,
333
-          help='Purge deleted rows older than age in days')
334
-    @args('--max_rows', type=int,
335
-          help='Limit number of records to delete')
336
-    def purge(self, age_in_days=30, max_rows=100):
337
-        """Purge deleted rows older than a given age from glance tables."""
332
+    def _purge(self, age_in_days, max_rows, purge_images_only=False):
338 333
         try:
339 334
             age_in_days = int(age_in_days)
340 335
         except ValueError:
341 336
             sys.exit(_("Invalid int value for age_in_days: "
342 337
                        "%(age_in_days)s") % {'age_in_days': age_in_days})
343
-
344 338
         try:
345 339
             max_rows = int(max_rows)
346 340
         except ValueError:
347 341
             sys.exit(_("Invalid int value for max_rows: "
348 342
                        "%(max_rows)s") % {'max_rows': max_rows})
349
-
350 343
         if age_in_days < 0:
351 344
             sys.exit(_("Must supply a non-negative value for age."))
352 345
         if age_in_days >= (int(time.time()) / 86400):
@@ -354,15 +347,34 @@ class DbCommands(object):
354 347
         if max_rows < 1:
355 348
             sys.exit(_("Minimal rows limit is 1."))
356 349
         ctx = context.get_admin_context(show_deleted=True)
357
-
358 350
         try:
359
-            db_api.purge_deleted_rows(ctx, age_in_days, max_rows)
351
+            if purge_images_only:
352
+                db_api.purge_deleted_rows_from_images(ctx, age_in_days,
353
+                                                      max_rows)
354
+            else:
355
+                db_api.purge_deleted_rows(ctx, age_in_days, max_rows)
360 356
         except exception.Invalid as exc:
361 357
             sys.exit(exc.msg)
362 358
         except db_exc.DBReferenceError:
363 359
             sys.exit(_("Purge command failed, check glance-manage"
364 360
                        " logs for more details."))
365 361
 
362
+    @args('--age_in_days', type=int,
363
+          help='Purge deleted rows older than age in days')
364
+    @args('--max_rows', type=int,
365
+          help='Limit number of records to delete')
366
+    def purge(self, age_in_days=30, max_rows=100):
367
+        """Purge deleted rows older than a given age from glance tables."""
368
+        self._purge(age_in_days, max_rows)
369
+
370
+    @args('--age_in_days', type=int,
371
+          help='Purge deleted rows older than age in days')
372
+    @args('--max_rows', type=int,
373
+          help='Limit number of records to delete')
374
+    def purge_images_table(self, age_in_days=30, max_rows=100):
375
+        """Purge deleted rows older than a given age from images table."""
376
+        self._purge(age_in_days, max_rows, purge_images_only=True)
377
+
366 378
 
367 379
 class DbLegacyCommands(object):
368 380
     """Class for managing the db using legacy commands"""

+ 49
- 0
glance/db/sqlalchemy/api.py View File

@@ -1307,6 +1307,11 @@ def purge_deleted_rows(context, age_in_days, max_rows, session=None):
1307 1307
             LOG.warning(_LW('Expected table %(tbl)s was not found in DB.'),
1308 1308
                         {'tbl': tbl})
1309 1309
         else:
1310
+            # NOTE(abhishekk): To mitigate OSSN-0075 images records should be
1311
+            # purged with new ``purge-images-table`` command.
1312
+            if tbl == 'images':
1313
+                continue
1314
+
1310 1315
             tables.append(tbl)
1311 1316
 
1312 1317
     for tbl in tables:
@@ -1339,6 +1344,50 @@ def purge_deleted_rows(context, age_in_days, max_rows, session=None):
1339 1344
                  {'rows': rows, 'tbl': tbl})
1340 1345
 
1341 1346
 
1347
+def purge_deleted_rows_from_images(context, age_in_days, max_rows,
1348
+                                   session=None):
1349
+    """Purges soft deleted rows
1350
+
1351
+    Deletes rows of table images table according to given age for
1352
+    relevant models.
1353
+    """
1354
+    # check max_rows for its maximum limit
1355
+    _validate_db_int(max_rows=max_rows)
1356
+
1357
+    session = session or get_session()
1358
+    metadata = MetaData(get_engine())
1359
+    deleted_age = timeutils.utcnow() - datetime.timedelta(days=age_in_days)
1360
+
1361
+    tbl = 'images'
1362
+    tab = Table(tbl, metadata, autoload=True)
1363
+    LOG.info(
1364
+        _LI('Purging deleted rows older than %(age_in_days)d day(s) '
1365
+            'from table %(tbl)s'),
1366
+        {'age_in_days': age_in_days, 'tbl': tbl})
1367
+
1368
+    column = tab.c.id
1369
+    deleted_at_column = tab.c.deleted_at
1370
+
1371
+    query_delete = sql.select(
1372
+        [column], deleted_at_column < deleted_age).order_by(
1373
+        deleted_at_column).limit(max_rows)
1374
+
1375
+    delete_statement = DeleteFromSelect(tab, query_delete, column)
1376
+
1377
+    try:
1378
+        with session.begin():
1379
+            result = session.execute(delete_statement)
1380
+    except db_exception.DBReferenceError as ex:
1381
+        with excutils.save_and_reraise_exception():
1382
+            LOG.error(_LE('DBError detected when purging from '
1383
+                      "%(tablename)s: %(error)s"),
1384
+                      {'tablename': tbl, 'error': six.text_type(ex)})
1385
+
1386
+    rows = result.rowcount
1387
+    LOG.info(_LI('Deleted %(rows)d row(s) from table %(tbl)s'),
1388
+             {'rows': rows, 'tbl': tbl})
1389
+
1390
+
1342 1391
 def user_get_storage_usage(context, owner_id, image_id=None, session=None):
1343 1392
     _check_image_id(image_id)
1344 1393
     session = session or get_session()

+ 18
- 2
glance/tests/functional/db/base.py View File

@@ -1990,11 +1990,27 @@ class DBPurgeTests(test_utils.BaseTestCase):
1990 1990
     def test_db_purge(self):
1991 1991
         self.db_api.purge_deleted_rows(self.adm_context, 1, 5)
1992 1992
         images = self.db_api.image_get_all(self.adm_context)
1993
+
1994
+        # Verify that no records from images have been deleted
1995
+        # as images table will be purged using 'purge_images_table'
1996
+        # command.
1997
+        self.assertEqual(len(images), 3)
1998
+        tasks = self.db_api.task_get_all(self.adm_context)
1999
+        self.assertEqual(len(tasks), 2)
2000
+
2001
+    def test_db_purge_images_table(self):
2002
+        # purge records from images_tags table
2003
+        self.db_api.purge_deleted_rows(self.adm_context, 1, 5)
2004
+
2005
+        # purge records from images table
2006
+        self.db_api.purge_deleted_rows_from_images(self.adm_context, 1, 5)
2007
+        images = self.db_api.image_get_all(self.adm_context)
2008
+
1993 2009
         self.assertEqual(len(images), 2)
1994 2010
         tasks = self.db_api.task_get_all(self.adm_context)
1995 2011
         self.assertEqual(len(tasks), 2)
1996 2012
 
1997
-    def test_purge_fk_constraint_failure(self):
2013
+    def test_purge_images_table_fk_constraint_failure(self):
1998 2014
         """Test foreign key constraint failure
1999 2015
 
2000 2016
         Test whether foreign key constraint failure during purge
@@ -2053,7 +2069,7 @@ class DBPurgeTests(test_utils.BaseTestCase):
2053 2069
 
2054 2070
         # Purge all records deleted at least 10 days ago
2055 2071
         self.assertRaises(db_exception.DBReferenceError,
2056
-                          db_api.purge_deleted_rows,
2072
+                          db_api.purge_deleted_rows_from_images,
2057 2073
                           self.adm_context,
2058 2074
                           age_in_days=10,
2059 2075
                           max_rows=50)

Loading…
Cancel
Save