Add manila-manage db purge command

Cloud admins delete lots of DB records manually that were
"soft-deleted" in Manila. They do it to reduce size of DB
and increase response speed. Therefore, add "manila-manage
db purge" command that will make cloud admins life easier
allowing them to purge such DB records in one click.

Examples of usage:
$ manila-manage db purge 5

Partially-implements: blueprint clean-deleted-row-in-db
Depends-on: 0ec71e29b2
Change-Id: I4fd83d44f4a1b6b7633a60249ba775f6d86d1cdc

manila/cmd/manage.py

@@ -240,6 +240,19 @@ class DbCommands(object):
         """Stamp the version table with the given version."""
         return migration.stamp(version)
 
+    @args('age_in_days', type=int, default=0, nargs='?',
+          help='A non-negative integer, denoting the age of soft-deleted '
+               'records in number of days. 0 can be specified to purge all '
+               'soft-deleted rows, default is %(default)d.')
+    def purge(self, age_in_days):
+        """Purge soft-deleted records older than a given age."""
+        age_in_days = int(age_in_days)
+        if age_in_days < 0:
+            print(_("Must supply a non-negative value for age."))
+            exit(1)
+        ctxt = context.get_admin_context()
+        db.purge_deleted_records(ctxt, age_in_days)
+
 
 class VersionCommands(object):
     """Class for exposing the codebase version."""

manila/db/api.py

@@ -1061,3 +1061,11 @@ def share_replica_update(context, share_replica_id, values,
 def share_replica_delete(context, share_replica_id):
     """Deletes a share replica."""
     return IMPL.share_replica_delete(context, share_replica_id)
+
+
+def purge_deleted_records(context, age_in_days):
+    """Purge deleted rows older than given age from all tables
+
+    :raises: InvalidParameterValue if age_in_days is incorrect.
+    """
+    return IMPL.purge_deleted_records(context, age_in_days=age_in_days)

manila/db/sqlalchemy/api.py

@@ -29,6 +29,7 @@ import manila.db.sqlalchemy.query  # noqa
 
 from oslo_config import cfg
 from oslo_db import api as oslo_db_api
+from oslo_db import exception as db_exc
 from oslo_db import exception as db_exception
 from oslo_db import options as db_options
 from oslo_db.sqlalchemy import session
@@ -38,6 +39,7 @@ from oslo_utils import timeutils
 from oslo_utils import uuidutils
 import six
 from sqlalchemy import and_
+from sqlalchemy import MetaData
 from sqlalchemy import or_
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import true
@@ -46,7 +48,7 @@ from sqlalchemy.sql import func
 from manila.common import constants
 from manila.db.sqlalchemy import models
 from manila import exception
-from manila.i18n import _, _LE, _LW
+from manila.i18n import _, _LE, _LI, _LW
 
 CONF = cfg.CONF
 
@@ -3753,3 +3755,52 @@ def cgsnapshot_member_update(context, member_id, values):
         session.add(member)
 
         return cgsnapshot_member_get(context, member_id, session=session)
+
+
+@require_admin_context
+def purge_deleted_records(context, age_in_days):
+    """Purge soft-deleted records older than(and equal) age from tables."""
+
+    if age_in_days < 0:
+        msg = _('Must supply a non-negative value for "age_in_days".')
+        LOG.error(msg)
+        raise exception.InvalidParameterValue(msg)
+
+    metadata = MetaData()
+    metadata.reflect(get_engine())
+    session = get_session()
+    session.begin()
+    deleted_age = timeutils.utcnow() - datetime.timedelta(days=age_in_days)
+
+    for table in reversed(metadata.sorted_tables):
+        if 'deleted' in table.columns.keys():
+            try:
+                mds = [m for m in models.__dict__.values() if
+                       (hasattr(m, '__tablename__') and
+                        m.__tablename__ == six.text_type(table))]
+                if len(mds) > 0:
+                    # collect all soft-deleted records
+                    with session.begin_nested():
+                        model = mds[0]
+                        s_deleted_records = session.query(model).filter(
+                            model.deleted_at <= deleted_age)
+                    deleted_count = 0
+                    # delete records one by one,
+                    # skip the records which has FK constraints
+                    for record in s_deleted_records:
+                        try:
+                            with session.begin_nested():
+                                session.delete(record)
+                                deleted_count += 1
+                        except db_exc.DBError:
+                            LOG.warning(
+                                _LW("Deleting soft-deleted resource %s "
+                                    "failed, skipping."), record)
+                    if deleted_count != 0:
+                        LOG.info(_LI("Deleted %(count)s records in "
+                                     "table %(table)s."),
+                                 {'count': deleted_count, 'table': table})
+            except db_exc.DBError:
+                LOG.warning(_LW("Querying table %s's soft-deleted records "
+                                "failed, skipping."), table)
+    session.commit()

manila/tests/db/sqlalchemy/test_api.py

@@ -18,11 +18,13 @@
 """Testing of SQLAlchemy backend."""
 
 import copy
-
+import datetime
 import ddt
 import mock
+import random
 
 from oslo_db import exception as db_exception
+from oslo_utils import timeutils
 from oslo_utils import uuidutils
 import six
 
@@ -2200,3 +2202,146 @@ class NetworkAllocationsDatabaseAPITestCase(test.TestCase):
         )
         for na in result:
             self.assertIn(na.label, ('admin', 'user', None))
+
+
+@ddt.ddt
+class PurgeDeletedTest(test.TestCase):
+
+    def setUp(self):
+        super(PurgeDeletedTest, self).setUp()
+        self.context = context.get_admin_context()
+
+    def _days_ago(self, begin, end):
+        return timeutils.utcnow() - datetime.timedelta(
+            days=random.randint(begin, end))
+
+    def _sqlite_has_fk_constraint(self):
+        # SQLAlchemy doesn't support it at all with < SQLite 3.6.19
+        import sqlite3
+        tup = sqlite3.sqlite_version_info
+        return tup[0] > 3 or (tup[0] == 3 and tup[1] >= 7)
+
+    def _turn_on_foreign_key(self):
+        engine = db_api.get_engine()
+        connection = engine.raw_connection()
+        try:
+            cursor = connection.cursor()
+            cursor.execute("PRAGMA foreign_keys = ON")
+        finally:
+            connection.close()
+
+    @ddt.data({"del_days": 0, "num_left": 0},
+              {"del_days": 10, "num_left": 2},
+              {"del_days": 20, "num_left": 4})
+    @ddt.unpack
+    def test_purge_records_with_del_days(self, del_days, num_left):
+        fake_now = timeutils.utcnow()
+        with mock.patch.object(timeutils, 'utcnow',
+                               mock.Mock(return_value=fake_now)):
+            # create resources soft-deleted in 0~9, 10~19 days ago
+            for start, end in ((0, 9), (10, 19)):
+                for unused in range(2):
+                    # share type
+                    db_utils.create_share_type(id=uuidutils.generate_uuid(),
+                                               deleted_at=self._days_ago(start,
+                                                                         end))
+                    # share
+                    share = db_utils.create_share_without_instance(
+                        metadata={},
+                        deleted_at=self._days_ago(start, end))
+                    # create share network
+                    network = db_utils.create_share_network(
+                        id=uuidutils.generate_uuid(),
+                        deleted_at=self._days_ago(start, end))
+                    # create security service
+                    db_utils.create_security_service(
+                        id=uuidutils.generate_uuid(),
+                        share_network_id=network.id,
+                        deleted_at=self._days_ago(start, end))
+                    # create share instance
+                    s_instance = db_utils.create_share_instance(
+                        id=uuidutils.generate_uuid(),
+                        share_network_id=network.id,
+                        share_id=share.id)
+                    # share access
+                    db_utils.create_share_access(
+                        id=uuidutils.generate_uuid(),
+                        share_id=share['id'],
+                        deleted_at=self._days_ago(start, end))
+                    # create share server
+                    db_utils.create_share_server(
+                        id=uuidutils.generate_uuid(),
+                        deleted_at=self._days_ago(start, end),
+                        share_network_id=network.id)
+                    # create snapshot
+                    db_api.share_snapshot_create(
+                        self.context, {'share_id': share['id'],
+                                       'deleted_at': self._days_ago(start,
+                                                                    end)},
+                        create_snapshot_instance=False)
+                    # create consistency group
+                    cg = db_utils.create_consistency_group(
+                        deleted_at=self._days_ago(start, end))
+                    # create cg snapshot
+                    db_utils.create_cgsnapshot(
+                        cg.id, deleted_at=self._days_ago(start, end))
+                    # create cgsnapshot member
+                    db_api.cgsnapshot_member_create(
+                        self.context,
+                        {'id': uuidutils.generate_uuid(),
+                         'share_id': share.id,
+                         'share_instance_id': s_instance.id,
+                         'deleted_at': self._days_ago(start, end)})
+                    # update share instance
+                    db_api.share_instance_update(
+                        self.context,
+                        s_instance.id,
+                        {'deleted_at': self._days_ago(start, end)})
+
+            db_api.purge_deleted_records(self.context, age_in_days=del_days)
+
+            for model in [models.ShareTypes, models.Share,
+                          models.ShareNetwork, models.ShareAccessMapping,
+                          models.ShareInstance, models.ShareServer,
+                          models.ShareSnapshot, models.ConsistencyGroup,
+                          models.CGSnapshot, models.SecurityService,
+                          models.CGSnapshotMember]:
+                rows = db_api.model_query(self.context, model).count()
+                self.assertEqual(num_left, rows)
+
+    def test_purge_records_with_illegal_args(self):
+        self.assertRaises(TypeError, db_api.purge_deleted_records,
+                          self.context)
+        self.assertRaises(exception.InvalidParameterValue,
+                          db_api.purge_deleted_records,
+                          self.context,
+                          age_in_days=-1)
+
+    def test_purge_records_with_constraint(self):
+        if not self._sqlite_has_fk_constraint():
+            self.skipTest(
+                'sqlite is too old for reliable SQLA foreign_keys')
+        self._turn_on_foreign_key()
+        type_id = uuidutils.generate_uuid()
+        # create share type1
+        db_utils.create_share_type(id=type_id,
+                                   deleted_at=self._days_ago(1, 1))
+        # create share type2
+        db_utils.create_share_type(id=uuidutils.generate_uuid(),
+                                   deleted_at=self._days_ago(1, 1))
+        # create share
+        share = db_utils.create_share(share_type_id=type_id)
+
+        db_api.purge_deleted_records(self.context, age_in_days=0)
+        type_row = db_api.model_query(self.context,
+                                      models.ShareTypes).count()
+        # share type1 should not be deleted
+        self.assertEqual(1, type_row)
+        db_api.model_query(self.context, models.ShareInstance).delete()
+        db_api.share_delete(self.context, share['id'])
+
+        db_api.purge_deleted_records(self.context, age_in_days=0)
+        s_row = db_api.model_query(self.context, models.Share).count()
+        type_row = db_api.model_query(self.context,
+                                      models.ShareTypes).count()
+        self.assertEqual(0, s_row + type_row)

manila/tests/db_utils.py

@@ -68,6 +68,15 @@ def create_cgsnapshot_member(cgsnapshot_id, **kwargs):
     return _create_db_row(db.cgsnapshot_member_create, member, kwargs)
 
 
+def create_share_access(**kwargs):
+    share_access = {
+        'id': 'fake_id',
+        'access_type': 'ip',
+        'access_to': 'fake_ip_address'
+    }
+    return _create_db_row(db.share_access_create, share_access, kwargs)
+
+
 def create_share(**kwargs):
     """Create a share object."""
     share = {
@@ -86,6 +95,24 @@ def create_share(**kwargs):
     return _create_db_row(db.share_create, share, kwargs)
 
 
+def create_share_without_instance(**kwargs):
+    share = {
+        'share_proto': "NFS",
+        'size': 0,
+        'snapshot_id': None,
+        'share_network_id': None,
+        'share_server_id': None,
+        'user_id': 'fake',
+        'project_id': 'fake',
+        'metadata': {},
+        'availability_zone': 'fake_availability_zone',
+        'status': constants.STATUS_CREATING,
+        'host': 'fake_host'
+    }
+    share.update(copy.deepcopy(kwargs))
+    return db.share_create(context.get_admin_context(), share, False)
+
+
 def create_share_instance(**kwargs):
     """Create a share instance object."""
     return db.share_instance_create(context.get_admin_context(),
@@ -165,6 +192,17 @@ def create_share_server(**kwargs):
                                share_srv['id'])
 
 
+def create_share_type(**kwargs):
+    """Create a share type object"""
+
+    share_type = {
+        'name': 'fake_type',
+        'is_public': True,
+    }
+
+    return _create_db_row(db.share_type_create, share_type, kwargs)
+
+
 def create_share_network(**kwargs):
     """Create a share network object."""
     net = {

releasenotes/notes/add-manage-db-purge-b32a24ee045d8d45.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Add ``purge`` sub command to the ``manila-manage db`` command for
+    administrators to be able to purge soft-deleted rows.