Implement purge_deleted command

This patch adds the missing implementation of 'purge_deleted'. The
command is useful when users are accumulating a lot of soft-deleted
objects in database.

Change-Id: Ice35ce3c90acfbde42be707f79c90d99b1c51819
Closes-Bug: #1526228

releasenotes/notes/purge-deleted-support-91ad29b09d75b35a.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Support to 'senlin-manage purge_deleted <age> [<unit>]' is added.

senlin/cmd/manage.py

@@ -43,7 +43,7 @@ def do_db_sync():
 def purge_deleted():
     """Remove database records that have been previously soft deleted."""
 
-    utils.purge_deleted(CONF.command.age, CONF.command.granularity)
+    utils.purge_deleted(CONF.command.age, CONF.command.unit)
 
 
 def add_command_parsers(subparsers):
@@ -60,9 +60,9 @@ def add_command_parsers(subparsers):
     parser.add_argument('age', nargs='?', default='90',
                         help=_('How long to preserve deleted data.'))
     parser.add_argument(
-        '-g', '--granularity', default='days',
+        '-u', '--unit', default='days',
         choices=['days', 'hours', 'minutes', 'seconds'],
-        help=_('Granularity to use for age argument, defaults to days.'))
+        help=_('Unit to use for age argument, defaults to days.'))
 
 command_opt = cfg.SubCommandOpt('command',
                                 title='Commands',

senlin/db/sqlalchemy/api.py

@@ -14,6 +14,7 @@
 Implementation of SQLAlchemy backend.
 '''
 
+import datetime
 import six
 import sys
 
@@ -23,6 +24,7 @@ from oslo_db.sqlalchemy import utils
 from oslo_log import log as logging
 from oslo_utils import timeutils
 
+import sqlalchemy
 from sqlalchemy.orm import session as orm_session
 
 from senlin.common import consts
@@ -1043,54 +1045,6 @@ def event_get_all_by_cluster(context, cluster_id, limit=None, marker=None,
                                         sort_dir=sort_dir)
 
 
-def purge_deleted(age, granularity='days'):
-    pass
-#    try:
-#        age = int(age)
-#    except ValueError:
-#        raise exception.Error(_("age should be an integer"))
-#    if age < 0:
-#        raise exception.Error(_("age should be a positive integer"))
-#
-#    if granularity not in ('days', 'hours', 'minutes', 'seconds'):
-#        raise exception.Error(
-#            _("granularity should be days, hours, minutes, or seconds"))
-#
-#    if granularity == 'days':
-#        age = age * 86400
-#    elif granularity == 'hours':
-#        age = age * 3600
-#    elif granularity == 'minutes':
-#        age = age * 60
-#
-#    time_line = datetime.datetime.now() - datetime.timedelta(seconds=age)
-#    engine = get_engine()
-#    meta = sqlalchemy.MetaData()
-#    meta.bind = engine
-#
-#    cluster = sqlalchemy.Table('cluster', meta, autoload=True)
-#    event = sqlalchemy.Table('event', meta, autoload=True)
-#    cluster_policies = sqlalchemy.Table('cluster_policy', meta, autoload=True)
-#    user_creds = sqlalchemy.Table('user_creds', meta, autoload=True)
-#
-#    stmt = sqlalchemy.select([cluster.c.id,
-#                              cluster.c.profile_id,
-#                              cluster.c.user_creds_id]).\
-#        where(cluster.c.deleted_at < time_line)
-#    deleted_clusters = engine.execute(stmt)
-#
-#    for s in deleted_clusters:
-#        event_del = event.delete().where(event.c.cluster_id == s[0])
-#        engine.execute(event_del)
-#        cluster_del = cluster.delete().where(cluster.c.id == s[0])
-#        engine.execute(cluster_del)
-#        cluster_policies_del = cluster_policies.delete().\
-#            where(.c.id == s[1])
-#        engine.execute(raw_template_del)
-#        user_creds_del = user_creds.delete().where(user_creds.c.id == s[2])
-#        engine.execute(user_creds_del)
-
-
 # Actions
 def action_create(context, values):
     action = models.Action()
@@ -1556,3 +1510,70 @@ def db_sync(engine, version=None):
 def db_version(engine):
     """Display the current database version."""
     return migration.db_version(engine)
+
+
+def purge_deleted(age, unit='days'):
+    """Delete objects that were marked as soft-deleted.
+
+    :param age: An integer to be interpreted as a length of time period.
+    :param unit: A string for the granularity of the 'age' param.
+    """
+    try:
+        age = int(age)
+    except ValueError:
+        raise exception.Error(_("age should be an integer"))
+
+    if age < 0:
+        raise exception.Error(_("age should be a positive integer"))
+
+    if unit not in ('days', 'hours', 'minutes', 'seconds'):
+        raise exception.Error(
+            _("unit must be one of days, hours, minutes, or seconds"))
+
+    if unit == 'days':
+        age = age * 86400
+    elif unit == 'hours':
+        age = age * 3600
+    elif unit == 'minutes':
+        age = age * 60
+
+    timeline = timeutils.utcnow() - datetime.timedelta(seconds=age)
+    engine = get_engine()
+    meta = sqlalchemy.MetaData()
+    meta.bind = engine
+
+    profile = sqlalchemy.Table('profile', meta, autoload=True)
+    policy = sqlalchemy.Table('policy', meta, autoload=True)
+    cluster = sqlalchemy.Table('cluster', meta, autoload=True)
+    node = sqlalchemy.Table('node', meta, autoload=True)
+    action = sqlalchemy.Table('action', meta, autoload=True)
+    receiver = sqlalchemy.Table('receiver', meta, autoload=True)
+    event = sqlalchemy.Table('event', meta, autoload=True)
+
+    # delete policies
+    policy_del = policy.delete().where(policy.c.deleted_time < timeline)
+    engine.execute(policy_del)
+
+    # delete events
+    event_del = event.delete().where(event.c.timestamp < timeline)
+    engine.execute(event_del)
+
+    # delete actions
+    action_del = action.delete().where(action.c.deleted_time < timeline)
+    engine.execute(action_del)
+
+    # delete receivers
+    receiver_del = receiver.delete().where(receiver.c.deleted_time < timeline)
+    engine.execute(receiver_del)
+
+    # delete nodes
+    node_del = node.delete().where(node.c.deleted_time < timeline)
+    engine.execute(node_del)
+
+    # delete clusters
+    cluster_del = cluster.delete().where(cluster.c.deleted_time < timeline)
+    engine.execute(cluster_del)
+
+    # delete profiles
+    profile_del = profile.delete().where(profile.c.deleted_time < timeline)
+    engine.execute(profile_del)

senlin/db/utils.py

@@ -41,5 +41,5 @@ class LazyPluggable(object):
 IMPL = LazyPluggable('backend', sqlalchemy='senlin.db.sqlalchemy.api')
 
 
-def purge_deleted(age, granularity='days'):
-    IMPL.purge_deleted(age, granularity)
+def purge_deleted(age, unit='days'):
+    IMPL.purge_deleted(age, unit)

senlin/tests/unit/db/test_purge_deleted.py

@@ -0,0 +1,74 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import datetime
+
+from oslo_utils import timeutils
+
+from senlin.db.sqlalchemy import api as db_api
+from senlin.tests.unit.common import base
+from senlin.tests.unit.common import utils
+from senlin.tests.unit.db import shared
+
+
+class DBAPIPurgeTest(base.SenlinTestCase):
+
+    def setUp(self):
+        super(DBAPIPurgeTest, self).setUp()
+        self.ctx = utils.dummy_context()
+
+    def create_objects(self, deleted_time=None):
+        profile = shared.create_profile(self.ctx,
+                                        deleted_time=deleted_time)
+        cluster = shared.create_cluster(self.ctx, profile,
+                                        deleted_time=deleted_time)
+        shared.create_node(self.ctx, cluster, profile,
+                           deleted_time=deleted_time)
+
+    def verify_left(self, count):
+        profiles = db_api.profile_get_all(self.ctx, show_deleted=True)
+        self.assertEqual(count, len(profiles))
+        clusters = db_api.cluster_get_all(self.ctx, show_deleted=True)
+        self.assertEqual(count, len(clusters))
+        nodes = db_api.node_get_all(self.ctx, show_deleted=True)
+        self.assertEqual(count, len(nodes))
+
+    def test_purge_deleted(self):
+        now = timeutils.utcnow()
+        delta = datetime.timedelta(seconds=3600 * 9)
+        for i in range(1, 7):
+            deleted_at = now - delta * i
+            self.create_objects(deleted_at)
+
+        self.verify_left(6)
+
+        db_api.purge_deleted(age=2, unit='days')
+        self.verify_left(5)
+
+        db_api.purge_deleted(age=44, unit='hours')
+        self.verify_left(4)
+
+        # 35 hours
+        db_api.purge_deleted(age=35, unit='hours')
+        self.verify_left(3)
+
+        # 25 hours
+        db_api.purge_deleted(age=1500, unit='minutes')
+        self.verify_left(2)
+
+        # 15 hours
+        db_api.purge_deleted(age=900, unit='minutes')
+        self.verify_left(1)
+
+        # 5 hours
+        db_api.purge_deleted(age=18000, unit='seconds')
+        self.verify_left(0)