Add user messages periodic cleanup task

Adds a task to manila-scheduler service which periodically (once per day
by default) cleans up expired user messages. In user messages spec I
preferred doing this cleanup by a "manila-manage" command invoked by
user. But because it would mean another task management for admin and
because cinder uses periodic task too, I diverged from the spec.

Change-Id: If0ba2383d4d7eaca4dee8a0d39f39c5481bfbfdd
Partially-Implements: bp user-messages

manila/cmd/scheduler.py

@@ -46,7 +46,8 @@ def main():
     log.setup(CONF, "manila")
     utils.monkey_patch()
     gmr.TextGuruMeditation.setup_autorun(version, conf=CONF)
-    server = service.Service.create(binary='manila-scheduler')
+    server = service.Service.create(binary='manila-scheduler',
+                                    coordination=True)
     service.serve(server)
     service.wait()
 

manila/db/api.py

@@ -1283,3 +1283,8 @@ def message_create(context, values):
 def message_destroy(context, message_id):
     """Deletes message with the specified ID."""
     return IMPL.message_destroy(context, message_id)
+
+
+def cleanup_expired_messages(context):
+    """Soft delete expired messages"""
+    return IMPL.cleanup_expired_messages(context)

manila/db/sqlalchemy/api.py

@@ -4632,3 +4632,12 @@ def message_destroy(context, message):
     with session.begin():
         (model_query(context, models.Message, session=session).
             filter_by(id=message.get('id')).soft_delete())
+
+
+@require_admin_context
+def cleanup_expired_messages(context):
+    session = get_session()
+    now = timeutils.utcnow()
+    with session.begin():
+        return session.query(models.Message).filter(
+            models.Message.expires_at < now).delete()

manila/message/api.py

@@ -27,6 +27,9 @@ from manila.message import message_levels
 messages_opts = [
     cfg.IntOpt('message_ttl', default=2592000,
                help='Message minimum life in seconds.'),
+    cfg.IntOpt('message_reap_interval', default=86400,
+               help='Interval between periodic task runs to clean expired '
+                    'messages in seconds.'),
 ]
 
 CONF = cfg.CONF
@@ -83,3 +86,8 @@ class API(base.Base):
     def delete(self, context, id):
         """Delete message with the specified message id."""
         return self.db.message_destroy(context, id)
+
+    def cleanup_expired_messages(self, context):
+        ctx = context.elevated()
+        count = self.db.cleanup_expired_messages(ctx)
+        LOG.info("Deleted %s expired messages.", count)

manila/scheduler/manager.py

@@ -27,6 +27,7 @@ from oslo_utils import importutils
 
 from manila.common import constants
 from manila import context
+from manila import coordination
 from manila import db
 from manila import exception
 from manila import manager
@@ -285,3 +286,9 @@ class SchedulerManager(manager.Manager):
             with excutils.save_and_reraise_exception():
                 self._set_share_replica_error_state(
                     context, 'create_share_replica', exc, request_spec)
+
+    @periodic_task.periodic_task(spacing=CONF.message_reap_interval,
+                                 run_immediately=True)
+    @coordination.synchronized('locked-clean-expired-messages')
+    def _clean_expired_messages(self, context):
+        self.message_api.cleanup_expired_messages(context)

manila/tests/cmd/test_scheduler.py

@@ -41,7 +41,7 @@ class ManilaCmdSchedulerTestCase(test.TestCase):
         manila_scheduler.log.register_options.assert_called_once_with(CONF)
         manila_scheduler.utils.monkey_patch.assert_called_once_with()
         manila_scheduler.service.Service.create.assert_called_once_with(
-            binary='manila-scheduler')
+            binary='manila-scheduler', coordination=True)
         manila_scheduler.service.wait.assert_called_once_with()
         manila_scheduler.service.serve.assert_called_once_with(
             manila_scheduler.service.Service.create.return_value)

manila/tests/db/sqlalchemy/test_api.py

@@ -2824,3 +2824,23 @@ class MessagesDatabaseAPITestCase(test.TestCase):
         result = db_api.message_get_all(self.ctxt, sort_key='action_id')
         result_ids = [r.id for r in result]
         self.assertEqual(result_ids, ids)
+
+    def test_cleanup_expired_messages(self):
+        adm_context = self.ctxt.elevated()
+
+        now = timeutils.utcnow()
+        db_utils.create_message(project_id=self.project_id,
+                                action_id='001',
+                                expires_at=now)
+        db_utils.create_message(project_id=self.project_id,
+                                action_id='001',
+                                expires_at=now - datetime.timedelta(days=1))
+        db_utils.create_message(project_id=self.project_id,
+                                action_id='001',
+                                expires_at=now + datetime.timedelta(days=1))
+
+        with mock.patch.object(timeutils, 'utcnow') as mock_time_now:
+            mock_time_now.return_value = now
+            db_api.cleanup_expired_messages(adm_context)
+            messages = db_api.message_get_all(adm_context)
+            self.assertEqual(2, len(messages))

manila/tests/message/test_api.py

@@ -90,3 +90,11 @@ class MessageApiTest(test.TestCase):
 
         self.message_api.db.message_destroy.assert_called_once_with(
             self.ctxt, 'fake_id')
+
+    def test_cleanup_expired_messages(self):
+        admin_context = mock.Mock()
+        self.mock_object(self.ctxt, 'elevated',
+                         mock.Mock(return_value=admin_context))
+        self.message_api.cleanup_expired_messages(self.ctxt)
+        self.message_api.db.cleanup_expired_messages.assert_called_once_with(
+            admin_context)

manila/tests/scheduler/test_manager.py

@@ -200,13 +200,22 @@ class SchedulerManagerTestCase(test.TestCase):
 
         mock_expire.assert_called_once_with(self.context)
 
+    @mock.patch('manila.message.api.API.cleanup_expired_messages')
+    def test__clean_expired_messages(self, mock_expire):
+        self.manager._clean_expired_messages(self.context)
+
+        mock_expire.assert_called_once_with(self.context)
+
     def test_periodic_tasks(self):
-        self.mock_periodic_task.assert_called_once_with(
-            spacing=600, run_immediately=True)
-        self.assertEqual(1, len(self.periodic_tasks))
+        self.assertEqual(2, self.mock_periodic_task.call_count)
+
+        self.assertEqual(2, len(self.periodic_tasks))
         self.assertEqual(
             self.periodic_tasks[0].__name__,
             self.manager._expire_reservations.__name__)
+        self.assertEqual(
+            self.periodic_tasks[1].__name__,
+            self.manager._clean_expired_messages.__name__)
 
     def test_get_pools(self):
         """Ensure get_pools exists and calls base_scheduler.get_pools."""

releasenotes/notes/clean-expired-messages-6161094d0c108aa7.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Added a periodic task which cleans up expired user messages. Cleanup
+    interval can be set by message_reap_interval config option.