Add a soft delete functionality for tasks.
Currently there is no mechanism for deleting tasks on regular basis. This patch adds a new function that is called in the tasks_get_all function, so that everytime tasks lists are called, the function checks if tasks in the database have surpassed the expired_at value. If that is the case, then it sets the deleted value as 1 for all the expired tasks. Co-Authored-By: Mike Fedosin <mfedosin@mirantis.com> Implements https://review.openstack.org/#/c/324648/ Change-Id: I0bde982de948901f6bfbfab9e57cf84891c22052
This commit is contained in:
parent
13a17a814e
commit
38563b0555
|
@ -959,6 +959,20 @@ def task_delete(context, task_id):
|
||||||
raise exception.TaskNotFound(task_id=task_id)
|
raise exception.TaskNotFound(task_id=task_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _task_soft_delete(context):
|
||||||
|
"""Scrub task entities which are expired """
|
||||||
|
global DATA
|
||||||
|
now = timeutils.utcnow()
|
||||||
|
tasks = DATA['tasks'].values()
|
||||||
|
|
||||||
|
for task in tasks:
|
||||||
|
if(task['owner'] == context.owner and task['deleted'] == False
|
||||||
|
and task['expires_at'] <= now):
|
||||||
|
|
||||||
|
task['deleted'] = True
|
||||||
|
task['deleted_at'] = timeutils.utcnow()
|
||||||
|
|
||||||
|
|
||||||
@log_call
|
@log_call
|
||||||
def task_get_all(context, filters=None, marker=None, limit=None,
|
def task_get_all(context, filters=None, marker=None, limit=None,
|
||||||
sort_key='created_at', sort_dir='desc'):
|
sort_key='created_at', sort_dir='desc'):
|
||||||
|
@ -972,6 +986,7 @@ def task_get_all(context, filters=None, marker=None, limit=None,
|
||||||
:param sort_dir: direction in which results should be sorted (asc, desc)
|
:param sort_dir: direction in which results should be sorted (asc, desc)
|
||||||
:returns: tasks set
|
:returns: tasks set
|
||||||
"""
|
"""
|
||||||
|
_task_soft_delete(context)
|
||||||
filters = filters or {}
|
filters = filters or {}
|
||||||
tasks = DATA['tasks'].values()
|
tasks = DATA['tasks'].values()
|
||||||
tasks = _filter_tasks(tasks, filters, context)
|
tasks = _filter_tasks(tasks, filters, context)
|
||||||
|
|
|
@ -1440,6 +1440,21 @@ def task_delete(context, task_id, session=None):
|
||||||
return _task_format(task_ref, task_ref.info)
|
return _task_format(task_ref, task_ref.info)
|
||||||
|
|
||||||
|
|
||||||
|
def _task_soft_delete(context, session=None):
|
||||||
|
"""Scrub task entities which are expired """
|
||||||
|
expires_at = models.Task.expires_at
|
||||||
|
session = session or get_session()
|
||||||
|
query = session.query(models.Task)
|
||||||
|
|
||||||
|
query = (query.filter(models.Task.owner == context.owner)
|
||||||
|
.filter_by(deleted=0)
|
||||||
|
.filter(expires_at <= timeutils.utcnow()))
|
||||||
|
values = {'deleted': 1, 'deleted_at': timeutils.utcnow()}
|
||||||
|
|
||||||
|
with session.begin():
|
||||||
|
query.update(values)
|
||||||
|
|
||||||
|
|
||||||
def task_get_all(context, filters=None, marker=None, limit=None,
|
def task_get_all(context, filters=None, marker=None, limit=None,
|
||||||
sort_key='created_at', sort_dir='desc', admin_as_user=False):
|
sort_key='created_at', sort_dir='desc', admin_as_user=False):
|
||||||
"""
|
"""
|
||||||
|
@ -1463,6 +1478,8 @@ def task_get_all(context, filters=None, marker=None, limit=None,
|
||||||
if not (context.is_admin or admin_as_user) and context.owner is not None:
|
if not (context.is_admin or admin_as_user) and context.owner is not None:
|
||||||
query = query.filter(models.Task.owner == context.owner)
|
query = query.filter(models.Task.owner == context.owner)
|
||||||
|
|
||||||
|
_task_soft_delete(context, session=session)
|
||||||
|
|
||||||
showing_deleted = False
|
showing_deleted = False
|
||||||
|
|
||||||
if 'deleted' in filters:
|
if 'deleted' in filters:
|
||||||
|
|
|
@ -1527,9 +1527,10 @@ class TaskTests(test_utils.BaseTestCase):
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TaskTests, self).setUp()
|
super(TaskTests, self).setUp()
|
||||||
self.owner_id = str(uuid.uuid4())
|
self.admin_id = 'admin'
|
||||||
self.adm_context = context.RequestContext(is_admin=True,
|
self.owner_id = 'user'
|
||||||
auth_token='user:user:admin')
|
self.adm_context = context.RequestContext(
|
||||||
|
is_admin=True, auth_token='user:admin:admin', tenant=self.admin_id)
|
||||||
self.context = context.RequestContext(
|
self.context = context.RequestContext(
|
||||||
is_admin=False, auth_token='user:user:user', user=self.owner_id)
|
is_admin=False, auth_token='user:user:user', user=self.owner_id)
|
||||||
self.db_api = db_tests.get_db(self.config)
|
self.db_api = db_tests.get_db(self.config)
|
||||||
|
@ -1623,13 +1624,15 @@ class TaskTests(test_utils.BaseTestCase):
|
||||||
self.assertEqual(0, len(tasks))
|
self.assertEqual(0, len(tasks))
|
||||||
|
|
||||||
def test_task_get_all_owned(self):
|
def test_task_get_all_owned(self):
|
||||||
|
then = timeutils.utcnow() + datetime.timedelta(days=365)
|
||||||
TENANT1 = str(uuid.uuid4())
|
TENANT1 = str(uuid.uuid4())
|
||||||
ctxt1 = context.RequestContext(is_admin=False,
|
ctxt1 = context.RequestContext(is_admin=False,
|
||||||
tenant=TENANT1,
|
tenant=TENANT1,
|
||||||
auth_token='user:%s:user' % TENANT1)
|
auth_token='user:%s:user' % TENANT1)
|
||||||
|
|
||||||
task_values = {'type': 'import', 'status': 'pending',
|
task_values = {'type': 'import', 'status': 'pending',
|
||||||
'input': '{"loc": "fake"}', 'owner': TENANT1}
|
'input': '{"loc": "fake"}', 'owner': TENANT1,
|
||||||
|
'expires_at': then}
|
||||||
self.db_api.task_create(ctxt1, task_values)
|
self.db_api.task_create(ctxt1, task_values)
|
||||||
|
|
||||||
TENANT2 = str(uuid.uuid4())
|
TENANT2 = str(uuid.uuid4())
|
||||||
|
@ -1638,7 +1641,8 @@ class TaskTests(test_utils.BaseTestCase):
|
||||||
auth_token='user:%s:user' % TENANT2)
|
auth_token='user:%s:user' % TENANT2)
|
||||||
|
|
||||||
task_values = {'type': 'export', 'status': 'pending',
|
task_values = {'type': 'export', 'status': 'pending',
|
||||||
'input': '{"loc": "fake"}', 'owner': TENANT2}
|
'input': '{"loc": "fake"}', 'owner': TENANT2,
|
||||||
|
'expires_at': then}
|
||||||
self.db_api.task_create(ctxt2, task_values)
|
self.db_api.task_create(ctxt2, task_values)
|
||||||
|
|
||||||
tasks = self.db_api.task_get_all(ctxt1)
|
tasks = self.db_api.task_get_all(ctxt1)
|
||||||
|
@ -1680,6 +1684,7 @@ class TaskTests(test_utils.BaseTestCase):
|
||||||
|
|
||||||
def test_task_get_all(self):
|
def test_task_get_all(self):
|
||||||
now = timeutils.utcnow()
|
now = timeutils.utcnow()
|
||||||
|
then = now + datetime.timedelta(days=365)
|
||||||
image_id = str(uuid.uuid4())
|
image_id = str(uuid.uuid4())
|
||||||
fixture1 = {
|
fixture1 = {
|
||||||
'owner': self.context.owner,
|
'owner': self.context.owner,
|
||||||
|
@ -1688,7 +1693,7 @@ class TaskTests(test_utils.BaseTestCase):
|
||||||
'input': '{"loc": "fake_1"}',
|
'input': '{"loc": "fake_1"}',
|
||||||
'result': "{'image_id': %s}" % image_id,
|
'result': "{'image_id': %s}" % image_id,
|
||||||
'message': 'blah_1',
|
'message': 'blah_1',
|
||||||
'expires_at': now,
|
'expires_at': then,
|
||||||
'created_at': now,
|
'created_at': now,
|
||||||
'updated_at': now
|
'updated_at': now
|
||||||
}
|
}
|
||||||
|
@ -1700,7 +1705,7 @@ class TaskTests(test_utils.BaseTestCase):
|
||||||
'input': '{"loc": "fake_2"}',
|
'input': '{"loc": "fake_2"}',
|
||||||
'result': "{'image_id': %s}" % image_id,
|
'result': "{'image_id': %s}" % image_id,
|
||||||
'message': 'blah_2',
|
'message': 'blah_2',
|
||||||
'expires_at': now,
|
'expires_at': then,
|
||||||
'created_at': now,
|
'created_at': now,
|
||||||
'updated_at': now
|
'updated_at': now
|
||||||
}
|
}
|
||||||
|
@ -1734,6 +1739,39 @@ class TaskTests(test_utils.BaseTestCase):
|
||||||
for key in task_details_keys:
|
for key in task_details_keys:
|
||||||
self.assertNotIn(key, task)
|
self.assertNotIn(key, task)
|
||||||
|
|
||||||
|
def test_task_soft_delete(self):
|
||||||
|
now = timeutils.utcnow()
|
||||||
|
then = now + datetime.timedelta(days=365)
|
||||||
|
|
||||||
|
fixture1 = build_task_fixture(id='1', expires_at=now,
|
||||||
|
owner=self.adm_context.owner)
|
||||||
|
fixture2 = build_task_fixture(id='2', expires_at=now,
|
||||||
|
owner=self.adm_context.owner)
|
||||||
|
fixture3 = build_task_fixture(id='3', expires_at=then,
|
||||||
|
owner=self.adm_context.owner)
|
||||||
|
fixture4 = build_task_fixture(id='4', expires_at=then,
|
||||||
|
owner=self.adm_context.owner)
|
||||||
|
|
||||||
|
task1 = self.db_api.task_create(self.adm_context, fixture1)
|
||||||
|
task2 = self.db_api.task_create(self.adm_context, fixture2)
|
||||||
|
task3 = self.db_api.task_create(self.adm_context, fixture3)
|
||||||
|
task4 = self.db_api.task_create(self.adm_context, fixture4)
|
||||||
|
|
||||||
|
self.assertIsNotNone(task1)
|
||||||
|
self.assertIsNotNone(task2)
|
||||||
|
self.assertIsNotNone(task3)
|
||||||
|
self.assertIsNotNone(task4)
|
||||||
|
|
||||||
|
tasks = self.db_api.task_get_all(
|
||||||
|
self.adm_context, sort_key='id', sort_dir='asc')
|
||||||
|
|
||||||
|
self.assertEqual(4, len(tasks))
|
||||||
|
|
||||||
|
self.assertTrue(tasks[0]['deleted'])
|
||||||
|
self.assertTrue(tasks[1]['deleted'])
|
||||||
|
self.assertFalse(tasks[2]['deleted'])
|
||||||
|
self.assertFalse(tasks[3]['deleted'])
|
||||||
|
|
||||||
def test_task_create(self):
|
def test_task_create(self):
|
||||||
task_id = str(uuid.uuid4())
|
task_id = str(uuid.uuid4())
|
||||||
self.context.tenant = self.context.owner
|
self.context.tenant = self.context.owner
|
||||||
|
|
|
@ -14,12 +14,14 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
import datetime
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_db import exception as db_exc
|
from oslo_db import exception as db_exc
|
||||||
from oslo_utils import encodeutils
|
from oslo_utils import encodeutils
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
|
||||||
from glance.common import crypt
|
from glance.common import crypt
|
||||||
from glance.common import exception
|
from glance.common import exception
|
||||||
|
@ -116,6 +118,7 @@ def _db_task_fixture(task_id, type, status, **kwargs):
|
||||||
'owner': None,
|
'owner': None,
|
||||||
'message': None,
|
'message': None,
|
||||||
'deleted': False,
|
'deleted': False,
|
||||||
|
'expires_at': timeutils.utcnow() + datetime.timedelta(days=365)
|
||||||
}
|
}
|
||||||
obj.update(kwargs)
|
obj.update(kwargs)
|
||||||
return obj
|
return obj
|
||||||
|
|
|
@ -56,7 +56,7 @@ def _db_fixture(task_id, **kwargs):
|
||||||
'result': None,
|
'result': None,
|
||||||
'owner': None,
|
'owner': None,
|
||||||
'message': None,
|
'message': None,
|
||||||
'expires_at': None,
|
'expires_at': default_datetime + datetime.timedelta(days=365),
|
||||||
'created_at': default_datetime,
|
'created_at': default_datetime,
|
||||||
'updated_at': default_datetime,
|
'updated_at': default_datetime,
|
||||||
'deleted_at': None,
|
'deleted_at': None,
|
||||||
|
|
|
@ -0,0 +1,25 @@
|
||||||
|
---
|
||||||
|
prelude: >
|
||||||
|
Adds a new function that is called in the
|
||||||
|
tasks_get_all function, so that everytime tasks
|
||||||
|
lists are called, the function checks if tasks in
|
||||||
|
the database have surpassed the expired_at value;
|
||||||
|
if that is the case, then it marks the deleted value
|
||||||
|
as 1 for all the expired tasks.
|
||||||
|
|
||||||
|
other:
|
||||||
|
- Tasks are soft deleted, in Glance, a resource can
|
||||||
|
be soft deleted in the Database Table, these resources
|
||||||
|
still exist in the database. The same thing happens
|
||||||
|
with tasks; they are marked as deleted using the
|
||||||
|
delete flag in the Tasks table which are not queried
|
||||||
|
on the regular list or show call. The tasks are not
|
||||||
|
instantly deleted because there may be information
|
||||||
|
contained in the task resource that may not be
|
||||||
|
available elsewhere(For example, a successful
|
||||||
|
import task will eventually result in the creation
|
||||||
|
of an image in Glance, and it would be useful to
|
||||||
|
know the UUID of this image. Similarly, if the
|
||||||
|
import task fails, end user should be given time
|
||||||
|
to read the task resource to analyze the error
|
||||||
|
message.)
|
Loading…
Reference in New Issue