openstack/deb-glance
Code Issues Proposed changes

Merge "Add db purge command"

Browse Source
This commit is contained in:
Jenkins 2015-12-19 02:23:19 +00:00 committed by Gerrit Code Review
commit 585206940a
5 changed files with 165 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
glance/cmd/manage.py
View File

@ -29,6 +29,7 @@ from __future__ import print_function
import os
import sys
import time
# If ../glance/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
@ -46,6 +47,7 @@ import six
from glance.common import config
from glance.common import exception
from glance import context
from glance.db import migration as db_migration
from glance.db.sqlalchemy import api as db_api
from glance.db.sqlalchemy import metadata
@ -145,6 +147,26 @@ class DbCommands(object):
metadata.db_export_metadefs(db_api.get_engine(),
path)
@args('--age_in_days', type=int,
help='Purge deleted rows older than age in days')
@args('--max_rows', type=int,
help='Limit number of records to delete')
def purge(self, age_in_days=30, max_rows=100):
"""Purge deleted rows older than a given age from glance tables."""
age_in_days = int(age_in_days)
max_rows = int(max_rows)
if age_in_days <= 0:
print(_("Must supply a positive, non-zero value for age."))
exit(1)
if age_in_days >= (int(time.time()) / 86400):
print(_("Maximal age is count of days since epoch."))
exit(1)
if max_rows < 1:
print(_("Minimal rows limit is 1."))
exit(1)
ctx = context.get_admin_context(show_deleted=True)
db_api.purge_deleted_rows(ctx, age_in_days, max_rows)
class DbLegacyCommands(object):
"""Class for managing the db using legacy commands"""

9
glance/context.py
View File

@ -58,3 +58,12 @@ class RequestContext(context.RequestContext):
def can_see_deleted(self):
"""Admins can see deleted by default"""
return self.show_deleted or self.is_admin
def get_admin_context(show_deleted=False):
"""Create an administrator context."""
return RequestContext(auth_token=None,
tenant=None,
is_admin=True,
show_deleted=show_deleted,
overwrite=False)

70
glance/db/sqlalchemy/api.py
View File

@ -21,6 +21,7 @@
"""Defines interface for DB access."""
import datetime
import threading
from oslo_config import cfg
@ -33,6 +34,7 @@ import six
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import sqlalchemy
from sqlalchemy import MetaData, Table, select
import sqlalchemy.orm as sa_orm
import sqlalchemy.sql as sa_sql
@ -50,7 +52,7 @@ from glance.db.sqlalchemy.metadef_api import object as metadef_object_api
from glance.db.sqlalchemy.metadef_api import property as metadef_property_api
from glance.db.sqlalchemy.metadef_api import tag as metadef_tag_api
from glance.db.sqlalchemy import models
from glance.i18n import _, _LW
from glance.i18n import _, _LW, _LE, _LI
BASE = models.BASE
sa_logger = None
@ -1226,6 +1228,69 @@ def image_tag_get_all(context, image_id, session=None):
return [tag[0] for tag in tags]
def purge_deleted_rows(context, age_in_days, max_rows, session=None):
"""Purges soft deleted rows
Deletes rows of table images, table tasks and all dependent tables
according to given age for relevant models.
"""
try:
age_in_days = int(age_in_days)
except ValueError:
LOG.exception(_LE('Invalid value for age, %(age)d'),
{'age': age_in_days})
raise exception.InvalidParameterValue(value=age_in_days,
param='age_in_days')
try:
max_rows = int(max_rows)
except ValueError:
LOG.exception(_LE('Invalid value for max_rows, %(max_rows)d'),
{'max_rows': max_rows})
raise exception.InvalidParameterValue(value=max_rows,
param='max_rows')
session = session or get_session()
metadata = MetaData(get_engine())
deleted_age = timeutils.utcnow() - datetime.timedelta(days=age_in_days)
tables = []
for model_class in models.__dict__.values():
if not hasattr(model_class, '__tablename__'):
continue
if hasattr(model_class, 'deleted'):
tables.append(model_class.__tablename__)
# get rid of FX constraints
for tbl in ('images', 'tasks'):
try:
tables.remove(tbl)
except ValueError:
LOG.warning(_LW('Expected table %(tbl)s was not found in DB.'),
**locals())
else:
tables.append(tbl)
for tbl in tables:
tab = Table(tbl, metadata, autoload=True)
LOG.info(
_LI('Purging deleted rows older than %(age_in_days)d day(s) '
'from table %(tbl)s'),
**locals()
)
with session.begin():
result = session.execute(
tab.delete().where(
tab.columns.id.in_(
select([tab.columns.id]).where(
tab.columns.deleted_at < deleted_age
).limit(max_rows)
)
)
)
rows = result.rowcount
LOG.info(_LI('Deleted %(rows)d row(s) from table %(tbl)s'),
**locals())
def user_get_storage_usage(context, owner_id, image_id=None, session=None):
_check_image_id(image_id)
session = session or get_session()
@ -1452,7 +1517,8 @@ def _task_get(context, task_id, session=None, force_show_deleted=False):
def _task_update(context, task_ref, values, session=None):
"""Apply supplied dictionary of values to a task object."""
values["deleted"] = False
if 'deleted' not in values:
values["deleted"] = False
task_ref.update(values)
task_ref.save(session=session)
return task_ref

58
glance/tests/functional/db/base.py
View File

@ -75,7 +75,7 @@ def build_task_fixture(**kwargs):
'message': None,
'expires_at': None,
'created_at': default_datetime,
'updated_at': default_datetime
'updated_at': default_datetime,
}
task.update(kwargs)
return task
@ -1795,6 +1795,62 @@ class TaskTests(test_utils.BaseTestCase):
self.assertIsNotNone(del_task['deleted_at'])
class DBPurgeTests(test_utils.BaseTestCase):
def setUp(self):
super(DBPurgeTests, self).setUp()
self.adm_context = context.get_admin_context(show_deleted=True)
self.db_api = db_tests.get_db(self.config)
db_tests.reset_db(self.db_api)
self.image_fixtures, self.task_fixtures = self.build_fixtures()
self.create_tasks(self.task_fixtures)
self.create_images(self.image_fixtures)
def build_fixtures(self):
dt1 = timeutils.utcnow() - datetime.timedelta(days=5)
dt2 = dt1 + datetime.timedelta(days=1)
dt3 = dt2 + datetime.timedelta(days=1)
fixtures = [
{
'created_at': dt1,
'updated_at': dt1,
'deleted_at': dt3,
'deleted': True,
},
{
'created_at': dt1,
'updated_at': dt2,
'deleted_at': timeutils.utcnow(),
'deleted': True,
},
{
'created_at': dt2,
'updated_at': dt2,
'deleted_at': None,
'deleted': False,
},
]
return (
[build_image_fixture(**fixture) for fixture in fixtures],
[build_task_fixture(**fixture) for fixture in fixtures],
)
def create_images(self, images):
for fixture in images:
self.db_api.image_create(self.adm_context, fixture)
def create_tasks(self, tasks):
for fixture in tasks:
self.db_api.task_create(self.adm_context, fixture)
def test_db_purge(self):
self.db_api.purge_deleted_rows(self.adm_context, 1, 5)
images = self.db_api.image_get_all(self.adm_context)
self.assertEqual(len(images), 2)
tasks = self.db_api.task_get_all(self.adm_context)
self.assertEqual(len(tasks), 2)
class TestVisibility(test_utils.BaseTestCase):
def setUp(self):
super(TestVisibility, self).setUp()

9
glance/tests/functional/db/test_sqlalchemy.py
View File

@ -160,6 +160,15 @@ class TestSqlAlchemyQuota(base.DriverQuotaTests,
self.addCleanup(db_tests.reset)
class TestDBPurge(base.DBPurgeTests,
base.FunctionalInitWrapper):
def setUp(self):
db_tests.load(get_db, reset_db)
super(TestDBPurge, self).setUp()
self.addCleanup(db_tests.reset)
class TestArtifacts(base_artifacts.ArtifactsTestDriver,
base_artifacts.ArtifactTests):
def setUp(self):