openstack/deb-ceilometer
Code Issues Proposed changes

Add the function of deleting alarm history

Browse Source 
Currently the ceilometer-expirer doesn't delete expired AlarmChanges.
Remained AlarmChanges would be cause of wasted disk-space and slow response.
This patch aims to delete AlarmChanges.

This patch adds a new config option:
[database]
alarm_history_time_to_live = -1

Implements: blueprint delete-alarmhistory
DocImpact
Change-Id: Ib6bda6df720cf8514b621bfe13dd3acba941949f
This commit is contained in:
Maho Koshiya 2014-08-15 17:58:03 +09:00 committed by ZhiQiang Fan
parent 521b8816ae
commit 9e1513c58a
9 changed files with 152 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ceilometer/alarm/storage/base.py
View File

@ -153,3 +153,15 @@ class Connection(object):
This is needed to evaluate the performance of each driver.
"""
return cls.STORAGE_CAPABILITIES
@staticmethod
def clear_expired_alarm_history_data(alarm_history_ttl):
"""Clear expired alarm history data from the backend storage system.
Clearing occurs according to the time-to-live.
:param alarm_history_ttl: Number of seconds to keep alarm history
records for.
"""
raise ceilometer.NotImplementedError('Clearing alarm history '
'not implemented')

12
ceilometer/alarm/storage/impl_log.py
View File

@ -16,6 +16,7 @@
"""
from ceilometer.alarm.storage import base
from ceilometer.i18n import _LI
from ceilometer.openstack.common import log
LOG = log.getLogger(__name__)
@ -46,3 +47,14 @@ class Connection(base.Connection):
def delete_alarm(self, alarm_id):
"""Delete an alarm."""
def clear_expired_alarm_history_data(self, alarm_history_ttl):
"""Clear expired alarm history data from the backend storage system.
Clearing occurs according to the time-to-live.
:param alarm_history_ttl: Number of seconds to keep alarm history
records for.
"""
LOG.info(_LI('Dropping alarm history data with TTL %d'),
alarm_history_ttl)

20
ceilometer/alarm/storage/impl_mongodb.py
View File

@ -20,13 +20,18 @@
# under the License.
"""MongoDB storage backend"""
from oslo_config import cfg
import pymongo
from ceilometer.alarm.storage import pymongo_base
from ceilometer.openstack.common import log
from ceilometer import storage
from ceilometer.storage import impl_mongodb
from ceilometer.storage.mongo import utils as pymongo_utils
cfg.CONF.import_opt('alarm_history_time_to_live', 'ceilometer.alarm.storage',
group="database")
LOG = log.getLogger(__name__)
@ -64,8 +69,23 @@ class Connection(pymongo_base.Connection):
self.db.conn.create_collection('alarm')
if 'alarm_history' not in self.db.conn.collection_names():
self.db.conn.create_collection('alarm_history')
# Establish indexes
ttl = cfg.CONF.database.alarm_history_time_to_live
impl_mongodb.Connection.update_ttl(
ttl, 'alarm_history_ttl', 'timestamp', self.db.alarm_history)
def clear(self):
self.conn.drop_database(self.db.name)
# Connection will be reopened automatically if needed
self.conn.close()
def clear_expired_alarm_history_data(self, alarm_history_ttl):
"""Clear expired alarm history data from the backend storage system.
Clearing occurs according to the time-to-live.
:param alarm_history_ttl: Number of seconds to keep alarm history
records for.
"""
LOG.debug("Clearing expired alarm history data is based on native "
"MongoDB time to live feature and going in background.")

21
ceilometer/alarm/storage/impl_sqlalchemy.py
View File

@ -13,15 +13,18 @@
"""SQLAlchemy storage backend."""
from __future__ import absolute_import
import datetime
import os
from oslo.db.sqlalchemy import session as db_session
from oslo_config import cfg
from oslo_utils import timeutils
from sqlalchemy import desc
import ceilometer
from ceilometer.alarm.storage import base
from ceilometer.alarm.storage import models as alarm_api_models
from ceilometer.i18n import _LI
from ceilometer.openstack.common import log
from ceilometer.storage.sqlalchemy import models
from ceilometer.storage.sqlalchemy import utils as sql_utils
@ -323,3 +326,21 @@ class Connection(base.Connection):
event_id=alarm_change['event_id'])
alarm_change_row.update(alarm_change)
session.add(alarm_change_row)
def clear_expired_alarm_history_data(self, alarm_history_ttl):
"""Clear expired alarm history data from the backend storage system.
Clearing occurs according to the time-to-live.
:param alarm_history_ttl: Number of seconds to keep alarm history
records for.
"""
session = self._engine_facade.get_session()
with session.begin():
valid_start = (timeutils.utcnow() -
datetime.timedelta(seconds=alarm_history_ttl))
deleted_rows = (session.query(models.AlarmChange)
.filter(models.AlarmChange.timestamp < valid_start)
.delete())
LOG.info(_LI("%d alarm histories are removed from database"),
deleted_rows)

22
ceilometer/cmd/storage.py
View File

@ -18,10 +18,11 @@ import logging
from oslo_config import cfg
from ceilometer.i18n import _
from ceilometer.i18n import _, _LI
from ceilometer import service
from ceilometer import storage
LOG = logging.getLogger(__name__)
@ -37,12 +38,12 @@ def expirer():
if cfg.CONF.database.metering_time_to_live > 0:
LOG.debug(_("Clearing expired metering data"))
storage_conn = storage.get_connection_from_config(cfg.CONF)
storage_conn = storage.get_connection_from_config(cfg.CONF, 'metering')
storage_conn.clear_expired_metering_data(
cfg.CONF.database.metering_time_to_live)
else:
LOG.info(_("Nothing to clean, database metering time to live "
"is disabled"))
LOG.info(_LI("Nothing to clean, database metering time to live "
"is disabled"))
if cfg.CONF.database.event_time_to_live > 0:
LOG.debug(_("Clearing expired event data"))
@ -50,5 +51,14 @@ def expirer():
event_conn.clear_expired_event_data(
cfg.CONF.database.event_time_to_live)
else:
LOG.info(_("Nothing to clean, database event time to live "
"is disabled"))
LOG.info(_LI("Nothing to clean, database event time to live "
"is disabled"))
if cfg.CONF.database.alarm_history_time_to_live > 0:
LOG.debug("Clearing expired alarm history data")
storage_conn = storage.get_connection_from_config(cfg.CONF, 'alarm')
storage_conn.clear_expired_alarm_history_data(
cfg.CONF.database.alarm_history_time_to_live)
else:
LOG.info(_LI("Nothing to clean, database alarm history time to live "
"is disabled"))

4
ceilometer/storage/__init__.py
View File

@ -57,6 +57,10 @@ OPTS = [
default=None,
help='The connection string used to connect to the alarm '
'database. (if unset, connection is used)'),
cfg.IntOpt('alarm_history_time_to_live',
default=-1,
help=("Number of seconds that alarm histories are kept "
"in the database for (<= 0 means forever).")),
cfg.StrOpt('event_connection',
default=None,
help='The connection string used to connect to the event '

8
ceilometer/tests/storage/test_impl_mongodb.py
View File

@ -109,6 +109,10 @@ class IndexTest(tests_db.TestBase,
self._test_ttl_index_absent(self.event_conn, 'event',
'event_time_to_live')
def test_alarm_history_ttl_index_absent(self):
self._test_ttl_index_absent(self.alarm_conn, 'alarm_history',
'alarm_history_time_to_live')
def _test_ttl_index_present(self, conn, coll_name, ttl_opt):
coll = getattr(conn.db, coll_name)
self.CONF.set_override(ttl_opt, 456789, group='database')
@ -130,6 +134,10 @@ class IndexTest(tests_db.TestBase,
self._test_ttl_index_present(self.event_conn, 'event',
'event_time_to_live')
def test_alarm_history_ttl_index_present(self):
self._test_ttl_index_present(self.alarm_conn, 'alarm_history',
'alarm_history_time_to_live')
@tests_db.run_with('mongodb')
class AlarmTestPagination(test_storage_scenarios.AlarmTestBase,

44
ceilometer/tests/storage/test_storage_scenarios.py
View File

@ -3018,6 +3018,50 @@ class AlarmTestPagination(AlarmTestBase,
[i.name for i in page1])
@tests_db.run_with('sqlite', 'mysql', 'pgsql', 'hbase', 'db2')
class AlarmHistoryTest(AlarmTestBase,
tests_db.MixinTestsWithBackendScenarios):
def setUp(self):
super(AlarmTestBase, self).setUp()
self.add_some_alarms()
self.prepare_alarm_history()
def prepare_alarm_history(self):
alarms = list(self.alarm_conn.get_alarms())
for alarm in alarms:
i = alarms.index(alarm)
alarm_change = {
"event_id": "3e11800c-a3ca-4991-b34b-d97efb6047d%s" % i,
"alarm_id": alarm.alarm_id,
"type": alarm_models.AlarmChange.CREATION,
"detail": "detail %s" % alarm.name,
"user_id": alarm.user_id,
"project_id": alarm.project_id,
"on_behalf_of": alarm.project_id,
"timestamp": datetime.datetime(2014, 4, 7, 7, 30 + i)
}
self.alarm_conn.record_alarm_change(alarm_change=alarm_change)
def _clear_alarm_history(self, utcnow, ttl, count):
self.mock_utcnow.return_value = utcnow
self.alarm_conn.clear_expired_alarm_history_data(ttl)
history = list(self.alarm_conn.query_alarm_history())
self.assertEqual(count, len(history))
def test_clear_alarm_history_no_data_to_remove(self):
utcnow = datetime.datetime(2013, 4, 7, 7, 30)
self._clear_alarm_history(utcnow, 1, 3)
def test_clear_some_alarm_history(self):
utcnow = datetime.datetime(2014, 4, 7, 7, 35)
self._clear_alarm_history(utcnow, 3 * 60, 1)
def test_clear_all_alarm_history(self):
utcnow = datetime.datetime(2014, 4, 7, 7, 45)
self._clear_alarm_history(utcnow, 3 * 60, 0)
class ComplexAlarmQueryTest(AlarmTestBase,
tests_db.MixinTestsWithBackendScenarios):

25
ceilometer/tests/test_bin.py
View File

@ -54,15 +54,19 @@ class BinTestCase(base.BaseTestCase):
stderr=subprocess.PIPE)
__, err = subp.communicate()
self.assertEqual(0, subp.poll())
self.assertIn("Nothing to clean", err)
self.assertIn("Nothing to clean, database metering "
"time to live is disabled", err)
self.assertIn("Nothing to clean, database event "
"time to live is disabled", err)
self.assertIn("Nothing to clean, database alarm history "
"time to live is disabled", err)
def _test_run_expirer_ttl_enabled(self, metering_ttl_name):
def _test_run_expirer_ttl_enabled(self, ttl_name, data_name):
content = ("[DEFAULT]\n"
"rpc_backend=fake\n"
"[database]\n"
"%s=1\n"
"event_time_to_live=1\n"
"connection=log://localhost\n" % metering_ttl_name)
"connection=log://localhost\n" % ttl_name)
self.tempfile = fileutils.write_to_tempfile(content=content,
prefix='ceilometer',
suffix='.conf')
@ -72,14 +76,15 @@ class BinTestCase(base.BaseTestCase):
stderr=subprocess.PIPE)
__, err = subp.communicate()
self.assertEqual(0, subp.poll())
self.assertIn("Dropping metering data with TTL 1", err)
self.assertIn("Dropping event data with TTL 1", err)
self.assertIn("Dropping %s data with TTL 1" % data_name, err)
def test_run_expirer_ttl_enabled(self):
self._test_run_expirer_ttl_enabled('metering_time_to_live')
def test_run_expirer_ttl_enabled_with_deprecated_opt_name(self):
self._test_run_expirer_ttl_enabled('time_to_live')
self._test_run_expirer_ttl_enabled('metering_time_to_live',
'metering')
self._test_run_expirer_ttl_enabled('time_to_live', 'metering')
self._test_run_expirer_ttl_enabled('event_time_to_live', 'event')
self._test_run_expirer_ttl_enabled('alarm_history_time_to_live',
'alarm history')
class BinSendSampleTestCase(base.BaseTestCase):