From 9e1513c58a7bbdddeecc016660214aaceec1a3fb Mon Sep 17 00:00:00 2001 From: Maho Koshiya Date: Fri, 15 Aug 2014 17:58:03 +0900 Subject: [PATCH] Add the function of deleting alarm history Currently the ceilometer-expirer doesn't delete expired AlarmChanges. Remained AlarmChanges would be cause of wasted disk-space and slow response. This patch aims to delete AlarmChanges. This patch adds a new config option: [database] alarm_history_time_to_live = -1 Implements: blueprint delete-alarmhistory DocImpact Change-Id: Ib6bda6df720cf8514b621bfe13dd3acba941949f --- ceilometer/alarm/storage/base.py | 12 +++++ ceilometer/alarm/storage/impl_log.py | 12 +++++ ceilometer/alarm/storage/impl_mongodb.py | 20 +++++++++ ceilometer/alarm/storage/impl_sqlalchemy.py | 21 +++++++++ ceilometer/cmd/storage.py | 22 +++++++--- ceilometer/storage/__init__.py | 4 ++ ceilometer/tests/storage/test_impl_mongodb.py | 8 ++++ .../tests/storage/test_storage_scenarios.py | 44 +++++++++++++++++++ ceilometer/tests/test_bin.py | 25 ++++++----- 9 files changed, 152 insertions(+), 16 deletions(-) diff --git a/ceilometer/alarm/storage/base.py b/ceilometer/alarm/storage/base.py index 8e4a0e4cfe..f1735c9b7e 100644 --- a/ceilometer/alarm/storage/base.py +++ b/ceilometer/alarm/storage/base.py @@ -153,3 +153,15 @@ class Connection(object): This is needed to evaluate the performance of each driver. """ return cls.STORAGE_CAPABILITIES + + @staticmethod + def clear_expired_alarm_history_data(alarm_history_ttl): + """Clear expired alarm history data from the backend storage system. + + Clearing occurs according to the time-to-live. + + :param alarm_history_ttl: Number of seconds to keep alarm history + records for. + """ + raise ceilometer.NotImplementedError('Clearing alarm history ' + 'not implemented') diff --git a/ceilometer/alarm/storage/impl_log.py b/ceilometer/alarm/storage/impl_log.py index e7fb190dfc..e427eda13e 100644 --- a/ceilometer/alarm/storage/impl_log.py +++ b/ceilometer/alarm/storage/impl_log.py @@ -16,6 +16,7 @@ """ from ceilometer.alarm.storage import base +from ceilometer.i18n import _LI from ceilometer.openstack.common import log LOG = log.getLogger(__name__) @@ -46,3 +47,14 @@ class Connection(base.Connection): def delete_alarm(self, alarm_id): """Delete an alarm.""" + + def clear_expired_alarm_history_data(self, alarm_history_ttl): + """Clear expired alarm history data from the backend storage system. + + Clearing occurs according to the time-to-live. + + :param alarm_history_ttl: Number of seconds to keep alarm history + records for. + """ + LOG.info(_LI('Dropping alarm history data with TTL %d'), + alarm_history_ttl) diff --git a/ceilometer/alarm/storage/impl_mongodb.py b/ceilometer/alarm/storage/impl_mongodb.py index f4549af3a5..c8a66e12f3 100644 --- a/ceilometer/alarm/storage/impl_mongodb.py +++ b/ceilometer/alarm/storage/impl_mongodb.py @@ -20,13 +20,18 @@ # under the License. """MongoDB storage backend""" +from oslo_config import cfg import pymongo from ceilometer.alarm.storage import pymongo_base from ceilometer.openstack.common import log from ceilometer import storage +from ceilometer.storage import impl_mongodb from ceilometer.storage.mongo import utils as pymongo_utils +cfg.CONF.import_opt('alarm_history_time_to_live', 'ceilometer.alarm.storage', + group="database") + LOG = log.getLogger(__name__) @@ -64,8 +69,23 @@ class Connection(pymongo_base.Connection): self.db.conn.create_collection('alarm') if 'alarm_history' not in self.db.conn.collection_names(): self.db.conn.create_collection('alarm_history') + # Establish indexes + ttl = cfg.CONF.database.alarm_history_time_to_live + impl_mongodb.Connection.update_ttl( + ttl, 'alarm_history_ttl', 'timestamp', self.db.alarm_history) def clear(self): self.conn.drop_database(self.db.name) # Connection will be reopened automatically if needed self.conn.close() + + def clear_expired_alarm_history_data(self, alarm_history_ttl): + """Clear expired alarm history data from the backend storage system. + + Clearing occurs according to the time-to-live. + + :param alarm_history_ttl: Number of seconds to keep alarm history + records for. + """ + LOG.debug("Clearing expired alarm history data is based on native " + "MongoDB time to live feature and going in background.") diff --git a/ceilometer/alarm/storage/impl_sqlalchemy.py b/ceilometer/alarm/storage/impl_sqlalchemy.py index a6926f26f0..40024c8867 100644 --- a/ceilometer/alarm/storage/impl_sqlalchemy.py +++ b/ceilometer/alarm/storage/impl_sqlalchemy.py @@ -13,15 +13,18 @@ """SQLAlchemy storage backend.""" from __future__ import absolute_import +import datetime import os from oslo.db.sqlalchemy import session as db_session from oslo_config import cfg +from oslo_utils import timeutils from sqlalchemy import desc import ceilometer from ceilometer.alarm.storage import base from ceilometer.alarm.storage import models as alarm_api_models +from ceilometer.i18n import _LI from ceilometer.openstack.common import log from ceilometer.storage.sqlalchemy import models from ceilometer.storage.sqlalchemy import utils as sql_utils @@ -323,3 +326,21 @@ class Connection(base.Connection): event_id=alarm_change['event_id']) alarm_change_row.update(alarm_change) session.add(alarm_change_row) + + def clear_expired_alarm_history_data(self, alarm_history_ttl): + """Clear expired alarm history data from the backend storage system. + + Clearing occurs according to the time-to-live. + + :param alarm_history_ttl: Number of seconds to keep alarm history + records for. + """ + session = self._engine_facade.get_session() + with session.begin(): + valid_start = (timeutils.utcnow() - + datetime.timedelta(seconds=alarm_history_ttl)) + deleted_rows = (session.query(models.AlarmChange) + .filter(models.AlarmChange.timestamp < valid_start) + .delete()) + LOG.info(_LI("%d alarm histories are removed from database"), + deleted_rows) diff --git a/ceilometer/cmd/storage.py b/ceilometer/cmd/storage.py index 1cf9805599..c6f734eda2 100644 --- a/ceilometer/cmd/storage.py +++ b/ceilometer/cmd/storage.py @@ -18,10 +18,11 @@ import logging from oslo_config import cfg -from ceilometer.i18n import _ +from ceilometer.i18n import _, _LI from ceilometer import service from ceilometer import storage + LOG = logging.getLogger(__name__) @@ -37,12 +38,12 @@ def expirer(): if cfg.CONF.database.metering_time_to_live > 0: LOG.debug(_("Clearing expired metering data")) - storage_conn = storage.get_connection_from_config(cfg.CONF) + storage_conn = storage.get_connection_from_config(cfg.CONF, 'metering') storage_conn.clear_expired_metering_data( cfg.CONF.database.metering_time_to_live) else: - LOG.info(_("Nothing to clean, database metering time to live " - "is disabled")) + LOG.info(_LI("Nothing to clean, database metering time to live " + "is disabled")) if cfg.CONF.database.event_time_to_live > 0: LOG.debug(_("Clearing expired event data")) @@ -50,5 +51,14 @@ def expirer(): event_conn.clear_expired_event_data( cfg.CONF.database.event_time_to_live) else: - LOG.info(_("Nothing to clean, database event time to live " - "is disabled")) + LOG.info(_LI("Nothing to clean, database event time to live " + "is disabled")) + + if cfg.CONF.database.alarm_history_time_to_live > 0: + LOG.debug("Clearing expired alarm history data") + storage_conn = storage.get_connection_from_config(cfg.CONF, 'alarm') + storage_conn.clear_expired_alarm_history_data( + cfg.CONF.database.alarm_history_time_to_live) + else: + LOG.info(_LI("Nothing to clean, database alarm history time to live " + "is disabled")) diff --git a/ceilometer/storage/__init__.py b/ceilometer/storage/__init__.py index 907f43b537..8c386c4bd5 100644 --- a/ceilometer/storage/__init__.py +++ b/ceilometer/storage/__init__.py @@ -57,6 +57,10 @@ OPTS = [ default=None, help='The connection string used to connect to the alarm ' 'database. (if unset, connection is used)'), + cfg.IntOpt('alarm_history_time_to_live', + default=-1, + help=("Number of seconds that alarm histories are kept " + "in the database for (<= 0 means forever).")), cfg.StrOpt('event_connection', default=None, help='The connection string used to connect to the event ' diff --git a/ceilometer/tests/storage/test_impl_mongodb.py b/ceilometer/tests/storage/test_impl_mongodb.py index 28f6776cae..772394f960 100644 --- a/ceilometer/tests/storage/test_impl_mongodb.py +++ b/ceilometer/tests/storage/test_impl_mongodb.py @@ -109,6 +109,10 @@ class IndexTest(tests_db.TestBase, self._test_ttl_index_absent(self.event_conn, 'event', 'event_time_to_live') + def test_alarm_history_ttl_index_absent(self): + self._test_ttl_index_absent(self.alarm_conn, 'alarm_history', + 'alarm_history_time_to_live') + def _test_ttl_index_present(self, conn, coll_name, ttl_opt): coll = getattr(conn.db, coll_name) self.CONF.set_override(ttl_opt, 456789, group='database') @@ -130,6 +134,10 @@ class IndexTest(tests_db.TestBase, self._test_ttl_index_present(self.event_conn, 'event', 'event_time_to_live') + def test_alarm_history_ttl_index_present(self): + self._test_ttl_index_present(self.alarm_conn, 'alarm_history', + 'alarm_history_time_to_live') + @tests_db.run_with('mongodb') class AlarmTestPagination(test_storage_scenarios.AlarmTestBase, diff --git a/ceilometer/tests/storage/test_storage_scenarios.py b/ceilometer/tests/storage/test_storage_scenarios.py index b7cae21361..fab1e0c915 100644 --- a/ceilometer/tests/storage/test_storage_scenarios.py +++ b/ceilometer/tests/storage/test_storage_scenarios.py @@ -3018,6 +3018,50 @@ class AlarmTestPagination(AlarmTestBase, [i.name for i in page1]) +@tests_db.run_with('sqlite', 'mysql', 'pgsql', 'hbase', 'db2') +class AlarmHistoryTest(AlarmTestBase, + tests_db.MixinTestsWithBackendScenarios): + + def setUp(self): + super(AlarmTestBase, self).setUp() + self.add_some_alarms() + self.prepare_alarm_history() + + def prepare_alarm_history(self): + alarms = list(self.alarm_conn.get_alarms()) + for alarm in alarms: + i = alarms.index(alarm) + alarm_change = { + "event_id": "3e11800c-a3ca-4991-b34b-d97efb6047d%s" % i, + "alarm_id": alarm.alarm_id, + "type": alarm_models.AlarmChange.CREATION, + "detail": "detail %s" % alarm.name, + "user_id": alarm.user_id, + "project_id": alarm.project_id, + "on_behalf_of": alarm.project_id, + "timestamp": datetime.datetime(2014, 4, 7, 7, 30 + i) + } + self.alarm_conn.record_alarm_change(alarm_change=alarm_change) + + def _clear_alarm_history(self, utcnow, ttl, count): + self.mock_utcnow.return_value = utcnow + self.alarm_conn.clear_expired_alarm_history_data(ttl) + history = list(self.alarm_conn.query_alarm_history()) + self.assertEqual(count, len(history)) + + def test_clear_alarm_history_no_data_to_remove(self): + utcnow = datetime.datetime(2013, 4, 7, 7, 30) + self._clear_alarm_history(utcnow, 1, 3) + + def test_clear_some_alarm_history(self): + utcnow = datetime.datetime(2014, 4, 7, 7, 35) + self._clear_alarm_history(utcnow, 3 * 60, 1) + + def test_clear_all_alarm_history(self): + utcnow = datetime.datetime(2014, 4, 7, 7, 45) + self._clear_alarm_history(utcnow, 3 * 60, 0) + + class ComplexAlarmQueryTest(AlarmTestBase, tests_db.MixinTestsWithBackendScenarios): diff --git a/ceilometer/tests/test_bin.py b/ceilometer/tests/test_bin.py index 4027088443..4ec3d6bbd0 100644 --- a/ceilometer/tests/test_bin.py +++ b/ceilometer/tests/test_bin.py @@ -54,15 +54,19 @@ class BinTestCase(base.BaseTestCase): stderr=subprocess.PIPE) __, err = subp.communicate() self.assertEqual(0, subp.poll()) - self.assertIn("Nothing to clean", err) + self.assertIn("Nothing to clean, database metering " + "time to live is disabled", err) + self.assertIn("Nothing to clean, database event " + "time to live is disabled", err) + self.assertIn("Nothing to clean, database alarm history " + "time to live is disabled", err) - def _test_run_expirer_ttl_enabled(self, metering_ttl_name): + def _test_run_expirer_ttl_enabled(self, ttl_name, data_name): content = ("[DEFAULT]\n" "rpc_backend=fake\n" "[database]\n" "%s=1\n" - "event_time_to_live=1\n" - "connection=log://localhost\n" % metering_ttl_name) + "connection=log://localhost\n" % ttl_name) self.tempfile = fileutils.write_to_tempfile(content=content, prefix='ceilometer', suffix='.conf') @@ -72,14 +76,15 @@ class BinTestCase(base.BaseTestCase): stderr=subprocess.PIPE) __, err = subp.communicate() self.assertEqual(0, subp.poll()) - self.assertIn("Dropping metering data with TTL 1", err) - self.assertIn("Dropping event data with TTL 1", err) + self.assertIn("Dropping %s data with TTL 1" % data_name, err) def test_run_expirer_ttl_enabled(self): - self._test_run_expirer_ttl_enabled('metering_time_to_live') - - def test_run_expirer_ttl_enabled_with_deprecated_opt_name(self): - self._test_run_expirer_ttl_enabled('time_to_live') + self._test_run_expirer_ttl_enabled('metering_time_to_live', + 'metering') + self._test_run_expirer_ttl_enabled('time_to_live', 'metering') + self._test_run_expirer_ttl_enabled('event_time_to_live', 'event') + self._test_run_expirer_ttl_enabled('alarm_history_time_to_live', + 'alarm history') class BinSendSampleTestCase(base.BaseTestCase):