diff --git a/ceilometer/alarm/threshold_evaluation.py b/ceilometer/alarm/threshold_evaluation.py index 3bff077d..4e87915f 100644 --- a/ceilometer/alarm/threshold_evaluation.py +++ b/ceilometer/alarm/threshold_evaluation.py @@ -124,13 +124,16 @@ class Evaluator(object): LOG.exception(_('alarm stats retrieval failed')) return [] - def _update(self, alarm, state, reason): + def _refresh(self, alarm, state, reason): """Refresh alarm state.""" - LOG.info(_('alarm %(id)s transitioning to %(state)s because ' - '%(reason)s') % {'id': alarm.alarm_id, 'state': state, - 'reason': reason}) try: - self._client.alarms.update(alarm.alarm_id, **dict(state=state)) + if alarm.state != state: + LOG.info(_('alarm %(id)s transitioning to %(state)s because ' + '%(reason)s') % {'id': alarm.alarm_id, + 'state': state, + 'reason': reason}) + + self._client.alarms.update(alarm.alarm_id, **dict(state=state)) alarm.state = state if self.notifier: self.notifier.notify(alarm, state, reason) @@ -146,7 +149,7 @@ class Evaluator(object): sufficient = len(statistics) >= self.quorum if not sufficient and alarm.state != UNKNOWN: reason = _('%d datapoints are unknown') % alarm.evaluation_periods - self._update(alarm, UNKNOWN, reason) + self._refresh(alarm, UNKNOWN, reason) return sufficient @staticmethod @@ -155,10 +158,16 @@ class Evaluator(object): count = len(statistics) disposition = 'inside' if state == OK else 'outside' last = getattr(statistics[-1], alarm.statistic) - return (_('Transition to %(state)s due to %(count)d samples' + transition = alarm.state != state + if transition: + return (_('Transition to %(state)s due to %(count)d samples' + ' %(disposition)s threshold, most recent: %(last)s') % + {'state': state, 'count': count, + 'disposition': disposition, 'last': last}) + return (_('Remaining as %(state)s due to %(count)d samples' ' %(disposition)s threshold, most recent: %(last)s') % - {'state': state, 'count': count, 'disposition': disposition, - 'last': last}) + {'state': state, 'count': count, + 'disposition': disposition, 'last': last}) def _transition(self, alarm, statistics, compared): """Transition alarm state if necessary. @@ -175,15 +184,19 @@ class Evaluator(object): """ distilled = all(compared) unequivocal = distilled or not any(compared) + unknown = alarm.state == UNKNOWN + continuous = alarm.repeat_actions + if unequivocal: state = ALARM if distilled else OK - if alarm.state != state: - reason = self._reason(alarm, statistics, distilled, state) - self._update(alarm, state, reason) - elif alarm.state == UNKNOWN: - state = ALARM if compared[-1] else OK reason = self._reason(alarm, statistics, distilled, state) - self._update(alarm, state, reason) + if alarm.state != state or continuous: + self._refresh(alarm, state, reason) + elif unknown or continuous: + trending_state = ALARM if compared[-1] else OK + state = trending_state if unknown else alarm.state + reason = self._reason(alarm, statistics, distilled, state) + self._refresh(alarm, state, reason) def evaluate(self): """Evaluate the alarms assigned to this evaluator.""" diff --git a/ceilometer/api/controllers/v2.py b/ceilometer/api/controllers/v2.py index 3a9ef263..8a660357 100644 --- a/ceilometer/api/controllers/v2.py +++ b/ceilometer/api/controllers/v2.py @@ -782,6 +782,9 @@ class Alarm(_Base): insufficient_data_actions = [wtypes.text] "The actions to do when alarm state change to insufficient data" + repeat_actions = bool + "The actions should be re-triggered on each evaluation cycle" + matching_metadata = {wtypes.text: wtypes.text} "The matching_metadata of the alarm" @@ -809,7 +812,8 @@ class Alarm(_Base): alarm_actions=["http://site:8000/alarm"], insufficient_data_actions=["http://site:8000/nodata"], matching_metadata={"key_name": - "key_value"} + "key_value"}, + repeat_actions=False, ) diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index f89e3055..86a70524 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -516,7 +516,8 @@ class Connection(base.Connection): alarm_actions=row.alarm_actions, insufficient_data_actions= row.insufficient_data_actions, - matching_metadata=row.matching_metadata) + matching_metadata=row.matching_metadata, + repeat_actions=row.repeat_actions) @staticmethod def _alarm_model_to_row(alarm, row=None): diff --git a/ceilometer/storage/models.py b/ceilometer/storage/models.py index 1129ca86..3d223634 100644 --- a/ceilometer/storage/models.py +++ b/ceilometer/storage/models.py @@ -263,6 +263,8 @@ class Alarm(Model): :param insufficient_data_actions: the list of webhooks to call when entering the insufficient data state :param matching_metadata: the key/values of metadata to match on. + :param repeat_actions: Is the actions should be triggered on each + alarm evaluation. """ def __init__(self, name, counter_name, comparison_operator, threshold, statistic, @@ -278,7 +280,8 @@ class Alarm(Model): ok_actions=[], alarm_actions=[], insufficient_data_actions=[], - matching_metadata={} + matching_metadata={}, + repeat_actions=False ): if not description: # make a nice user friendly description by default @@ -307,4 +310,5 @@ class Alarm(Model): alarm_actions=alarm_actions, insufficient_data_actions= insufficient_data_actions, + repeat_actions=repeat_actions, matching_metadata=matching_metadata) diff --git a/ceilometer/storage/sqlalchemy/alembic/versions/43b1a023dfaa_add_column_repeat_al.py b/ceilometer/storage/sqlalchemy/alembic/versions/43b1a023dfaa_add_column_repeat_al.py new file mode 100644 index 00000000..e1ef27c7 --- /dev/null +++ b/ceilometer/storage/sqlalchemy/alembic/versions/43b1a023dfaa_add_column_repeat_al.py @@ -0,0 +1,41 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 eNovance +# +# Authors: Mehdi Abaakouk +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Add column repeat_alarms + +Revision ID: 43b1a023dfaa +Revises: None +Create Date: 2013-07-29 17:25:53.931326 + +""" + +# revision identifiers, used by Alembic. +revision = '43b1a023dfaa' +down_revision = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('alarm', sa.Column('repeat_actions', + sa.Boolean, + server_default=sa.sql.expression.false())) + + +def downgrade(): + op.drop_column('alarm', 'repeat_actions') diff --git a/ceilometer/storage/sqlalchemy/models.py b/ceilometer/storage/sqlalchemy/models.py index f17a283c..4af71a42 100644 --- a/ceilometer/storage/sqlalchemy/models.py +++ b/ceilometer/storage/sqlalchemy/models.py @@ -190,6 +190,7 @@ class Alarm(Base): ok_actions = Column(JSONEncodedDict) alarm_actions = Column(JSONEncodedDict) insufficient_data_actions = Column(JSONEncodedDict) + repeat_actions = Column(Boolean) matching_metadata = Column(JSONEncodedDict) diff --git a/tests/alarm/test_threshold_evaluation.py b/tests/alarm/test_threshold_evaluation.py index 7fc77bcd..3f70c336 100644 --- a/tests/alarm/test_threshold_evaluation.py +++ b/tests/alarm/test_threshold_evaluation.py @@ -188,6 +188,69 @@ class TestEvaluate(base.TestCase): []) self.assertEqual(self.notifier.notify.call_args_list, []) + def test_equivocal_from_known_state_and_repeat_actions(self): + self._set_all_alarms('ok') + self.alarms[1].repeat_actions = True + with mock.patch('ceilometerclient.client.get_client', + return_value=self.api_client): + avgs = [self._get_stat('avg', self.alarms[0].threshold + v) + for v in xrange(5)] + maxs = [self._get_stat('max', self.alarms[1].threshold - v) + for v in xrange(-1, 3)] + self.api_client.statistics.list.side_effect = [avgs, maxs] + self.evaluator.evaluate() + self._assert_all_alarms('ok') + self.assertEqual(self.api_client.alarms.update.call_args_list, + []) + reason = 'Remaining as ok due to 4 samples inside' \ + ' threshold, most recent: 8.0' + expected = [mock.call(self.alarms[1], 'ok', reason)] + self.assertEqual(self.notifier.notify.call_args_list, expected) + + def test_unequivocal_from_known_state_and_repeat_actions(self): + self._set_all_alarms('alarm') + self.alarms[1].repeat_actions = True + with mock.patch('ceilometerclient.client.get_client', + return_value=self.api_client): + avgs = [self._get_stat('avg', self.alarms[0].threshold + v) + for v in xrange(1, 6)] + maxs = [self._get_stat('max', self.alarms[1].threshold - v) + for v in xrange(4)] + self.api_client.statistics.list.side_effect = [avgs, maxs] + self.evaluator.evaluate() + self._assert_all_alarms('alarm') + self.assertEqual(self.api_client.alarms.update.call_args_list, + []) + reason = 'Remaining as alarm due to 4 samples outside' \ + ' threshold, most recent: 7.0' + expected = [mock.call(self.alarms[1], 'alarm', reason)] + self.assertEqual(self.notifier.notify.call_args_list, expected) + + def test_state_change_and_repeat_actions(self): + self._set_all_alarms('ok') + self.alarms[0].repeat_actions = True + self.alarms[1].repeat_actions = True + with mock.patch('ceilometerclient.client.get_client', + return_value=self.api_client): + avgs = [self._get_stat('avg', self.alarms[0].threshold + v) + for v in xrange(1, 6)] + maxs = [self._get_stat('max', self.alarms[1].threshold - v) + for v in xrange(4)] + self.api_client.statistics.list.side_effect = [avgs, maxs] + self.evaluator.evaluate() + self._assert_all_alarms('alarm') + expected = [mock.call(alarm.alarm_id, state='alarm') + for alarm in self.alarms] + update_calls = self.api_client.alarms.update.call_args_list + self.assertEqual(update_calls, expected) + reasons = ['Transition to alarm due to 5 samples outside' + ' threshold, most recent: 85.0', + 'Transition to alarm due to 4 samples outside' + ' threshold, most recent: 7.0'] + expected = [mock.call(alarm, 'alarm', reason) + for alarm, reason in zip(self.alarms, reasons)] + self.assertEqual(self.notifier.notify.call_args_list, expected) + def test_equivocal_from_unknown(self): self._set_all_alarms('insufficient data') with mock.patch('ceilometerclient.client.get_client', diff --git a/tests/api/v2/test_alarm_scenarios.py b/tests/api/v2/test_alarm_scenarios.py index 92dedd2b..e2a3dfad 100644 --- a/tests/api/v2/test_alarm_scenarios.py +++ b/tests/api/v2/test_alarm_scenarios.py @@ -53,6 +53,7 @@ class TestAlarms(FunctionalTest, counter_name='meter.test', comparison_operator='gt', threshold=2.0, statistic='avg', + repeat_actions=True, user_id=self.auth_headers['X-User-Id'], project_id=self.auth_headers['X-Project-Id']), Alarm(name='name2', @@ -94,6 +95,7 @@ class TestAlarms(FunctionalTest, self.assertEquals(one['name'], 'name1') self.assertEquals(one['counter_name'], 'meter.test') self.assertEquals(one['alarm_id'], alarms[0]['alarm_id']) + self.assertEquals(one['repeat_actions'], alarms[0]['repeat_actions']) def test_post_invalid_alarm(self): json = { @@ -115,15 +117,18 @@ class TestAlarms(FunctionalTest, 'comparison_operator': 'gt', 'threshold': 2.0, 'statistic': 'avg', + 'repeat_actions': True, } self.post_json('/alarms', params=json, status=200, headers=self.auth_headers) alarms = list(self.conn.get_alarms()) self.assertEquals(4, len(alarms)) + self.assertEquals(alarms[3].repeat_actions, True) def test_put_alarm(self): json = { 'name': 'renamed_alarm', + 'repeat_actions': True, } data = self.get_json('/alarms', q=[{'field': 'name', @@ -137,6 +142,7 @@ class TestAlarms(FunctionalTest, headers=self.auth_headers) alarm = list(self.conn.get_alarms(alarm_id=alarm_id))[0] self.assertEquals(alarm.name, json['name']) + self.assertEquals(alarm.repeat_actions, json['repeat_actions']) def test_put_alarm_wrong_field(self): # Note: wsme will ignore unknown fields so will just not appear in