Add repeat_actions to alarm

If a user want to be sure that they don't miss a notification it can set
repeat_actions to true in a alarm.

All actions will be called of each evaluation of the alarm instead of
each change state of the alarm.

Change-Id: Ibed79db99531a702c99a0b3746a76586d927eb06

ceilometer/alarm/threshold_evaluation.py

@@ -124,13 +124,16 @@ class Evaluator(object):
             LOG.exception(_('alarm stats retrieval failed'))
             return []
 
-    def _update(self, alarm, state, reason):
+    def _refresh(self, alarm, state, reason):
         """Refresh alarm state."""
-        LOG.info(_('alarm %(id)s transitioning to %(state)s because '
-                   '%(reason)s') % {'id': alarm.alarm_id, 'state': state,
-                                    'reason': reason})
         try:
-            self._client.alarms.update(alarm.alarm_id, **dict(state=state))
+            if alarm.state != state:
+                LOG.info(_('alarm %(id)s transitioning to %(state)s because '
+                           '%(reason)s') % {'id': alarm.alarm_id,
+                                            'state': state,
+                                            'reason': reason})
+
+                self._client.alarms.update(alarm.alarm_id, **dict(state=state))
             alarm.state = state
             if self.notifier:
                 self.notifier.notify(alarm, state, reason)
@@ -146,7 +149,7 @@ class Evaluator(object):
         sufficient = len(statistics) >= self.quorum
         if not sufficient and alarm.state != UNKNOWN:
             reason = _('%d datapoints are unknown') % alarm.evaluation_periods
-            self._update(alarm, UNKNOWN, reason)
+            self._refresh(alarm, UNKNOWN, reason)
         return sufficient
 
     @staticmethod
@@ -155,10 +158,16 @@ class Evaluator(object):
         count = len(statistics)
         disposition = 'inside' if state == OK else 'outside'
         last = getattr(statistics[-1], alarm.statistic)
-        return (_('Transition to %(state)s due to %(count)d samples'
+        transition = alarm.state != state
+        if transition:
+            return (_('Transition to %(state)s due to %(count)d samples'
+                      ' %(disposition)s threshold, most recent: %(last)s') %
+                    {'state': state, 'count': count,
+                     'disposition': disposition, 'last': last})
+        return (_('Remaining as %(state)s due to %(count)d samples'
                   ' %(disposition)s threshold, most recent: %(last)s') %
-                {'state': state, 'count': count, 'disposition': disposition,
+                {'state': state, 'count': count,
-                 'last': last})
+                 'disposition': disposition, 'last': last})
 
     def _transition(self, alarm, statistics, compared):
         """Transition alarm state if necessary.
@@ -175,15 +184,19 @@ class Evaluator(object):
         """
         distilled = all(compared)
         unequivocal = distilled or not any(compared)
+        unknown = alarm.state == UNKNOWN
+        continuous = alarm.repeat_actions
+
         if unequivocal:
             state = ALARM if distilled else OK
-            if alarm.state != state:
-                reason = self._reason(alarm, statistics, distilled, state)
-                self._update(alarm, state, reason)
-        elif alarm.state == UNKNOWN:
-            state = ALARM if compared[-1] else OK
             reason = self._reason(alarm, statistics, distilled, state)
-            self._update(alarm, state, reason)
+            if alarm.state != state or continuous:
+                self._refresh(alarm, state, reason)
+        elif unknown or continuous:
+            trending_state = ALARM if compared[-1] else OK
+            state = trending_state if unknown else alarm.state
+            reason = self._reason(alarm, statistics, distilled, state)
+            self._refresh(alarm, state, reason)
 
     def evaluate(self):
         """Evaluate the alarms assigned to this evaluator."""

ceilometer/api/controllers/v2.py

@@ -782,6 +782,9 @@ class Alarm(_Base):
     insufficient_data_actions = [wtypes.text]
     "The actions to do when alarm state change to insufficient data"
 
+    repeat_actions = bool
+    "The actions should be re-triggered on each evaluation cycle"
+
     matching_metadata = {wtypes.text: wtypes.text}
     "The matching_metadata of the alarm"
 
@@ -809,7 +812,8 @@ class Alarm(_Base):
                    alarm_actions=["http://site:8000/alarm"],
                    insufficient_data_actions=["http://site:8000/nodata"],
                    matching_metadata={"key_name":
-                                      "key_value"}
+                                      "key_value"},
+                   repeat_actions=False,
                    )
 
 

ceilometer/storage/impl_sqlalchemy.py

@@ -516,7 +516,8 @@ class Connection(base.Connection):
                                 alarm_actions=row.alarm_actions,
                                 insufficient_data_actions=
                                 row.insufficient_data_actions,
-                                matching_metadata=row.matching_metadata)
+                                matching_metadata=row.matching_metadata,
+                                repeat_actions=row.repeat_actions)
 
     @staticmethod
     def _alarm_model_to_row(alarm, row=None):

ceilometer/storage/models.py

@@ -263,6 +263,8 @@ class Alarm(Model):
     :param insufficient_data_actions: the list of webhooks to call when
                                       entering the insufficient data state
     :param matching_metadata: the key/values of metadata to match on.
+    :param repeat_actions: Is the actions should be triggered on each
+                           alarm evaluation.
     """
     def __init__(self, name, counter_name,
                  comparison_operator, threshold, statistic,
@@ -278,7 +280,8 @@ class Alarm(Model):
                  ok_actions=[],
                  alarm_actions=[],
                  insufficient_data_actions=[],
-                 matching_metadata={}
+                 matching_metadata={},
+                 repeat_actions=False
                  ):
         if not description:
             # make a nice user friendly description by default
@@ -307,4 +310,5 @@ class Alarm(Model):
             alarm_actions=alarm_actions,
             insufficient_data_actions=
             insufficient_data_actions,
+            repeat_actions=repeat_actions,
             matching_metadata=matching_metadata)

ceilometer/storage/sqlalchemy/alembic/versions/43b1a023dfaa_add_column_repeat_al.py

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 eNovance <licensing@enovance.com>
+#
+# Authors: Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Add column repeat_alarms
+
+Revision ID: 43b1a023dfaa
+Revises: None
+Create Date: 2013-07-29 17:25:53.931326
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '43b1a023dfaa'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('alarm', sa.Column('repeat_actions',
+                                     sa.Boolean,
+                                     server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    op.drop_column('alarm', 'repeat_actions')

ceilometer/storage/sqlalchemy/models.py

@@ -190,6 +190,7 @@ class Alarm(Base):
     ok_actions = Column(JSONEncodedDict)
     alarm_actions = Column(JSONEncodedDict)
     insufficient_data_actions = Column(JSONEncodedDict)
+    repeat_actions = Column(Boolean)
 
     matching_metadata = Column(JSONEncodedDict)
 

tests/alarm/test_threshold_evaluation.py

@@ -188,6 +188,69 @@ class TestEvaluate(base.TestCase):
                              [])
             self.assertEqual(self.notifier.notify.call_args_list, [])
 
+    def test_equivocal_from_known_state_and_repeat_actions(self):
+        self._set_all_alarms('ok')
+        self.alarms[1].repeat_actions = True
+        with mock.patch('ceilometerclient.client.get_client',
+                        return_value=self.api_client):
+            avgs = [self._get_stat('avg', self.alarms[0].threshold + v)
+                    for v in xrange(5)]
+            maxs = [self._get_stat('max', self.alarms[1].threshold - v)
+                    for v in xrange(-1, 3)]
+            self.api_client.statistics.list.side_effect = [avgs, maxs]
+            self.evaluator.evaluate()
+            self._assert_all_alarms('ok')
+            self.assertEqual(self.api_client.alarms.update.call_args_list,
+                             [])
+            reason = 'Remaining as ok due to 4 samples inside' \
+                     ' threshold, most recent: 8.0'
+            expected = [mock.call(self.alarms[1], 'ok', reason)]
+            self.assertEqual(self.notifier.notify.call_args_list, expected)
+
+    def test_unequivocal_from_known_state_and_repeat_actions(self):
+        self._set_all_alarms('alarm')
+        self.alarms[1].repeat_actions = True
+        with mock.patch('ceilometerclient.client.get_client',
+                        return_value=self.api_client):
+            avgs = [self._get_stat('avg', self.alarms[0].threshold + v)
+                    for v in xrange(1, 6)]
+            maxs = [self._get_stat('max', self.alarms[1].threshold - v)
+                    for v in xrange(4)]
+            self.api_client.statistics.list.side_effect = [avgs, maxs]
+            self.evaluator.evaluate()
+            self._assert_all_alarms('alarm')
+            self.assertEqual(self.api_client.alarms.update.call_args_list,
+                             [])
+            reason = 'Remaining as alarm due to 4 samples outside' \
+                     ' threshold, most recent: 7.0'
+            expected = [mock.call(self.alarms[1], 'alarm', reason)]
+            self.assertEqual(self.notifier.notify.call_args_list, expected)
+
+    def test_state_change_and_repeat_actions(self):
+        self._set_all_alarms('ok')
+        self.alarms[0].repeat_actions = True
+        self.alarms[1].repeat_actions = True
+        with mock.patch('ceilometerclient.client.get_client',
+                        return_value=self.api_client):
+            avgs = [self._get_stat('avg', self.alarms[0].threshold + v)
+                    for v in xrange(1, 6)]
+            maxs = [self._get_stat('max', self.alarms[1].threshold - v)
+                    for v in xrange(4)]
+            self.api_client.statistics.list.side_effect = [avgs, maxs]
+            self.evaluator.evaluate()
+            self._assert_all_alarms('alarm')
+            expected = [mock.call(alarm.alarm_id, state='alarm')
+                        for alarm in self.alarms]
+            update_calls = self.api_client.alarms.update.call_args_list
+            self.assertEqual(update_calls, expected)
+            reasons = ['Transition to alarm due to 5 samples outside'
+                       ' threshold, most recent: 85.0',
+                       'Transition to alarm due to 4 samples outside'
+                       ' threshold, most recent: 7.0']
+            expected = [mock.call(alarm, 'alarm', reason)
+                        for alarm, reason in zip(self.alarms, reasons)]
+            self.assertEqual(self.notifier.notify.call_args_list, expected)
+
     def test_equivocal_from_unknown(self):
         self._set_all_alarms('insufficient data')
         with mock.patch('ceilometerclient.client.get_client',

tests/api/v2/test_alarm_scenarios.py

@@ -53,6 +53,7 @@ class TestAlarms(FunctionalTest,
                             counter_name='meter.test',
                             comparison_operator='gt', threshold=2.0,
                             statistic='avg',
+                            repeat_actions=True,
                             user_id=self.auth_headers['X-User-Id'],
                             project_id=self.auth_headers['X-Project-Id']),
                       Alarm(name='name2',
@@ -94,6 +95,7 @@ class TestAlarms(FunctionalTest,
         self.assertEquals(one['name'], 'name1')
         self.assertEquals(one['counter_name'], 'meter.test')
         self.assertEquals(one['alarm_id'], alarms[0]['alarm_id'])
+        self.assertEquals(one['repeat_actions'], alarms[0]['repeat_actions'])
 
     def test_post_invalid_alarm(self):
         json = {
@@ -115,15 +117,18 @@ class TestAlarms(FunctionalTest,
             'comparison_operator': 'gt',
             'threshold': 2.0,
             'statistic': 'avg',
+            'repeat_actions': True,
         }
         self.post_json('/alarms', params=json, status=200,
                        headers=self.auth_headers)
         alarms = list(self.conn.get_alarms())
         self.assertEquals(4, len(alarms))
+        self.assertEquals(alarms[3].repeat_actions, True)
 
     def test_put_alarm(self):
         json = {
             'name': 'renamed_alarm',
+            'repeat_actions': True,
         }
         data = self.get_json('/alarms',
                              q=[{'field': 'name',
@@ -137,6 +142,7 @@ class TestAlarms(FunctionalTest,
                       headers=self.auth_headers)
         alarm = list(self.conn.get_alarms(alarm_id=alarm_id))[0]
         self.assertEquals(alarm.name, json['name'])
+        self.assertEquals(alarm.repeat_actions, json['repeat_actions'])
 
     def test_put_alarm_wrong_field(self):
         # Note: wsme will ignore unknown fields so will just not appear in