Move cluster status notifications out of driver

Move cluster status change notifications into the
periodic task so that drivers do not have to have
any knowledge of Magnum notification strategy.

Change-Id: I5c71dd780f7bd6d4b683e491f5b4ce22cecb396c
Partial-Blueprint: bp-driver-consolodation

magnum/drivers/heat/driver.py

@@ -12,7 +12,6 @@
 
 import abc
 import os
-from pycadf import cadftaxonomy as taxonomy
 import six
 
 from oslo_config import cfg
@@ -151,17 +150,6 @@ class HeatDriver(driver.Driver):
 
 class HeatPoller(object):
 
-    status_to_event = {
-        fields.ClusterStatus.DELETE_COMPLETE: taxonomy.ACTION_DELETE,
-        fields.ClusterStatus.CREATE_COMPLETE: taxonomy.ACTION_CREATE,
-        fields.ClusterStatus.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE,
-        fields.ClusterStatus.ROLLBACK_COMPLETE: taxonomy.ACTION_UPDATE,
-        fields.ClusterStatus.CREATE_FAILED: taxonomy.ACTION_CREATE,
-        fields.ClusterStatus.DELETE_FAILED: taxonomy.ACTION_DELETE,
-        fields.ClusterStatus.UPDATE_FAILED: taxonomy.ACTION_UPDATE,
-        fields.ClusterStatus.ROLLBACK_FAILED: taxonomy.ACTION_UPDATE
-    }
-
     def __init__(self, openstack_client, cluster, cluster_driver):
         self.openstack_client = openstack_client
         self.context = self.openstack_client.context
@@ -184,18 +172,10 @@ class HeatPoller(object):
         # so another user/client can call delete cluster/stack.
         if stack.stack_status == fields.ClusterStatus.DELETE_COMPLETE:
             self._delete_complete()
-            # TODO(randall): Move the status notification up the stack
-            conductor_utils.notify_about_cluster_operation(
-                self.context, self.status_to_event[stack.stack_status],
-                taxonomy.OUTCOME_SUCCESS)
 
         if stack.stack_status in (fields.ClusterStatus.CREATE_COMPLETE,
                                   fields.ClusterStatus.UPDATE_COMPLETE):
             self._sync_cluster_and_template_status(stack)
-            # TODO(randall): Move the status notification up the stack
-            conductor_utils.notify_about_cluster_operation(
-                self.context, self.status_to_event[stack.stack_status],
-                taxonomy.OUTCOME_SUCCESS)
         elif stack.stack_status != self.cluster.status:
             self._sync_cluster_status(stack)
 
@@ -206,10 +186,6 @@ class HeatPoller(object):
                                   fields.ClusterStatus.ROLLBACK_FAILED):
             self._sync_cluster_and_template_status(stack)
             self._cluster_failed(stack)
-            # TODO(randall): Move the status notification up the stack
-            conductor_utils.notify_about_cluster_operation(
-                self.context, self.status_to_event[stack.stack_status],
-                taxonomy.OUTCOME_FAILURE)
 
     def _delete_complete(self):
         LOG.info(_LI('Cluster has been deleted, stack_id: %s')
@@ -220,7 +196,6 @@ class HeatPoller(object):
                                                    self.cluster)
             cert_manager.delete_certificates_from_cluster(self.cluster,
                                                           context=self.context)
-            self.cluster.destroy()
         except exception.ClusterNotFound:
             LOG.info(_LI('The cluster %s has been deleted by others.')
                      % self.cluster.uuid)
@@ -274,10 +249,6 @@ class HeatPoller(object):
         self.cluster.status_reason = _("Stack with id %s not found in "
                                        "Heat.") % self.cluster.stack_id
         self.cluster.save()
-        # TODO(randall): Move the status notification up the stack
-        conductor_utils.notify_about_cluster_operation(
-            self.context, self.status_to_event[self.cluster.status],
-            taxonomy.OUTCOME_FAILURE)
         LOG.info(_LI("Cluster with id %(id)s has been set to "
                      "%(status)s due to stack with id %(sid)s "
                      "not found in Heat."),

magnum/service/periodic.py

@@ -19,9 +19,12 @@ from oslo_log import log
 from oslo_service import loopingcall
 from oslo_service import periodic_task
 
+from pycadf import cadftaxonomy as taxonomy
+
 from magnum.common import context
 from magnum.common import rpc
 from magnum.conductor import monitors
+from magnum.conductor import utils as conductor_utils
 import magnum.conf
 from magnum.drivers.common import driver
 from magnum.i18n import _LW
@@ -44,6 +47,17 @@ def set_context(func):
 
 class ClusterUpdateJob(object):
 
+    status_to_event = {
+        objects.fields.ClusterStatus.DELETE_COMPLETE: taxonomy.ACTION_DELETE,
+        objects.fields.ClusterStatus.CREATE_COMPLETE: taxonomy.ACTION_CREATE,
+        objects.fields.ClusterStatus.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE,
+        objects.fields.ClusterStatus.ROLLBACK_COMPLETE: taxonomy.ACTION_UPDATE,
+        objects.fields.ClusterStatus.CREATE_FAILED: taxonomy.ACTION_CREATE,
+        objects.fields.ClusterStatus.DELETE_FAILED: taxonomy.ACTION_DELETE,
+        objects.fields.ClusterStatus.UPDATE_FAILED: taxonomy.ACTION_UPDATE,
+        objects.fields.ClusterStatus.ROLLBACK_FAILED: taxonomy.ACTION_UPDATE
+    }
+
     def __init__(self, ctx, cluster):
         self.ctx = ctx
         self.cluster = cluster
@@ -54,13 +68,22 @@ class ClusterUpdateJob(object):
         cdriver = driver.Driver.get_driver_for_cluster(self.ctx, self.cluster)
         # ask the driver to sync status
         cdriver.update_cluster_status(self.ctx, self.cluster)
-        # end the "loop"
         LOG.debug("Status for cluster %s updated to %s (%s)",
                   self.cluster.id, self.cluster.status,
                   self.cluster.status_reason)
+        # status update notifications
+        if self.cluster.status.endswith("_COMPLETE"):
+            conductor_utils.notify_about_cluster_operation(
+                self.ctx, self.status_to_event[self.cluster.status],
+                taxonomy.OUTCOME_SUCCESS)
+        if self.cluster.status.endswith("_FAILED"):
+            conductor_utils.notify_about_cluster_operation(
+                self.ctx, self.status_to_event[self.cluster.status],
+                taxonomy.OUTCOME_FAILURE)
         # if we're done with it, delete it
         if self.cluster.status == objects.fields.ClusterStatus.DELETE_COMPLETE:
             self.cluster.destroy()
+        # end the "loop"
         raise loopingcall.LoopingCallDone()
 
 

magnum/tests/unit/drivers/test_heat_driver.py

@@ -12,7 +12,6 @@
 
 import mock
 from mock import patch
-from pycadf import cadftaxonomy as taxonomy
 
 import magnum.conf
 from magnum.drivers.heat import driver as heat_driver
@@ -20,7 +19,6 @@ from magnum.drivers.k8s_fedora_atomic_v1 import driver as k8s_atomic_dr
 from magnum import objects
 from magnum.objects.fields import ClusterStatus as cluster_status
 from magnum.tests import base
-from magnum.tests import fake_notifier
 from magnum.tests.unit.db import utils
 
 CONF = magnum.conf.CONF
@@ -52,54 +50,6 @@ class TestHeatPoller(base.TestCase):
         poller.get_version_info = mock.MagicMock()
         return (mock_heat_stack, cluster, poller)
 
-    def test_poll_and_check_send_notification(self):
-        mock_heat_stack, cluster, poller = self.setup_poll_test()
-        mock_heat_stack.stack_status = cluster_status.CREATE_COMPLETE
-        self.assertIsNone(poller.poll_and_check())
-        self.assertEqual(mock_heat_stack.stack_status, cluster.status)
-        mock_heat_stack.stack_status = cluster_status.CREATE_FAILED
-        self.assertIsNone(poller.poll_and_check())
-        self.assertEqual(mock_heat_stack.stack_status, cluster.status)
-        mock_heat_stack.stack_status = cluster_status.DELETE_COMPLETE
-        self.assertIsNone(poller.poll_and_check())
-        self.assertEqual(mock_heat_stack.stack_status, cluster.status)
-        mock_heat_stack.stack_status = cluster_status.DELETE_FAILED
-        self.assertIsNone(poller.poll_and_check())
-        self.assertEqual(mock_heat_stack.stack_status, cluster.status)
-        mock_heat_stack.stack_status = cluster_status.UPDATE_COMPLETE
-        self.assertIsNone(poller.poll_and_check())
-        self.assertEqual(mock_heat_stack.stack_status, cluster.status)
-        mock_heat_stack.stack_status = cluster_status.UPDATE_FAILED
-        self.assertIsNone(poller.poll_and_check())
-        self.assertEqual(mock_heat_stack.stack_status, cluster.status)
-
-        notifications = fake_notifier.NOTIFICATIONS
-        self.assertEqual(6, len(notifications))
-        self.assertEqual(
-            'magnum.cluster.create', notifications[0].event_type)
-        self.assertEqual(
-            taxonomy.OUTCOME_SUCCESS, notifications[0].payload['outcome'])
-        self.assertEqual(
-            'magnum.cluster.create', notifications[1].event_type)
-        self.assertEqual(
-            taxonomy.OUTCOME_FAILURE, notifications[1].payload['outcome'])
-        self.assertEqual(
-            'magnum.cluster.delete', notifications[2].event_type)
-        self.assertEqual(
-            taxonomy.OUTCOME_SUCCESS, notifications[2].payload['outcome'])
-        self.assertEqual(
-            'magnum.cluster.delete', notifications[3].event_type)
-        self.assertEqual(
-            taxonomy.OUTCOME_FAILURE, notifications[3].payload['outcome'])
-        self.assertEqual(
-            'magnum.cluster.update', notifications[4].event_type)
-        self.assertEqual(
-            taxonomy.OUTCOME_SUCCESS, notifications[4].payload['outcome'])
-        self.assertEqual(
-            'magnum.cluster.update', notifications[5].event_type)
-        self.assertEqual(
-            taxonomy.OUTCOME_FAILURE, notifications[5].payload['outcome'])
-
     def test_poll_no_save(self):
         mock_heat_stack, cluster, poller = self.setup_poll_test()
 
@@ -188,11 +138,8 @@ class TestHeatPoller(base.TestCase):
 
         mock_heat_stack.stack_status = cluster_status.DELETE_COMPLETE
         self.assertIsNone(poller.poll_and_check())
-        # The cluster status should still be DELETE_IN_PROGRESS, because
+        # destroy and notifications are handled up the stack now
-        # the destroy() method may be failed. If success, this cluster record
-        # will delete directly, change status is meaningless.
         self.assertEqual(cluster_status.DELETE_COMPLETE, cluster.status)
-        self.assertEqual(1, cluster.destroy.call_count)
 
     def test_poll_node_count(self):
         mock_heat_stack, cluster, poller = self.setup_poll_test()
@@ -217,11 +164,9 @@ class TestHeatPoller(base.TestCase):
     def test_delete_complete(self, cert_manager, trust_manager):
         mock_heat_stack, cluster, poller = self.setup_poll_test()
         poller._delete_complete()
-        self.assertEqual(1, cluster.destroy.call_count)
         self.assertEqual(
             1, cert_manager.delete_certificates_from_cluster.call_count)
-        self.assertEqual(1,
+        self.assertEqual(1, trust_manager.delete_trustee_and_trust.call_count)
-                         trust_manager.delete_trustee_and_trust.call_count)
 
     def test_create_or_complete(self):
         mock_heat_stack, cluster, poller = self.setup_poll_test()

magnum/tests/unit/service/test_periodic.py

@@ -22,6 +22,7 @@ from magnum import objects
 from magnum.objects.fields import ClusterStatus as cluster_status
 from magnum.service import periodic
 from magnum.tests import base
+from magnum.tests import fake_notifier
 from magnum.tests import fakes
 from magnum.tests.unit.db import utils
 
@@ -151,6 +152,8 @@ class PeriodicTestCase(base.TestCase):
         self.assertEqual(cluster_status.ROLLBACK_COMPLETE,
                          self.cluster5.status)
         self.assertEqual('fake_reason_55', self.cluster5.status_reason)
+        notifications = fake_notifier.NOTIFICATIONS
+        self.assertEqual(4, len(notifications))
 
     @mock.patch('oslo_service.loopingcall.FixedIntervalLoopingCall',
                 new=fakes.FakeLoopingCall)
@@ -180,6 +183,8 @@ class PeriodicTestCase(base.TestCase):
         self.assertEqual(cluster_status.ROLLBACK_IN_PROGRESS,
                          self.cluster5.status)
         self.assertEqual('no change', self.cluster5.status_reason)
+        notifications = fake_notifier.NOTIFICATIONS
+        self.assertEqual(0, len(notifications))
 
     @mock.patch('oslo_service.loopingcall.FixedIntervalLoopingCall',
                 new=fakes.FakeLoopingCall)
@@ -208,6 +213,8 @@ class PeriodicTestCase(base.TestCase):
             mock.call(self.cluster4.uuid)
         ])
         self.assertEqual(2, mock_db_destroy.call_count)
+        notifications = fake_notifier.NOTIFICATIONS
+        self.assertEqual(5, len(notifications))
 
     @mock.patch('magnum.conductor.monitors.create_monitor')
     @mock.patch('magnum.objects.Cluster.list')