Add periodic task to sync up bay status

This patch adds periodic task `sync_bay_status` which will be used for syncing
bay's status from heat stack, we will pull bays which status in
[bay_status.CREATE_IN_PROGRESS,
 bay_status.UPDATE_IN_PROGRESS,
 bay_status.DELETE_IN_PROGRESS]
which are all in a temporary status, and try to sync up the status with heat's
stack.

status changes will be like this:

bay_status              stack_status           sync up bay_status to
----------              ------------           ---------------------
CREATE_IN_PROGRESS      CREATE_COMPLETE        CREATE_COMPLETE
UPDATE_IN_PROGRESS      UPDATE_COMPLETE        UPDATE_COMPLETE
DELETE_IN_PROGRESS      DELETE_COMPLETE        DELETE_COMPLETE

CREATE_IN_PROGRESS      CREATE_FAILED          CREATE_FAILED
UPDATE_IN_PROGRESS      UPDATE_FAILED          UPDATE_FAILED
DELETE_IN_PROGRESS      DELETE_FAILED          DELETE_FAILED

CREATE_IN_PROGRESS      Not Found              CREATE_FAILED
UPDATE_IN_PROGRESS      Not Found              UPDATE_FAILED
DELETE_IN_PROGRESS      Not Found              destroy

Partial-Implements: blueprint add-periodic-task
Co-Authored-By: ShaoHe Feng <shaohe.feng@intel.com>
Change-Id: Ie9cc4d3f03c7938a8d988010604da79c9b8a22fd

magnum/common/context.py

@@ -10,6 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from eventlet.green import threading
 from oslo_context import context
 
 
@@ -87,3 +88,26 @@ def make_admin_context(show_deleted=False):
                              is_admin=True,
                              show_deleted=show_deleted)
     return context
+
+
+_CTX_STORE = threading.local()
+_CTX_KEY = 'current_ctx'
+
+
+def has_ctx():
+    return hasattr(_CTX_STORE, _CTX_KEY)
+
+
+def ctx():
+    return getattr(_CTX_STORE, _CTX_KEY)
+
+
+def set_ctx(new_ctx):
+    if not new_ctx and has_ctx():
+        delattr(_CTX_STORE, _CTX_KEY)
+        if hasattr(context._request_store, 'context'):
+            delattr(context._request_store, 'context')
+
+    if new_ctx:
+        setattr(_CTX_STORE, _CTX_KEY, new_ctx)
+        setattr(context._request_store, 'context', new_ctx)

magnum/service/periodic.py

@@ -12,19 +12,101 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import functools
+import six
+
 from oslo_log import log
 from oslo_service import periodic_task
 from oslo_service import threadgroup
 
+from magnum.common import clients
+from magnum.common import context
+from magnum.i18n import _LI
+from magnum.i18n import _LW
+from magnum import objects
+from magnum.objects.bay import Status as bay_status
+
 LOG = log.getLogger(__name__)
 
 
+def set_context(func):
+    @functools.wraps(func)
+    def handler(self, ctx):
+        ctx = context.make_admin_context()
+        context.set_ctx(ctx)
+        func(self, ctx)
+        context.set_ctx(None)
+    return handler
+
+
 class MagnumPeriodicTasks(periodic_task.PeriodicTasks):
     '''Magnum periodic Task class
 
     Any periodic task job need to be added into this class
     '''
-    pass
+
+    @periodic_task.periodic_task(run_immediately=True)
+    @set_context
+    def sync_bay_status(self, ctx):
+        try:
+            LOG.debug('Starting to sync up bay status')
+            osc = clients.OpenStackClients(ctx)
+            filters = [bay_status.CREATE_IN_PROGRESS,
+                       bay_status.UPDATE_IN_PROGRESS,
+                       bay_status.DELETE_IN_PROGRESS]
+            bays = objects.Bay.list_all(ctx, filters=filters)
+            if not bays:
+                return
+            sid_to_bay_mapping = {bay.stack_id: bay for bay in bays}
+            bay_stack_ids = sid_to_bay_mapping.keys()
+
+            stacks = osc.heat().stacks.list(global_tenant=True,
+                                            filters={'id': bay_stack_ids})
+            sid_to_stack_mapping = {s.id: s for s in stacks}
+
+            for sid in (six.viewkeys(sid_to_bay_mapping) &
+                        six.viewkeys(sid_to_stack_mapping)):
+                stack = sid_to_stack_mapping[sid]
+                bay = sid_to_bay_mapping[sid]
+                if bay.status != stack.stack_status:
+                    old_status = bay.status
+                    bay.status = stack.stack_status
+                    bay.save()
+                    LOG.info(_LI("Sync up bay with id %(id)s from "
+                                 "%(old_status)s to %(status)s"),
+                             {'id': bay.id, 'old_status': old_status,
+                              'status': bay.status})
+
+            for sid in (six.viewkeys(sid_to_bay_mapping) -
+                        six.viewkeys(sid_to_stack_mapping)):
+                bay = sid_to_bay_mapping[sid]
+                if bay.status == bay_status.DELETE_IN_PROGRESS:
+                    bay.destroy()
+                    LOG.info(_LI("Bay with id %(id)s has been deleted due "
+                                 "to stack with id %(sid)s not found in "
+                                 "heat."),
+                             {'id': bay.id, 'sid': sid})
+                elif bay.status == bay_status.CREATE_IN_PROGRESS:
+                    bay.status = bay_status.CREATE_FAILED
+                    bay.save()
+                    LOG.info(_LI("Bay with id %(id)s has been set to "
+                                 "%(status)s due to stack with id %(sid)s "
+                                 "not found in heat."),
+                             {'id': bay.id, 'status': bay.status,
+                              'sid': sid})
+                elif bay.status == bay_status.UPDATE_IN_PROGRESS:
+                    bay.status = bay_status.UPDATE_FAILED
+                    bay.save()
+                    LOG.info(_LI("Bay with id %(id)s has been set to "
+                                 "%(status)s due to stack with id %(sid)s "
+                                 "not found in heat."),
+                             {'id': bay.id, 'status': bay.status,
+                              'sid': sid})
+
+        except Exception as e:
+            LOG.warn(_LW("Ignore error [%s] when syncing up bay status."), e,
+                     exc_info=True)
 
 
 def setup(conf):

magnum/tests/unit/service/test_periodic.py

@@ -0,0 +1,126 @@
+# Copyright 2015 Intel, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from oslo_config import cfg
+
+from magnum.common import context
+from magnum.common.rpc_service import CONF
+from magnum.db.sqlalchemy import api as dbapi
+from magnum import objects
+from magnum.objects.bay import Status as bay_status
+from magnum.service import periodic
+from magnum.tests import base
+from magnum.tests.unit.db import utils
+
+periodic_opts = [
+    cfg.BoolOpt('periodic_enable',
+                default=True,
+                help='Enable periodic tasks.'),
+    cfg.IntOpt('periodic_interval_max',
+               default=60,
+               help='Max interval size between periodic tasks execution in '
+                    'seconds.'),
+]
+
+
+ctx = context.make_admin_context()
+
+bay1 = utils.get_test_bay(id=1, stack_id='11',
+                          status=bay_status.CREATE_IN_PROGRESS)
+bay2 = utils.get_test_bay(id=2, stack_id='22',
+                          status=bay_status.DELETE_IN_PROGRESS)
+bay3 = utils.get_test_bay(id=3, stack_id='33',
+                          status=bay_status.UPDATE_IN_PROGRESS)
+
+bay1 = objects.Bay(ctx, **bay1)
+bay2 = objects.Bay(ctx, **bay2)
+bay3 = objects.Bay(ctx, **bay3)
+
+
+class fake_stack(object):
+    def __init__(self, **kw):
+        for key, val in kw.items():
+            setattr(self, key, val)
+
+
+class PeriodictTestCase(base.TestCase):
+
+    @mock.patch.object(objects.Bay, 'list_all')
+    @mock.patch('magnum.common.clients.OpenStackClients')
+    @mock.patch.object(dbapi.Connection, 'destroy_bay')
+    @mock.patch.object(dbapi.Connection, 'update_bay')
+    def test_sync_bay_status_changes(self, mock_db_update, mock_db_destroy,
+                                     mock_oscc, mock_bay_list):
+        mock_heat_client = mock.MagicMock()
+        stack1 = fake_stack(id='11', stack_status=bay_status.CREATE_COMPLETE)
+        stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE)
+        mock_heat_client.stacks.list.return_value = [stack1, stack3]
+        mock_osc = mock_oscc.return_value
+        mock_osc.heat.return_value = mock_heat_client
+        mock_bay_list.return_value = [bay1, bay2, bay3]
+
+        mock_keystone_client = mock.MagicMock()
+        mock_keystone_client.client.project_id = "fake_project"
+        mock_osc.keystone.return_value = mock_keystone_client
+
+        periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
+
+        self.assertEqual(bay1.status, bay_status.CREATE_COMPLETE)
+        mock_db_destroy.assert_called_once_with(bay2.uuid)
+        self.assertEqual(bay3.status, bay_status.UPDATE_COMPLETE)
+
+    @mock.patch.object(objects.Bay, 'list_all')
+    @mock.patch('magnum.common.clients.OpenStackClients')
+    def test_sync_bay_status_not_changes(self, mock_oscc, mock_bay_list):
+        mock_heat_client = mock.MagicMock()
+        stack1 = fake_stack(id='11',
+                            stack_status=bay_status.CREATE_IN_PROGRESS)
+        stack2 = fake_stack(id='22',
+                            stack_status=bay_status.DELETE_IN_PROGRESS)
+        stack3 = fake_stack(id='33',
+                            stack_status=bay_status.UPDATE_IN_PROGRESS)
+        mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3]
+        mock_osc = mock_oscc.return_value
+        mock_osc.heat.return_value = mock_heat_client
+        mock_bay_list.return_value = [bay1, bay2, bay3]
+        periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
+
+        self.assertEqual(bay1.status, bay_status.CREATE_IN_PROGRESS)
+        self.assertEqual(bay2.status, bay_status.DELETE_IN_PROGRESS)
+        self.assertEqual(bay3.status, bay_status.UPDATE_IN_PROGRESS)
+
+    @mock.patch.object(objects.Bay, 'list_all')
+    @mock.patch('magnum.common.clients.OpenStackClients')
+    @mock.patch.object(dbapi.Connection, 'destroy_bay')
+    @mock.patch.object(dbapi.Connection, 'update_bay')
+    def test_sync_bay_status_heat_not_found(self, mock_db_update,
+                                            mock_db_destroy, mock_oscc,
+                                            mock_bay_list):
+        mock_heat_client = mock.MagicMock()
+        mock_heat_client.stacks.list.return_value = []
+        mock_osc = mock_oscc.return_value
+        mock_osc.heat.return_value = mock_heat_client
+        mock_bay_list.return_value = [bay1, bay2, bay3]
+
+        mock_keystone_client = mock.MagicMock()
+        mock_keystone_client.client.project_id = "fake_project"
+        mock_osc.keystone.return_value = mock_keystone_client
+
+        periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
+
+        self.assertEqual(bay1.status, bay_status.CREATE_FAILED)
+        mock_db_destroy.assert_called_once_with(bay2.uuid)
+        self.assertEqual(bay3.status, bay_status.UPDATE_FAILED)