Merge "Combine the patch audit with the subcloud audit"

distributedcloud/dcmanager/audit/patch_audit.py

@@ -0,0 +1,248 @@
+# Copyright 2017 Ericsson AB.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright (c) 2017-2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+from keystoneauth1 import exceptions as keystone_exceptions
+from oslo_log import log as logging
+
+from dccommon.drivers.openstack import patching_v1
+from dccommon.drivers.openstack.patching_v1 import PatchingClient
+from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
+from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
+
+from dcorch.common import consts as dcorch_consts
+
+from dcmanager.common import consts
+from dcmanager.common.i18n import _
+
+
+LOG = logging.getLogger(__name__)
+
+
+class PatchAuditData(object):
+    def __init__(self, patches, applied_patch_ids,
+                 committed_patch_ids,
+                 software_version):
+        self.patches = patches
+        self.applied_patch_ids = applied_patch_ids
+        self.committed_patch_ids = committed_patch_ids
+        self.software_version = software_version
+
+
+class PatchAudit(object):
+    """Manages tasks related to patch audits."""
+
+    def __init__(self, context, dcmanager_rpc_client):
+        LOG.info(_('PatchAudit initialization...'))
+        self.context = context
+        self.dcmanager_rpc_client = dcmanager_rpc_client
+        self.audit_count = 0
+
+    def _update_subcloud_sync_status(self, sc_name, sc_endpoint_type,
+                                     sc_status):
+        self.dcmanager_rpc_client.update_subcloud_endpoint_status(
+            self.context,
+            subcloud_name=sc_name,
+            endpoint_type=sc_endpoint_type,
+            sync_status=sc_status)
+
+    @staticmethod
+    def _get_upgrades(sysinv_client):
+        upgrades = None
+        try:
+            upgrades = sysinv_client.get_upgrades()
+        except Exception:
+            LOG.exception('Cannot retrieve upgrade info for subcloud: %s' %
+                          sysinv_client.region_name)
+        return upgrades
+
+    def get_regionone_audit_data(self):
+        """Query RegionOne to determine what patches should be applied
+
+        to the system as well as the current software version
+
+        :return: A new PatchAuditData object
+
+        """
+        try:
+            m_os_ks_client = OpenStackDriver(
+                region_name=consts.DEFAULT_REGION_NAME,
+                region_clients=None).keystone_client
+            patching_client = PatchingClient(
+                consts.DEFAULT_REGION_NAME, m_os_ks_client.session)
+            sysinv_client = SysinvClient(
+                consts.DEFAULT_REGION_NAME, m_os_ks_client.session)
+        except Exception:
+            LOG.exception('Failure initializing OS Client, skip patch audit.')
+            return None
+
+        # First query RegionOne to determine what patches should be applied
+        # to the system.
+        regionone_patches = patching_client.query()
+        LOG.debug("regionone_patches: %s" % regionone_patches)
+
+        # Get the active software version in RegionOne as it may be needed
+        # later for subcloud load audit.
+        regionone_software_version = \
+            sysinv_client.get_system().software_version
+
+        # Build lists of patches that should be applied or committed in all
+        # subclouds, based on their state in RegionOne. Check repostate
+        # (not patchstate) as we only care if the patch has been applied to
+        # the repo (not whether it is installed on the hosts).
+        applied_patch_ids = list()
+        committed_patch_ids = list()
+        for patch_id in regionone_patches.keys():
+            if regionone_patches[patch_id]['repostate'] == \
+                    patching_v1.PATCH_STATE_APPLIED:
+                applied_patch_ids.append(patch_id)
+            elif regionone_patches[patch_id]['repostate'] == \
+                    patching_v1.PATCH_STATE_COMMITTED:
+                committed_patch_ids.append(patch_id)
+        LOG.debug("RegionOne applied_patch_ids: %s" % applied_patch_ids)
+        LOG.debug("RegionOne committed_patch_ids: %s" % committed_patch_ids)
+        return PatchAuditData(regionone_patches, applied_patch_ids,
+                              committed_patch_ids, regionone_software_version)
+
+    def subcloud_patch_audit(self, subcloud_name, audit_data, do_load_audit):
+        LOG.info('Triggered patch audit for subcloud: %s.' % subcloud_name)
+        try:
+            sc_os_client = OpenStackDriver(region_name=subcloud_name,
+                                           region_clients=None)
+            session = sc_os_client.keystone_client.session
+            patching_client = PatchingClient(subcloud_name, session)
+            sysinv_client = SysinvClient(subcloud_name, session)
+        except (keystone_exceptions.EndpointNotFound,
+                keystone_exceptions.ConnectFailure,
+                keystone_exceptions.ConnectTimeout,
+                IndexError):
+            LOG.exception("Endpoint for online subcloud %s not found, skip "
+                          "patch audit." % subcloud_name)
+            return
+
+        # Retrieve all the patches that are present in this subcloud.
+        try:
+            subcloud_patches = patching_client.query()
+            LOG.debug("Patches for subcloud %s: %s" %
+                      (subcloud_name, subcloud_patches))
+        except Exception:
+            LOG.warn('Cannot retrieve patches for subcloud: %s, skip patch '
+                     'audit' % subcloud_name)
+            return
+
+        # Determine which loads are present in this subcloud. During an
+        # upgrade, there will be more than one load installed.
+        installed_loads = list()
+        try:
+            loads = sysinv_client.get_loads()
+        except Exception:
+            LOG.exception('Cannot retrieve installed loads for subcloud: %s, '
+                          'skip patch audit' % subcloud_name)
+            return
+
+        for load in loads:
+            installed_loads.append(load.software_version)
+
+        out_of_sync = False
+
+        # Check that all patches in this subcloud are in the correct
+        # state, based on the state of the patch in RegionOne. For the
+        # subcloud, we use the patchstate because we care whether the
+        # patch is installed on the hosts.
+        for patch_id in subcloud_patches.keys():
+            if subcloud_patches[patch_id]['patchstate'] == \
+                    patching_v1.PATCH_STATE_APPLIED:
+                if patch_id not in audit_data.applied_patch_ids:
+                    if patch_id not in audit_data.committed_patch_ids:
+                        LOG.debug("Patch %s should not be applied in %s" %
+                                  (patch_id, subcloud_name))
+                    else:
+                        LOG.debug("Patch %s should be committed in %s" %
+                                  (patch_id, subcloud_name))
+                    out_of_sync = True
+            elif subcloud_patches[patch_id]['patchstate'] == \
+                    patching_v1.PATCH_STATE_COMMITTED:
+                if patch_id not in audit_data.committed_patch_ids:
+                    LOG.warn("Patch %s should not be committed in %s" %
+                             (patch_id, subcloud_name))
+                    out_of_sync = True
+            else:
+                # In steady state, all patches should either be applied
+                # or committed in each subcloud. Patches in other
+                # states mean a sync is required.
+                out_of_sync = True
+
+        # Check that all applied or committed patches in RegionOne are
+        # present in the subcloud.
+        for patch_id in audit_data.applied_patch_ids:
+            if audit_data.patches[patch_id]['sw_version'] in \
+                    installed_loads and patch_id not in \
+                    subcloud_patches:
+                LOG.debug("Patch %s missing from %s" %
+                          (patch_id, subcloud_name))
+                out_of_sync = True
+        for patch_id in audit_data.committed_patch_ids:
+            if audit_data.patches[patch_id]['sw_version'] in \
+                    installed_loads and patch_id not in \
+                    subcloud_patches:
+                LOG.debug("Patch %s missing from %s" %
+                          (patch_id, subcloud_name))
+                out_of_sync = True
+
+        if out_of_sync:
+            self._update_subcloud_sync_status(
+                subcloud_name, dcorch_consts.ENDPOINT_TYPE_PATCHING,
+                consts.SYNC_STATUS_OUT_OF_SYNC)
+        else:
+            self._update_subcloud_sync_status(
+                subcloud_name, dcorch_consts.ENDPOINT_TYPE_PATCHING,
+                consts.SYNC_STATUS_IN_SYNC)
+
+        # Check subcloud software version every other audit cycle
+        if do_load_audit:
+            LOG.info('Auditing load of subcloud %s' % subcloud_name)
+            try:
+                upgrades = sysinv_client.get_upgrades()
+            except Exception:
+                LOG.warn('Cannot retrieve upgrade info for subcloud: %s, skip '
+                         'software version audit' % subcloud_name)
+                return
+
+            if not upgrades:
+                # No upgrade in progress
+                subcloud_software_version = \
+                    sysinv_client.get_system().software_version
+
+                if subcloud_software_version == audit_data.software_version:
+                    self._update_subcloud_sync_status(
+                        subcloud_name, dcorch_consts.ENDPOINT_TYPE_LOAD,
+                        consts.SYNC_STATUS_IN_SYNC)
+                else:
+                    self._update_subcloud_sync_status(
+                        subcloud_name, dcorch_consts.ENDPOINT_TYPE_LOAD,
+                        consts.SYNC_STATUS_OUT_OF_SYNC)
+            else:
+                # As upgrade is still in progress, set the subcloud load
+                # status as out-of-sync.
+                self._update_subcloud_sync_status(
+                    subcloud_name, dcorch_consts.ENDPOINT_TYPE_LOAD,
+                    consts.SYNC_STATUS_OUT_OF_SYNC)
+        LOG.info('Patch audit completed for subcloud: %s.' % subcloud_name)

distributedcloud/dcmanager/audit/rpcapi.py

@@ -0,0 +1,63 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+"""
+Client side of the DC Manager Audit RPC API.
+"""
+
+from dcmanager.common import consts
+from dcmanager.common import messaging
+
+
+class ManagerAuditClient(object):
+    """Client side of the DC Manager Audit rpc API.
+
+    Version History:
+     1.0 - Initial version
+    """
+
+    BASE_RPC_API_VERSION = '1.0'
+
+    def __init__(self):
+        self._client = messaging.get_rpc_client(
+            topic=consts.TOPIC_DC_MANAGER_AUDIT,
+            version=self.BASE_RPC_API_VERSION)
+
+    @staticmethod
+    def make_msg(method, **kwargs):
+        return method, kwargs
+
+    def call(self, ctxt, msg, version=None):
+        method, kwargs = msg
+        if version is not None:
+            client = self._client.prepare(version=version)
+        else:
+            client = self._client
+        return client.call(ctxt, method, **kwargs)
+
+    def cast(self, ctxt, msg, version=None):
+        method, kwargs = msg
+        if version is not None:
+            client = self._client.prepare(version=version)
+        else:
+            client = self._client
+        return client.cast(ctxt, method, **kwargs)
+
+    def trigger_patch_audit(self, ctxt):
+        return self.cast(ctxt, self.make_msg('trigger_patch_audit'))

distributedcloud/dcmanager/audit/service.py

@@ -20,6 +20,7 @@
 
 import six
 
+import functools
 from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging
@@ -27,6 +28,8 @@ from oslo_service import service
 
 from dcmanager.audit.subcloud_audit_manager import SubcloudAuditManager
 from dcmanager.common import consts
+from dcmanager.common import context
+from dcmanager.common import exceptions
 from dcmanager.common.i18n import _
 from dcmanager.common import messaging as rpc_messaging
 from dcmanager.common import scheduler
@@ -35,6 +38,19 @@ CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
 
+def request_context(func):
+    @functools.wraps(func)
+    def wrapped(self, ctx, *args, **kwargs):
+        if ctx is not None and not isinstance(ctx, context.RequestContext):
+            ctx = context.RequestContext.from_dict(ctx.to_dict())
+        try:
+            return func(self, ctx, *args, **kwargs)
+        except exceptions.DCManagerException:
+            raise oslo_messaging.rpc.dispatcher.ExpectedException()
+
+    return wrapped
+
+
 class DCManagerAuditService(service.Service):
     """Lifecycle manager for a running audit service."""
 
@@ -91,3 +107,10 @@ class DCManagerAuditService(service.Service):
         # Terminate the engine process
         LOG.info("All threads were gone, terminating engine")
         super(DCManagerAuditService, self).stop()
+
+    @request_context
+    def trigger_patch_audit(self, context):
+        """Used to force a patch audit on the next interval"""
+
+        LOG.info("Trigger patch audit.")
+        return self.subcloud_audit_manager.trigger_patch_audit(context)

distributedcloud/dcmanager/audit/subcloud_audit_manager.py

@@ -32,6 +32,7 @@ from dccommon import consts as dccommon_consts
 from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
 
 from dcmanager.audit import alarm_aggregation
+from dcmanager.audit import patch_audit
 from dcmanager.common import consts
 from dcmanager.common import context
 from dcmanager.common import exceptions
@@ -49,10 +50,18 @@ LOG = logging.getLogger(__name__)
 SUBCLOUD_STATE_UPDATE_ITERATIONS = \
     dccommon_consts.SECONDS_IN_HOUR / CONF.scheduler.subcloud_audit_interval
 
+# Patch audit normally happens every DEFAULT_PATCH_AUDIT_DELAY_SECONDS, but
+# can be forced to happen on the next audit interval by calling
+# trigger_patch_audit.
+DEFAULT_PATCH_AUDIT_DELAY_SECONDS = 300
+
 
 class SubcloudAuditManager(manager.Manager):
     """Manages tasks related to audits."""
 
+    # Used to force patch audit on the next interval
+    force_patch_audit = False
+
     def __init__(self, *args, **kwargs):
         LOG.debug(_('SubcloudAuditManager initialization...'))
 
@@ -67,7 +76,24 @@ class SubcloudAuditManager(manager.Manager):
         self.subcloud_workers = dict()
         # Number of audits since last subcloud state update
         self.audit_count = 0
+        # Number of patch audits
+        self.patch_audit_count = 0
         self.alarm_aggr = alarm_aggregation.AlarmAggregation(self.context)
+        self.patch_audit = patch_audit.PatchAudit(
+            self.context, self.dcmanager_rpc_client)
+        self.patch_audit_wait_time_passed = DEFAULT_PATCH_AUDIT_DELAY_SECONDS
+
+    @classmethod
+    def trigger_patch_audit(cls, context):
+        """Trigger patch audit at next interval.
+
+        This can be called from outside the dcmanager audit
+        """
+        cls.force_patch_audit = True
+
+    @classmethod
+    def reset_force_patch_audit(cls):
+        cls.force_patch_audit = False
 
     def periodic_subcloud_audit(self):
         """Audit availability of subclouds."""
@@ -85,6 +111,35 @@ class SubcloudAuditManager(manager.Manager):
             except Exception:
                 LOG.exception("Error in periodic subcloud audit loop")
 
+    def _get_patch_audit(self):
+        """Return the patch audit data if the patch audit should be triggered.
+
+           Also, returns whether to audit the load.
+        """
+        patch_audit_data = None
+        audit_load = False
+
+        # This won't be super accurate as we aren't woken up after exactly
+        # the interval seconds, but it is good enough for an audit.
+        self.patch_audit_wait_time_passed +=\
+            CONF.scheduler.subcloud_audit_interval
+        # Determine whether to trigger a patch audit of each subcloud
+        if (SubcloudAuditManager.force_patch_audit or
+                self.patch_audit_wait_time_passed >=
+                DEFAULT_PATCH_AUDIT_DELAY_SECONDS):
+            LOG.info("Trigger patch audit")
+            self.patch_audit_count += 1
+            # Query RegionOne patches and software version
+            patch_audit_data = self.patch_audit.get_regionone_audit_data()
+            # Check subcloud software version every other audit cycle
+            if self.patch_audit_count % 2 != 0:
+                LOG.info("Trigger load audit")
+                audit_load = True
+            SubcloudAuditManager.reset_force_patch_audit()
+            self.patch_audit_wait_time_passed = 0
+
+        return patch_audit_data, audit_load
+
     def _periodic_subcloud_audit_loop(self):
         """Audit availability of subclouds loop."""
 
@@ -95,9 +150,12 @@ class SubcloudAuditManager(manager.Manager):
         # Determine whether to trigger a state update to each subcloud
         if self.audit_count >= SUBCLOUD_STATE_UPDATE_ITERATIONS:
             update_subcloud_state = True
+            self.audit_count = 0
         else:
             update_subcloud_state = False
 
+        patch_audit_data, do_load_audit = self._get_patch_audit()
+
         openstack_installed = False
         # The feature of syncing openstack resources to the subclouds was not
         # completed, therefore, auditing the openstack application is disabled
@@ -131,7 +189,9 @@ class SubcloudAuditManager(manager.Manager):
                 self.thread_group_manager.start(self._audit_subcloud,
                                                 subcloud.name,
                                                 update_subcloud_state,
-                                                openstack_installed)
+                                                openstack_installed,
+                                                patch_audit_data,
+                                                do_load_audit)
 
         # Wait for all greenthreads to complete
         LOG.info('Waiting for subcloud audits to complete.')
@@ -236,7 +296,7 @@ class SubcloudAuditManager(manager.Manager):
                 openstack_installed_current)
 
     def _audit_subcloud(self, subcloud_name, update_subcloud_state,
-                        audit_openstack):
+                        audit_openstack, patch_audit_data, do_load_audit):
         """Audit a single subcloud."""
 
         # Retrieve the subcloud
@@ -326,13 +386,20 @@ class SubcloudAuditManager(manager.Manager):
                 subcloud_name,
                 availability_status=avail_status_current,
                 update_state_only=True)
-            self.audit_count = 0
 
-        if avail_to_set == consts.AVAILABILITY_ONLINE:
-            # If subcloud is online, get alarm summary and store in db,
+        # If subcloud is managed and online, audit additional resources
+        if (subcloud.management_state == consts.MANAGEMENT_MANAGED and
+                avail_to_set == consts.AVAILABILITY_ONLINE):
+            # Get alarm summary and store in db,
             if fm_client:
                 self.alarm_aggr.update_alarm_summary(subcloud_name, fm_client)
 
+            # If we have patch audit data, audit the subcloud
+            if patch_audit_data:
+                self.patch_audit.subcloud_patch_audit(subcloud_name,
+                                                      patch_audit_data,
+                                                      do_load_audit)
+
             # Audit openstack application in the subcloud
             if audit_openstack and sysinv_client:
                 self._audit_subcloud_openstack_app(

distributedcloud/dcmanager/manager/patch_audit_manager.py

@@ -1,300 +0,0 @@
-# Copyright 2017 Ericsson AB.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Copyright (c) 2017-2020 Wind River Systems, Inc.
-#
-# The right to copy, distribute, modify, or otherwise make use
-# of this software may be licensed only pursuant to the terms
-# of an applicable Wind River license agreement.
-#
-
-from keystoneauth1 import exceptions as keystone_exceptions
-from oslo_config import cfg
-from oslo_log import log as logging
-
-from dccommon.drivers.openstack import patching_v1
-from dccommon.drivers.openstack.patching_v1 import PatchingClient
-from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
-from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
-
-from dcorch.common import consts as dcorch_consts
-
-from dcmanager.common import consts
-from dcmanager.common import context
-from dcmanager.common.i18n import _
-from dcmanager.common import manager
-from dcmanager.db import api as db_api
-
-
-LOG = logging.getLogger(__name__)
-
-# By default the patch audit will only occur every five minutes.
-DEFAULT_PATCH_AUDIT_DELAY_SECONDS = 300
-
-
-class PatchAuditManager(manager.Manager):
-    """Manages tasks related to patch audits."""
-
-    def __init__(self, *args, **kwargs):
-        LOG.info(_('PatchAuditManager initialization...'))
-
-        super(PatchAuditManager, self).__init__(
-            service_name="patch_audit_manager")
-        self.context = context.get_admin_context()
-        self.subcloud_manager = kwargs['subcloud_manager']
-        # Wait 20 seconds before doing the first audit
-        self.wait_time_passed = DEFAULT_PATCH_AUDIT_DELAY_SECONDS - 25
-        self.audit_count = 0
-
-    # Used to force an audit on the next interval
-    _force_audit = False
-
-    @classmethod
-    def trigger_audit(cls):
-        """Trigger audit at next interval.
-
-        This can be called from outside the audit greenthread.
-        """
-        cls._force_audit = True
-
-    def periodic_patch_audit(self):
-        """Audit patch status of subclouds.
-
-        Audit normally happens every DEFAULT_PATCH_AUDIT_DELAY_SECONDS, but
-        can be forced to happen on the next audit interval by calling
-        trigger_audit.
-        """
-
-        do_audit = False
-
-        if PatchAuditManager._force_audit:
-            # Audit has been triggered.
-            do_audit = True
-        else:
-            # This won't be super accurate as we aren't woken up after exactly
-            # the interval seconds, but it is good enough for an audit.
-            self.wait_time_passed += cfg.CONF.scheduler.patch_audit_interval
-            if self.wait_time_passed >= DEFAULT_PATCH_AUDIT_DELAY_SECONDS:
-                do_audit = True
-
-        if do_audit:
-            self.wait_time_passed = 0
-            PatchAuditManager._force_audit = False
-            # Blanket catch all exceptions in the audit so that the audit
-            # does not die.
-            try:
-                self._periodic_patch_audit_loop()
-            except Exception as e:
-                LOG.exception(e)
-
-    def _update_subcloud_sync_status(self, sc_name, sc_endpoint_type, sc_status):
-        self.subcloud_manager.update_subcloud_endpoint_status(
-            self.context,
-            subcloud_name=sc_name,
-            endpoint_type=sc_endpoint_type,
-            sync_status=sc_status)
-
-    def _periodic_patch_audit_loop(self):
-        """Audit patch status of subclouds loop."""
-
-        # We are running in our own green thread here.
-        LOG.info('Triggered patch audit.')
-        self.audit_count += 1
-
-        try:
-            m_os_ks_client = OpenStackDriver(
-                region_name=consts.DEFAULT_REGION_NAME,
-                region_clients=None).keystone_client
-        except Exception:
-            LOG.warn('Failure initializing KeystoneClient, exiting audit.')
-            return
-
-        # First query RegionOne to determine what patches should be applied
-        # to the system.
-        patching_client = PatchingClient(
-            consts.DEFAULT_REGION_NAME, m_os_ks_client.session)
-        regionone_patches = patching_client.query()
-        LOG.debug("regionone_patches: %s" % regionone_patches)
-
-        # Get the active software version in RegionOne as it may be needed
-        # later for subcloud load audit.
-        sysinv_client = SysinvClient(
-            consts.DEFAULT_REGION_NAME, m_os_ks_client.session)
-        regionone_software_version = sysinv_client.get_system().software_version
-
-        # Build lists of patches that should be applied or committed in all
-        # subclouds, based on their state in RegionOne. Check repostate
-        # (not patchstate) as we only care if the patch has been applied to
-        # the repo (not whether it is installed on the hosts).
-        applied_patch_ids = list()
-        committed_patch_ids = list()
-        for patch_id in regionone_patches.keys():
-            if regionone_patches[patch_id]['repostate'] == \
-                    patching_v1.PATCH_STATE_APPLIED:
-                applied_patch_ids.append(patch_id)
-            elif regionone_patches[patch_id]['repostate'] == \
-                    patching_v1.PATCH_STATE_COMMITTED:
-                committed_patch_ids.append(patch_id)
-        LOG.debug("RegionOne applied_patch_ids: %s" % applied_patch_ids)
-        LOG.debug("RegionOne committed_patch_ids: %s" % committed_patch_ids)
-
-        # For each subcloud, check whether the patches match the target.
-        for subcloud in db_api.subcloud_get_all(self.context):
-            # Only audit patching on subclouds that are managed and online
-            if (subcloud.management_state != consts.MANAGEMENT_MANAGED or
-                    subcloud.availability_status !=
-                    consts.AVAILABILITY_ONLINE):
-                continue
-
-            try:
-                sc_os_client = OpenStackDriver(region_name=subcloud.name,
-                                               region_clients=None)
-            except (keystone_exceptions.EndpointNotFound,
-                    keystone_exceptions.ConnectFailure,
-                    keystone_exceptions.ConnectTimeout,
-                    IndexError):
-                # Since it takes some time to detect that a subcloud has gone
-                # offline, these errors are expected from time to time.
-                LOG.info("Identity endpoint for online subcloud %s not found."
-                         % subcloud.name)
-                continue
-
-            try:
-                patching_client = PatchingClient(
-                    subcloud.name, sc_os_client.keystone_client.session)
-            except keystone_exceptions.EndpointNotFound:
-                LOG.warn("Patching endpoint for online subcloud %s not found."
-                         % subcloud.name)
-                continue
-
-            try:
-                sysinv_client = SysinvClient(
-                    subcloud.name, sc_os_client.keystone_client.session)
-            except keystone_exceptions.EndpointNotFound:
-                LOG.warn("Sysinv endpoint for online subcloud %s not found."
-                         % subcloud.name)
-                continue
-
-            # Retrieve all the patches that are present in this subcloud.
-            try:
-                subcloud_patches = patching_client.query()
-                LOG.debug("Patches for subcloud %s: %s" %
-                          (subcloud.name, subcloud_patches))
-            except Exception:
-                LOG.warn('Cannot retrieve patches for subcloud: %s' %
-                         subcloud.name)
-                continue
-
-            # Determine which loads are present in this subcloud. During an
-            # upgrade, there will be more than one load installed.
-            installed_loads = list()
-            try:
-                loads = sysinv_client.get_loads()
-            except Exception:
-                LOG.warn('Cannot retrieve loads for subcloud: %s' %
-                         subcloud.name)
-                continue
-
-            for load in loads:
-                installed_loads.append(load.software_version)
-
-            out_of_sync = False
-
-            # Check that all patches in this subcloud are in the correct
-            # state, based on the state of the patch in RegionOne. For the
-            # subcloud, we use the patchstate because we care whether the
-            # patch is installed on the hosts.
-            for patch_id in subcloud_patches.keys():
-                if subcloud_patches[patch_id]['patchstate'] == \
-                        patching_v1.PATCH_STATE_APPLIED:
-                    if patch_id not in applied_patch_ids:
-                        if patch_id not in committed_patch_ids:
-                            LOG.debug("Patch %s should not be applied in %s" %
-                                      (patch_id, subcloud.name))
-                        else:
-                            LOG.debug("Patch %s should be committed in %s" %
-                                      (patch_id, subcloud.name))
-                        out_of_sync = True
-                elif subcloud_patches[patch_id]['patchstate'] == \
-                        patching_v1.PATCH_STATE_COMMITTED:
-                    if patch_id not in committed_patch_ids:
-                        LOG.warn("Patch %s should not be committed in %s" %
-                                 (patch_id, subcloud.name))
-                        out_of_sync = True
-                else:
-                    # In steady state, all patches should either be applied
-                    # or committed in each subcloud. Patches in other
-                    # states mean a sync is required.
-                    out_of_sync = True
-
-            # Check that all applied or committed patches in RegionOne are
-            # present in the subcloud.
-            for patch_id in applied_patch_ids:
-                if regionone_patches[patch_id]['sw_version'] in \
-                        installed_loads and patch_id not in subcloud_patches:
-                    LOG.debug("Patch %s missing from %s" %
-                              (patch_id, subcloud.name))
-                    out_of_sync = True
-            for patch_id in committed_patch_ids:
-                if regionone_patches[patch_id]['sw_version'] in \
-                        installed_loads and patch_id not in subcloud_patches:
-                    LOG.debug("Patch %s missing from %s" %
-                              (patch_id, subcloud.name))
-                    out_of_sync = True
-
-            if out_of_sync:
-                LOG.debug("Subcloud %s is out-of-sync for patching" %
-                          subcloud.name)
-                self._update_subcloud_sync_status(
-                    subcloud.name, dcorch_consts.ENDPOINT_TYPE_PATCHING,
-                    consts.SYNC_STATUS_OUT_OF_SYNC)
-            else:
-                LOG.debug("Subcloud %s is in-sync for patching" %
-                          subcloud.name)
-                self._update_subcloud_sync_status(
-                    subcloud.name, dcorch_consts.ENDPOINT_TYPE_PATCHING,
-                    consts.SYNC_STATUS_IN_SYNC)
-
-            # Check subcloud software version every other audit cycle
-            if self.audit_count % 2 != 0:
-                LOG.debug('Auditing load of subcloud %s' % subcloud.name)
-                try:
-                    upgrades = sysinv_client.get_upgrades()
-                except Exception:
-                    LOG.warn('Cannot retrieve upgrade info for subcloud: %s' %
-                             subcloud.name)
-                    continue
-
-                if not upgrades:
-                    # No upgrade in progress
-                    subcloud_software_version = \
-                        sysinv_client.get_system().software_version
-
-                    if subcloud_software_version == regionone_software_version:
-                        self._update_subcloud_sync_status(
-                            subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD,
-                            consts.SYNC_STATUS_IN_SYNC)
-                    else:
-                        self._update_subcloud_sync_status(
-                            subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD,
-                            consts.SYNC_STATUS_OUT_OF_SYNC)
-                else:
-                    # As upgrade is still in progress, set the subcloud load
-                    # status as out-of-sync.
-                    self._update_subcloud_sync_status(
-                        subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD,
-                        consts.SYNC_STATUS_OUT_OF_SYNC)
-
-        LOG.info('Patch audit completed.')

distributedcloud/dcmanager/manager/patch_orch_thread.py

@@ -39,7 +39,6 @@ from dcmanager.common import exceptions
 from dcmanager.common import scheduler
 from dcmanager.common import utils
 from dcmanager.db import api as db_api
-from dcmanager.manager.patch_audit_manager import PatchAuditManager
 
 LOG = logging.getLogger(__name__)
 
@@ -61,12 +60,14 @@ class PatchOrchThread(threading.Thread):
       database as it goes, with state and progress information.
     """
 
-    def __init__(self, strategy_lock):
+    def __init__(self, strategy_lock, audit_rpc_client):
         super(PatchOrchThread, self).__init__()
         self.context = context.get_admin_context()
         self._stop = threading.Event()
         # Used to protect strategy when an atomic read/update is required.
         self.strategy_lock = strategy_lock
+        # Used to notify dcmanager-audit to trigger a patch audit
+        self.audit_rpc_client = audit_rpc_client
         # Keeps track of greenthreads we create to do work.
         self.thread_group_manager = scheduler.ThreadGroupManager(
             thread_pool_size=100)
@@ -192,7 +193,7 @@ class PatchOrchThread(threading.Thread):
                             self.context, state=consts.SW_UPDATE_STATE_FAILED)
                     # Trigger patch audit to update the sync status for
                     # each subcloud.
-                    PatchAuditManager.trigger_audit()
+                    self.audit_rpc_client.trigger_patch_audit(self.context)
                     return
                 elif sw_update_strategy.stop_on_failure:
                     # We have been told to stop on failures
@@ -222,7 +223,7 @@ class PatchOrchThread(threading.Thread):
                     db_api.sw_update_strategy_update(
                         self.context, state=consts.SW_UPDATE_STATE_COMPLETE)
             # Trigger patch audit to update the sync status for each subcloud.
-            PatchAuditManager.trigger_audit()
+            self.audit_rpc_client.trigger_patch_audit(self.context)
             return
 
         if stop_after_stage is not None:
@@ -246,7 +247,7 @@ class PatchOrchThread(threading.Thread):
                         self.context, state=consts.SW_UPDATE_STATE_FAILED)
                 # Trigger patch audit to update the sync status for each
                 # subcloud.
-                PatchAuditManager.trigger_audit()
+                self.audit_rpc_client.trigger_patch_audit(self.context)
                 return
 
         LOG.debug("Working on stage %d" % current_stage)

distributedcloud/dcmanager/manager/service.py

@@ -10,7 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 #
-# Copyright (c) 2017 Wind River Systems, Inc.
+# Copyright (c) 2017-2020 Wind River Systems, Inc.
 #
 # The right to copy, distribute, modify, or otherwise make use
 # of this software may be licensed only pursuant to the terms
@@ -18,7 +18,6 @@
 #
 
 import six
-import time
 
 import functools
 from oslo_config import cfg
@@ -29,13 +28,13 @@ from oslo_utils import uuidutils
 
 from dcorch.common import consts as dcorch_consts
 
+from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client
 from dcmanager.common import consts
 from dcmanager.common import context
 from dcmanager.common import exceptions
 from dcmanager.common.i18n import _
 from dcmanager.common import messaging as rpc_messaging
 from dcmanager.common import scheduler
-from dcmanager.manager.patch_audit_manager import PatchAuditManager
 from dcmanager.manager.subcloud_manager import SubcloudManager
 from dcmanager.manager.sw_update_manager import SwUpdateManager
 
@@ -76,20 +75,15 @@ class DCManagerService(service.Service):
         # happens after the fork when spawning multiple worker processes
         self.engine_id = None
         self.TG = None
-        self.periodic_enable = cfg.CONF.scheduler.periodic_enable
         self.target = None
         self._rpc_server = None
         self.subcloud_manager = None
         self.sw_update_manager = None
-        self.patch_audit_manager = None
+        self.audit_rpc_client = None
 
     def init_tgm(self):
         self.TG = scheduler.ThreadGroupManager()
 
-    def init_audit_managers(self):
-        self.patch_audit_manager = PatchAuditManager(
-            subcloud_manager=self.subcloud_manager)
-
     def init_managers(self):
         self.subcloud_manager = SubcloudManager()
         self.sw_update_manager = SwUpdateManager()
@@ -101,26 +95,16 @@ class DCManagerService(service.Service):
         self.dcmanager_id = uuidutils.generate_uuid()
         self.init_tgm()
         self.init_managers()
-        self.init_audit_managers()
         target = oslo_messaging.Target(version=self.rpc_api_version,
                                        server=self.host,
                                        topic=self.topic)
         self.target = target
         self._rpc_server = rpc_messaging.get_rpc_server(self.target, self)
         self._rpc_server.start()
+        # Used to notify dcmanager-audit
+        self.audit_rpc_client = dcmanager_audit_rpc_client.ManagerAuditClient()
 
         super(DCManagerService, self).start()
-        if self.periodic_enable:
-            LOG.info("Adding periodic tasks for the manager to perform")
-            self.TG.add_timer(cfg.CONF.scheduler.patch_audit_interval,
-                              self.patch_audit, initial_delay=60)
-
-    def patch_audit(self):
-        # Audit patch status of all subclouds.
-        # Note this will run in a separate green thread
-        LOG.debug("Patch audit job started at: %s",
-                  time.strftime("%c"))
-        self.patch_audit_manager.periodic_patch_audit()
 
     @request_context
     def add_subcloud(self, context, payload):
@@ -147,7 +131,7 @@ class DCManagerService(service.Service):
         # If a subcloud has been set to the managed state, trigger the
         # patching audit so it can update the sync status ASAP.
         if management_state == consts.MANAGEMENT_MANAGED:
-            PatchAuditManager.trigger_audit()
+            self.audit_rpc_client.trigger_patch_audit(context)
 
         return subcloud
 
@@ -172,7 +156,7 @@ class DCManagerService(service.Service):
         # patching audit so it can update the sync status ASAP.
         if endpoint_type == dcorch_consts.ENDPOINT_TYPE_PATCHING and \
                 sync_status == consts.SYNC_STATUS_UNKNOWN:
-            PatchAuditManager.trigger_audit()
+            self.audit_rpc_client.trigger_patch_audit(context)
 
         return
 

distributedcloud/dcmanager/manager/sw_update_manager.py

@@ -23,6 +23,7 @@ import threading
 
 from oslo_log import log as logging
 
+from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client
 from dcmanager.common import consts
 from dcmanager.common import exceptions
 from dcmanager.common import manager
@@ -45,12 +46,17 @@ class SwUpdateManager(manager.Manager):
         # Used to protect strategies when an atomic read/update is required.
         self.strategy_lock = threading.Lock()
 
+        # Used to notify dcmanager-audit
+        self.audit_rpc_client = dcmanager_audit_rpc_client.ManagerAuditClient()
+
         # Start worker threads
         # - patch orchestration thread
-        self.patch_orch_thread = PatchOrchThread(self.strategy_lock)
+        self.patch_orch_thread = PatchOrchThread(self.strategy_lock,
+                                                 self.audit_rpc_client)
         self.patch_orch_thread.start()
         # - sw upgrade orchestration thread
-        self.sw_upgrade_orch_thread = SwUpgradeOrchThread(self.strategy_lock)
+        self.sw_upgrade_orch_thread = SwUpgradeOrchThread(self.strategy_lock,
+                                                          self.audit_rpc_client)
         self.sw_upgrade_orch_thread.start()
 
     def stop(self):

distributedcloud/dcmanager/manager/sw_upgrade_orch_thread.py

@@ -33,7 +33,6 @@ from dcmanager.common import context
 from dcmanager.common import exceptions
 from dcmanager.common import scheduler
 from dcmanager.db import api as db_api
-from dcmanager.manager.patch_audit_manager import PatchAuditManager
 
 LOG = logging.getLogger(__name__)
 
@@ -58,12 +57,14 @@ class SwUpgradeOrchThread(threading.Thread):
       database as it goes, with state and progress information.
     """
 
-    def __init__(self, strategy_lock):
+    def __init__(self, strategy_lock, audit_rpc_client):
         super(SwUpgradeOrchThread, self).__init__()
         self.context = context.get_admin_context()
         self._stop = threading.Event()
         # Used to protect strategy when an atomic read/update is required.
         self.strategy_lock = strategy_lock
+        # Used to notify dcmanager-audit to trigger a patch audit
+        self.audit_rpc_client = audit_rpc_client
         # Keeps track of greenthreads we create to do work.
         self.thread_group_manager = scheduler.ThreadGroupManager(
             thread_pool_size=100)
@@ -199,7 +200,7 @@ class SwUpgradeOrchThread(threading.Thread):
                             self.context, state=consts.SW_UPDATE_STATE_FAILED)
                     # Trigger audit to update the sync status for
                     # each subcloud.
-                    PatchAuditManager.trigger_audit()
+                    self.audit_rpc_client.trigger_patch_audit(self.context)
                     return
                 elif sw_update_strategy.stop_on_failure:
                     # We have been told to stop on failures
@@ -229,7 +230,7 @@ class SwUpgradeOrchThread(threading.Thread):
                     db_api.sw_update_strategy_update(
                         self.context, state=consts.SW_UPDATE_STATE_COMPLETE)
             # Trigger audit to update the sync status for each subcloud.
-            PatchAuditManager.trigger_audit()
+            self.audit_rpc_client.trigger_patch_audit(self.context)
             return
 
         if stop_after_stage is not None:
@@ -252,7 +253,7 @@ class SwUpgradeOrchThread(threading.Thread):
                     db_api.sw_update_strategy_update(
                         self.context, state=consts.SW_UPDATE_STATE_FAILED)
                 # Trigger audit to update the sync status for each subcloud.
-                PatchAuditManager.trigger_audit()
+                self.audit_rpc_client.trigger_patch_audit(self.context)
                 return
 
         LOG.info("Working on stage %d" % current_stage)

distributedcloud/dcmanager/tests/unit/audit/test_patch_audit_manager.py

@@ -24,32 +24,23 @@ from oslo_config import cfg
 import sys
 sys.modules['fm_core'] = mock.Mock()
 
+from dcmanager.audit import patch_audit
+from dcmanager.audit import subcloud_audit_manager
 from dcmanager.common import consts
-from dcmanager.manager import patch_audit_manager
-from dcmanager.manager import subcloud_manager
 from dcmanager.tests import base
 from dcmanager.tests import utils
 
 from dcorch.common import consts as dcorch_consts
-from dcorch.common import messaging as dcorch_messaging
 
 
 CONF = cfg.CONF
 
 
-class Subcloud(object):
-    def __init__(self, id, name, is_managed, is_online):
-        self.id = id
-        self.name = name
-        self.software_version = '17.07'
-        if is_managed:
-            self.management_state = consts.MANAGEMENT_MANAGED
-        else:
-            self.management_state = consts.MANAGEMENT_UNMANAGED
-        if is_online:
-            self.availability_status = consts.AVAILABILITY_ONLINE
-        else:
-            self.availability_status = consts.AVAILABILITY_OFFLINE
+class FakeDCManagerAPI(object):
+
+    def __init__(self):
+        self.update_subcloud_availability = mock.MagicMock()
+        self.update_subcloud_endpoint_status = mock.MagicMock()
 
 
 class Load(object):
@@ -240,105 +231,78 @@ class FakeSysinvClientOneLoadUpgradeInProgress(object):
         return self.system
 
 
-class TestAuditManager(base.DCManagerTestCase):
+class TestPatchAudit(base.DCManagerTestCase):
     def setUp(self):
-        super(TestAuditManager, self).setUp()
+        super(TestPatchAudit, self).setUp()
         self.ctxt = utils.dummy_context()
-        dcorch_messaging.setup("fake://", optional=True)
 
-    @mock.patch.object(patch_audit_manager, 'PatchingClient')
-    @mock.patch.object(patch_audit_manager, 'OpenStackDriver')
-    @mock.patch.object(patch_audit_manager, 'context')
-    def test_init(self, mock_context,
-                  mock_openstack_driver,
-                  mock_patching_client):
+        # Mock the DCManager API
+        self.fake_dcmanager_api = FakeDCManagerAPI()
+        p = mock.patch('dcmanager.rpc.client.ManagerClient')
+        self.mock_dcmanager_api = p.start()
+        self.mock_dcmanager_api.return_value = self.fake_dcmanager_api
+        self.addCleanup(p.stop)
+
+    def test_init(self):
+        pm = patch_audit.PatchAudit(self.ctxt,
+                                    self.fake_dcmanager_api)
+        self.assertIsNotNone(pm)
+        self.assertEqual(self.ctxt, pm.context)
+        self.assertEqual(self.fake_dcmanager_api, pm.dcmanager_rpc_client)
+
+    @mock.patch.object(patch_audit, 'SysinvClient')
+    @mock.patch.object(patch_audit, 'PatchingClient')
+    @mock.patch.object(patch_audit, 'OpenStackDriver')
+    @mock.patch.object(subcloud_audit_manager, 'context')
+    def test_periodic_patch_audit_in_sync(self, mock_context,
+                                          mock_openstack_driver,
+                                          mock_patching_client,
+                                          mock_sysinv_client):
         mock_context.get_admin_context.return_value = self.ctxt
-
-        sm = subcloud_manager.SubcloudManager()
-        am = patch_audit_manager.PatchAuditManager(subcloud_manager=sm)
-        self.assertIsNotNone(am)
-        self.assertEqual('patch_audit_manager', am.service_name)
-        self.assertEqual('localhost', am.host)
-        self.assertEqual(self.ctxt, am.context)
-
-    @mock.patch.object(patch_audit_manager, 'SysinvClient')
-    @mock.patch.object(patch_audit_manager, 'db_api')
-    @mock.patch.object(patch_audit_manager, 'PatchingClient')
-    @mock.patch.object(patch_audit_manager, 'OpenStackDriver')
-    @mock.patch.object(patch_audit_manager, 'context')
-    def test_periodic_patch_audit_in_sync(
-            self, mock_context,
-            mock_openstack_driver,
-            mock_patching_client,
-            mock_db_api,
-            mock_sysinv_client):
-        mock_context.get_admin_context.return_value = self.ctxt
-        mock_sm = mock.Mock()
-        am = patch_audit_manager.PatchAuditManager(subcloud_manager=mock_sm)
-
         mock_patching_client.side_effect = FakePatchingClientInSync
         mock_sysinv_client.side_effect = FakeSysinvClientOneLoad
-        fake_subcloud1 = Subcloud(1, 'subcloud1',
-                                  is_managed=True, is_online=True)
-        fake_subcloud2 = Subcloud(2, 'subcloud2',
-                                  is_managed=True, is_online=True)
-        mock_db_api.subcloud_get_all.return_value = [fake_subcloud1,
-                                                     fake_subcloud2]
 
-        am._periodic_patch_audit_loop()
-        expected_calls = [
-            mock.call(mock.ANY,
-                      subcloud_name='subcloud1',
-                      endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
-                      sync_status=consts.SYNC_STATUS_IN_SYNC),
-            mock.call(mock.ANY,
-                      subcloud_name='subcloud1',
-                      endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
-                      sync_status=consts.SYNC_STATUS_IN_SYNC),
-            mock.call(mock.ANY,
-                      subcloud_name='subcloud2',
-                      endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
-                      sync_status=consts.SYNC_STATUS_IN_SYNC),
-            mock.call(mock.ANY,
-                      subcloud_name='subcloud2',
-                      endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
-                      sync_status=consts.SYNC_STATUS_IN_SYNC),
-        ]
-        mock_sm.update_subcloud_endpoint_status.assert_has_calls(
-            expected_calls)
+        pm = patch_audit.PatchAudit(self.ctxt,
+                                    self.fake_dcmanager_api)
+        am = subcloud_audit_manager.SubcloudAuditManager()
+        am.patch_audit = pm
 
-    @mock.patch.object(patch_audit_manager, 'SysinvClient')
-    @mock.patch.object(patch_audit_manager, 'db_api')
-    @mock.patch.object(patch_audit_manager, 'PatchingClient')
-    @mock.patch.object(patch_audit_manager, 'OpenStackDriver')
-    @mock.patch.object(patch_audit_manager, 'context')
-    def test_periodic_patch_audit_out_of_sync(
-            self, mock_context,
-            mock_openstack_driver,
-            mock_patching_client,
-            mock_db_api,
-            mock_sysinv_client):
+        patch_audit_data, do_load_audit = am._get_patch_audit()
+        for name in ['subcloud1', 'subcloud2']:
+            pm.subcloud_patch_audit(name, patch_audit_data, do_load_audit)
+            expected_calls = [
+                mock.call(mock.ANY,
+                          subcloud_name=name,
+                          endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
+                          sync_status=consts.SYNC_STATUS_IN_SYNC),
+                mock.call(mock.ANY,
+                          subcloud_name=name,
+                          endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
+                          sync_status=consts.SYNC_STATUS_IN_SYNC)]
+            self.fake_dcmanager_api.update_subcloud_endpoint_status. \
+                assert_has_calls(expected_calls)
+
+    @mock.patch.object(patch_audit, 'SysinvClient')
+    @mock.patch.object(patch_audit, 'PatchingClient')
+    @mock.patch.object(patch_audit, 'OpenStackDriver')
+    @mock.patch.object(subcloud_audit_manager, 'context')
+    def test_periodic_patch_audit_out_of_sync(self, mock_context,
+                                              mock_openstack_driver,
+                                              mock_patching_client,
+                                              mock_sysinv_client):
         mock_context.get_admin_context.return_value = self.ctxt
-        mock_sm = mock.Mock()
-        am = patch_audit_manager.PatchAuditManager(
-            subcloud_manager=mock_sm)
+        pm = patch_audit.PatchAudit(self.ctxt,
+                                    self.fake_dcmanager_api)
+        am = subcloud_audit_manager.SubcloudAuditManager()
+        am.patch_audit = pm
 
         mock_patching_client.side_effect = FakePatchingClientOutOfSync
         mock_sysinv_client.side_effect = FakeSysinvClientOneLoad
-        fake_subcloud1 = Subcloud(1, 'subcloud1',
-                                  is_managed=True, is_online=True)
-        fake_subcloud2 = Subcloud(2, 'subcloud2',
-                                  is_managed=True, is_online=True)
-        fake_subcloud3 = Subcloud(3, 'subcloud3',
-                                  is_managed=True, is_online=True)
-        fake_subcloud4 = Subcloud(4, 'subcloud4',
-                                  is_managed=True, is_online=True)
-        mock_db_api.subcloud_get_all.return_value = [fake_subcloud1,
-                                                     fake_subcloud2,
-                                                     fake_subcloud3,
-                                                     fake_subcloud4]
 
-        am._periodic_patch_audit_loop()
+        patch_audit_data, do_load_audit = am._get_patch_audit()
+        for name in ['subcloud1', 'subcloud2', 'subcloud3', 'subcloud4']:
+            pm.subcloud_patch_audit(name, patch_audit_data, do_load_audit)
+
         expected_calls = [
             mock.call(mock.ANY,
                       subcloud_name='subcloud1',
@@ -372,109 +336,64 @@ class TestAuditManager(base.DCManagerTestCase):
                       subcloud_name='subcloud4',
                       endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
                       sync_status=consts.SYNC_STATUS_IN_SYNC),
-        ]
-        mock_sm.update_subcloud_endpoint_status.assert_has_calls(
-            expected_calls)
+            ]
 
-    @mock.patch.object(patch_audit_manager, 'SysinvClient')
-    @mock.patch.object(patch_audit_manager, 'db_api')
-    @mock.patch.object(patch_audit_manager, 'PatchingClient')
-    @mock.patch.object(patch_audit_manager, 'OpenStackDriver')
-    @mock.patch.object(patch_audit_manager, 'context')
-    def test_periodic_patch_audit_ignore_unmanaged_or_offline(
-            self, mock_context,
-            mock_openstack_driver,
-            mock_patching_client,
-            mock_db_api,
-            mock_sysinv_client):
+        self.fake_dcmanager_api.update_subcloud_endpoint_status.\
+            assert_has_calls(expected_calls)
+
+    @mock.patch.object(patch_audit, 'SysinvClient')
+    @mock.patch.object(patch_audit, 'PatchingClient')
+    @mock.patch.object(patch_audit, 'OpenStackDriver')
+    @mock.patch.object(subcloud_audit_manager, 'context')
+    def test_periodic_patch_audit_extra_patches(self, mock_context,
+                                                mock_openstack_driver,
+                                                mock_patching_client,
+                                                mock_sysinv_client):
         mock_context.get_admin_context.return_value = self.ctxt
-        mock_sm = mock.Mock()
-        am = patch_audit_manager.PatchAuditManager(
-            subcloud_manager=mock_sm)
-
-        mock_patching_client.side_effect = FakePatchingClientOutOfSync
-        fake_subcloud1 = Subcloud(1, 'subcloud1',
-                                  is_managed=False, is_online=True)
-        fake_subcloud2 = Subcloud(2, 'subcloud2',
-                                  is_managed=True, is_online=False)
-        mock_db_api.subcloud_get_all.return_value = [fake_subcloud1,
-                                                     fake_subcloud2]
-
-        am._periodic_patch_audit_loop()
-        mock_sm.update_subcloud_endpoint_status.assert_not_called()
-
-    @mock.patch.object(patch_audit_manager, 'SysinvClient')
-    @mock.patch.object(patch_audit_manager, 'db_api')
-    @mock.patch.object(patch_audit_manager, 'PatchingClient')
-    @mock.patch.object(patch_audit_manager, 'OpenStackDriver')
-    @mock.patch.object(patch_audit_manager, 'context')
-    def test_periodic_patch_audit_extra_patches(
-            self, mock_context,
-            mock_openstack_driver,
-            mock_patching_client,
-            mock_db_api,
-            mock_sysinv_client):
-        mock_context.get_admin_context.return_value = self.ctxt
-        mock_sm = mock.Mock()
-        am = patch_audit_manager.PatchAuditManager(
-            subcloud_manager=mock_sm)
+        pm = patch_audit.PatchAudit(self.ctxt,
+                                    self.fake_dcmanager_api)
+        am = subcloud_audit_manager.SubcloudAuditManager()
+        am.patch_audit = pm
 
         mock_patching_client.side_effect = FakePatchingClientExtraPatches
         mock_sysinv_client.side_effect = FakeSysinvClientOneLoad
-        fake_subcloud1 = Subcloud(1, 'subcloud1',
-                                  is_managed=True, is_online=True)
-        fake_subcloud2 = Subcloud(2, 'subcloud2',
-                                  is_managed=True, is_online=True)
-        mock_db_api.subcloud_get_all.return_value = [fake_subcloud1,
-                                                     fake_subcloud2]
 
-        am._periodic_patch_audit_loop()
-        expected_calls = [
-            mock.call(mock.ANY,
-                      subcloud_name='subcloud1',
-                      endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
-                      sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
-            mock.call(mock.ANY,
-                      subcloud_name='subcloud1',
-                      endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
-                      sync_status=consts.SYNC_STATUS_IN_SYNC),
-            mock.call(mock.ANY,
-                      subcloud_name='subcloud2',
-                      endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
-                      sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
-            mock.call(mock.ANY,
-                      subcloud_name='subcloud2',
-                      endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
-                      sync_status=consts.SYNC_STATUS_IN_SYNC),
-        ]
-        mock_sm.update_subcloud_endpoint_status.assert_has_calls(
-            expected_calls)
+        patch_audit_data, do_load_audit = am._get_patch_audit()
+        for name in ['subcloud1', 'subcloud2']:
+            pm.subcloud_patch_audit(name, patch_audit_data, do_load_audit)
+            expected_calls = [
+                mock.call(mock.ANY,
+                          subcloud_name=name,
+                          endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
+                          sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
+                mock.call(mock.ANY,
+                          subcloud_name=name,
+                          endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
+                          sync_status=consts.SYNC_STATUS_IN_SYNC)]
+            self.fake_dcmanager_api.update_subcloud_endpoint_status.\
+                assert_has_calls(expected_calls)
 
-    @mock.patch.object(patch_audit_manager, 'SysinvClient')
-    @mock.patch.object(patch_audit_manager, 'db_api')
-    @mock.patch.object(patch_audit_manager, 'PatchingClient')
-    @mock.patch.object(patch_audit_manager, 'OpenStackDriver')
-    @mock.patch.object(patch_audit_manager, 'context')
+    @mock.patch.object(patch_audit, 'SysinvClient')
+    @mock.patch.object(patch_audit, 'PatchingClient')
+    @mock.patch.object(patch_audit, 'OpenStackDriver')
+    @mock.patch.object(subcloud_audit_manager, 'context')
     def test_periodic_patch_audit_unmatched_software_version(
             self, mock_context,
             mock_openstack_driver,
             mock_patching_client,
-            mock_db_api,
             mock_sysinv_client):
         mock_context.get_admin_context.return_value = self.ctxt
-        mock_sm = mock.Mock()
-        am = patch_audit_manager.PatchAuditManager(subcloud_manager=mock_sm)
-
+        pm = patch_audit.PatchAudit(self.ctxt,
+                                    self.fake_dcmanager_api)
+        am = subcloud_audit_manager.SubcloudAuditManager()
+        am.patch_audit = pm
         mock_patching_client.side_effect = FakePatchingClientInSync
         mock_sysinv_client.side_effect = FakeSysinvClientOneLoadUnmatchedSoftwareVersion
-        fake_subcloud1 = Subcloud(1, 'subcloud1',
-                                  is_managed=True, is_online=True)
-        fake_subcloud2 = Subcloud(2, 'subcloud2',
-                                  is_managed=True, is_online=True)
-        mock_db_api.subcloud_get_all.return_value = [fake_subcloud1,
-                                                     fake_subcloud2]
 
-        am._periodic_patch_audit_loop()
+        patch_audit_data, do_load_audit = am._get_patch_audit()
+        for name in ['subcloud1', 'subcloud2']:
+            pm.subcloud_patch_audit(name, patch_audit_data, do_load_audit)
+
         expected_calls = [
             mock.call(mock.ANY,
                       subcloud_name='subcloud1',
@@ -493,34 +412,30 @@ class TestAuditManager(base.DCManagerTestCase):
                       endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
                       sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
         ]
-        mock_sm.update_subcloud_endpoint_status.assert_has_calls(
-            expected_calls)
+        self.fake_dcmanager_api.update_subcloud_endpoint_status.\
+            assert_has_calls(expected_calls)
 
-    @mock.patch.object(patch_audit_manager, 'SysinvClient')
-    @mock.patch.object(patch_audit_manager, 'db_api')
-    @mock.patch.object(patch_audit_manager, 'PatchingClient')
-    @mock.patch.object(patch_audit_manager, 'OpenStackDriver')
-    @mock.patch.object(patch_audit_manager, 'context')
+    @mock.patch.object(patch_audit, 'SysinvClient')
+    @mock.patch.object(patch_audit, 'PatchingClient')
+    @mock.patch.object(patch_audit, 'OpenStackDriver')
+    @mock.patch.object(subcloud_audit_manager, 'context')
     def test_periodic_patch_audit_upgrade_in_progress(
             self, mock_context,
             mock_openstack_driver,
             mock_patching_client,
-            mock_db_api,
             mock_sysinv_client):
         mock_context.get_admin_context.return_value = self.ctxt
-        mock_sm = mock.Mock()
-        am = patch_audit_manager.PatchAuditManager(subcloud_manager=mock_sm)
-
+        pm = patch_audit.PatchAudit(self.ctxt,
+                                    self.fake_dcmanager_api)
+        am = subcloud_audit_manager.SubcloudAuditManager()
+        am.patch_audit = pm
         mock_patching_client.side_effect = FakePatchingClientInSync
         mock_sysinv_client.side_effect = FakeSysinvClientOneLoadUpgradeInProgress
-        fake_subcloud1 = Subcloud(1, 'subcloud1',
-                                  is_managed=True, is_online=True)
-        fake_subcloud2 = Subcloud(2, 'subcloud2',
-                                  is_managed=True, is_online=True)
-        mock_db_api.subcloud_get_all.return_value = [fake_subcloud1,
-                                                     fake_subcloud2]
 
-        am._periodic_patch_audit_loop()
+        patch_audit_data, do_load_audit = am._get_patch_audit()
+        for name in ['subcloud1', 'subcloud2']:
+            pm.subcloud_patch_audit(name, patch_audit_data, do_load_audit)
+
         expected_calls = [
             mock.call(mock.ANY,
                       subcloud_name='subcloud1',
@@ -539,5 +454,5 @@ class TestAuditManager(base.DCManagerTestCase):
                       endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
                       sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
         ]
-        mock_sm.update_subcloud_endpoint_status.assert_has_calls(
-            expected_calls)
+        self.fake_dcmanager_api.update_subcloud_endpoint_status.\
+            assert_has_calls(expected_calls)

distributedcloud/dcmanager/tests/unit/audit/test_subcloud_audit_manager.py

@@ -27,7 +27,6 @@ from dccommon import consts as dccommon_consts
 from dcmanager.audit import subcloud_audit_manager
 from dcmanager.common import consts
 from dcmanager.db.sqlalchemy import api as db_api
-# from dcmanager.manager import subcloud_manager
 
 from dcmanager.tests import base
 
@@ -45,6 +44,13 @@ class FakeAlarmAggregation(object):
         self.update_alarm_summary = mock.MagicMock()
 
 
+class FakePatchAudit(object):
+
+    def __init__(self):
+        self.subcloud_patch_audit = mock.MagicMock()
+        self.get_regionone_audit_data = mock.MagicMock()
+
+
 class FakeServiceGroup(object):
     def __init__(self, status, desired_state, service_group_name, uuid,
                  node_name, state, condition, name):
@@ -230,6 +236,15 @@ class TestAuditManager(base.DCManagerTestCase):
             self.fake_alarm_aggr
         self.addCleanup(p.stop)
 
+        # Mock patch audit
+        self.fake_patch_audit = FakePatchAudit()
+        p = mock.patch.object(subcloud_audit_manager,
+                              'patch_audit')
+        self.mock_patch_audit = p.start()
+        self.mock_patch_audit.PatchAudit.return_value = \
+            self.fake_patch_audit
+        self.addCleanup(p.stop)
+
     @staticmethod
     def create_subcloud_static(ctxt, **kwargs):
         values = {
@@ -260,16 +275,24 @@ class TestAuditManager(base.DCManagerTestCase):
         am = subcloud_audit_manager.SubcloudAuditManager()
         am._periodic_subcloud_audit_loop()
 
-    def test_audit_subcloud_online(self):
+    def test_audit_subcloud_online_managed(self):
 
         subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
         self.assertIsNotNone(subcloud)
 
+        # Set the subcloud to managed
+        db_api.subcloud_update(
+            self.ctx, subcloud.id,
+            management_state='managed')
+
         am = subcloud_audit_manager.SubcloudAuditManager()
 
         # Audit the subcloud
+        patch_audit_data, do_load_audit = am._get_patch_audit()
         am._audit_subcloud(subcloud.name, update_subcloud_state=False,
-                           audit_openstack=False)
+                           audit_openstack=False,
+                           patch_audit_data=patch_audit_data,
+                           do_load_audit=do_load_audit)
 
         # Verify the subcloud was set to online
         self.fake_dcmanager_api.update_subcloud_availability.assert_called_with(
@@ -284,6 +307,39 @@ class TestAuditManager(base.DCManagerTestCase):
         self.fake_alarm_aggr.update_alarm_summary.assert_called_with(
             subcloud.name, self.fake_openstack_client.fm_client)
 
+        # Verify patch audit is called
+        self.fake_patch_audit.subcloud_patch_audit.assert_called_with(
+            subcloud.name, patch_audit_data, do_load_audit)
+
+    def test_audit_subcloud_online_unmanaged(self):
+
+        subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
+        self.assertIsNotNone(subcloud)
+
+        am = subcloud_audit_manager.SubcloudAuditManager()
+
+        # Audit the subcloud
+        patch_audit_data, do_load_audit = am._get_patch_audit()
+        am._audit_subcloud(subcloud.name, update_subcloud_state=False,
+                           audit_openstack=False,
+                           patch_audit_data=patch_audit_data,
+                           do_load_audit=do_load_audit)
+
+        # Verify the subcloud was set to online
+        self.fake_dcmanager_api.update_subcloud_availability.assert_called_with(
+            mock.ANY, subcloud.name, consts.AVAILABILITY_ONLINE,
+            False, 0)
+
+        # Verify the openstack endpoints were not added
+        self.fake_dcmanager_api.update_subcloud_sync_endpoint_type.\
+            assert_not_called()
+
+        # Verify alarm update is not called
+        self.fake_alarm_aggr.update_alarm_summary.assert_not_called()
+
+        # Verify patch audit is not called
+        self.fake_patch_audit.subcloud_patch_audit.assert_not_called()
+
     def test_audit_subcloud_online_no_change(self):
 
         subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
@@ -298,7 +354,8 @@ class TestAuditManager(base.DCManagerTestCase):
 
         # Audit the subcloud
         am._audit_subcloud(subcloud.name, update_subcloud_state=False,
-                           audit_openstack=False)
+                           audit_openstack=False, patch_audit_data=None,
+                           do_load_audit=False)
 
         # Verify the subcloud state was not updated
         self.fake_dcmanager_api.update_subcloud_availability.\
@@ -308,9 +365,11 @@ class TestAuditManager(base.DCManagerTestCase):
         self.fake_dcmanager_api.update_subcloud_sync_endpoint_type.\
             assert_not_called()
 
-        # Verify alarm update is called
-        self.fake_alarm_aggr.update_alarm_summary.assert_called_with(
-            'subcloud1', self.fake_openstack_client.fm_client)
+        # Verify alarm update is not called
+        self.fake_alarm_aggr.update_alarm_summary.assert_not_called()
+
+        # Verify patch audit is not called
+        self.fake_patch_audit.subcloud_patch_audit.assert_not_called()
 
     def test_audit_subcloud_online_no_change_force_update(self):
 
@@ -326,7 +385,8 @@ class TestAuditManager(base.DCManagerTestCase):
 
         # Audit the subcloud and force a state update
         am._audit_subcloud(subcloud.name, update_subcloud_state=True,
-                           audit_openstack=False)
+                           audit_openstack=False, patch_audit_data=None,
+                           do_load_audit=False)
 
         # Verify the subcloud state was updated even though no change
         self.fake_dcmanager_api.update_subcloud_availability.assert_called_with(
@@ -337,9 +397,11 @@ class TestAuditManager(base.DCManagerTestCase):
         self.fake_dcmanager_api.update_subcloud_sync_endpoint_type.\
             assert_not_called()
 
-        # Verify alarm update is called
-        self.fake_alarm_aggr.update_alarm_summary.assert_called_with(
-            'subcloud1', self.fake_openstack_client.fm_client)
+        # Verify alarm update is not called
+        self.fake_alarm_aggr.update_alarm_summary.assert_not_called()
+
+        # Verify patch audit is not called
+        self.fake_patch_audit.subcloud_patch_audit.assert_not_called()
 
     def test_audit_subcloud_go_offline(self):
 
@@ -348,9 +410,10 @@ class TestAuditManager(base.DCManagerTestCase):
 
         am = subcloud_audit_manager.SubcloudAuditManager()
 
-        # Set the subcloud to online
+        # Set the subcloud to managed/online
         db_api.subcloud_update(
             self.ctx, subcloud.id,
+            management_state='managed',
             availability_status=consts.AVAILABILITY_ONLINE)
 
         # Mark a service group as inactive
@@ -360,8 +423,11 @@ class TestAuditManager(base.DCManagerTestCase):
             get_service_groups_result[3].state = 'inactive'
 
         # Audit the subcloud
+        patch_audit_data, do_load_audit = am._get_patch_audit()
         am._audit_subcloud(subcloud.name, update_subcloud_state=False,
-                           audit_openstack=False)
+                           audit_openstack=False,
+                           patch_audit_data=patch_audit_data,
+                           do_load_audit=do_load_audit)
 
         # Verify the audit fail count was updated
         audit_fail_count = 1
@@ -374,7 +440,9 @@ class TestAuditManager(base.DCManagerTestCase):
 
         # Audit the subcloud again
         am._audit_subcloud(subcloud.name, update_subcloud_state=False,
-                           audit_openstack=False)
+                           audit_openstack=False,
+                           patch_audit_data=patch_audit_data,
+                           do_load_audit=do_load_audit)
         audit_fail_count = audit_fail_count + 1
 
         # Verify the subcloud was set to offline
@@ -387,6 +455,10 @@ class TestAuditManager(base.DCManagerTestCase):
         self.fake_alarm_aggr.update_alarm_summary.assert_called_once_with(
             subcloud.name, self.fake_openstack_client.fm_client)
 
+        # Verify patch audit is called only once
+        self.fake_patch_audit.subcloud_patch_audit.assert_called_once_with(
+            subcloud.name, mock.ANY, True)
+
     def test_audit_subcloud_offline_no_change(self):
         subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
         self.assertIsNotNone(subcloud)
@@ -403,8 +475,11 @@ class TestAuditManager(base.DCManagerTestCase):
             get_service_groups_result[3].state = 'inactive'
 
         # Audit the subcloud
+        patch_audit_data, do_load_audit = am._get_patch_audit()
         am._audit_subcloud(subcloud.name, update_subcloud_state=False,
-                           audit_openstack=True)
+                           audit_openstack=True,
+                           patch_audit_data=patch_audit_data,
+                           do_load_audit=do_load_audit)
 
         # Verify the subcloud state was not updated
         self.fake_dcmanager_api.update_subcloud_availability.\
@@ -417,6 +492,9 @@ class TestAuditManager(base.DCManagerTestCase):
         # Verify alarm update is not called
         self.fake_alarm_aggr.update_alarm_summary.assert_not_called()
 
+        # Verify patch audit is not called
+        self.fake_patch_audit.subcloud_patch_audit.assert_not_called()
+
     def test_audit_subcloud_online_with_openstack_installed(self):
 
         subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
@@ -427,11 +505,13 @@ class TestAuditManager(base.DCManagerTestCase):
         # Set the subcloud to online
         db_api.subcloud_update(
             self.ctx, subcloud.id,
+            management_state='managed',
             availability_status=consts.AVAILABILITY_ONLINE)
 
         # Audit the subcloud
         am._audit_subcloud(subcloud.name, update_subcloud_state=False,
-                           audit_openstack=True)
+                           audit_openstack=True, patch_audit_data=None,
+                           do_load_audit=False)
 
         # Verify the subcloud state was not updated
         self.fake_dcmanager_api.update_subcloud_availability.\
@@ -447,6 +527,9 @@ class TestAuditManager(base.DCManagerTestCase):
         self.fake_alarm_aggr.update_alarm_summary.assert_called_once_with(
             'subcloud1', self.fake_openstack_client.fm_client)
 
+        # Verify patch audit is not called
+        self.fake_patch_audit.subcloud_patch_audit.assert_not_called()
+
     def test_audit_subcloud_online_with_openstack_removed(self):
 
         subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
@@ -457,6 +540,7 @@ class TestAuditManager(base.DCManagerTestCase):
         # Set the subcloud to online and openstack installed
         db_api.subcloud_update(
             self.ctx, subcloud.id,
+            management_state='managed',
             availability_status=consts.AVAILABILITY_ONLINE,
             openstack_installed=True)
 
@@ -465,7 +549,8 @@ class TestAuditManager(base.DCManagerTestCase):
 
         # Audit the subcloud
         am._audit_subcloud(subcloud.name, update_subcloud_state=False,
-                           audit_openstack=True)
+                           audit_openstack=True, patch_audit_data=None,
+                           do_load_audit=False)
 
         # Verify the subcloud state was not updated
         self.fake_dcmanager_api.update_subcloud_availability.\
@@ -480,6 +565,9 @@ class TestAuditManager(base.DCManagerTestCase):
         self.fake_alarm_aggr.update_alarm_summary.assert_called_once_with(
             'subcloud1', self.fake_openstack_client.fm_client)
 
+        # Verify patch audit is not called
+        self.fake_patch_audit.subcloud_patch_audit.assert_not_called()
+
     def test_audit_subcloud_online_with_openstack_inactive(self):
 
         subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
@@ -490,6 +578,7 @@ class TestAuditManager(base.DCManagerTestCase):
         # Set the subcloud to online and openstack installed
         db_api.subcloud_update(
             self.ctx, subcloud.id,
+            management_state='managed',
             availability_status=consts.AVAILABILITY_ONLINE,
             openstack_installed=True)
 
@@ -498,7 +587,8 @@ class TestAuditManager(base.DCManagerTestCase):
 
         # Audit the subcloud
         am._audit_subcloud(subcloud.name, update_subcloud_state=False,
-                           audit_openstack=True)
+                           audit_openstack=True, patch_audit_data=None,
+                           do_load_audit=False)
 
         # Verify the subcloud state was not updated
         self.fake_dcmanager_api.update_subcloud_availability.\
@@ -512,3 +602,6 @@ class TestAuditManager(base.DCManagerTestCase):
         # Verify alarm update is called
         self.fake_alarm_aggr.update_alarm_summary.assert_called_once_with(
             'subcloud1', self.fake_openstack_client.fm_client)
+
+        # Verify patch audit is not called
+        self.fake_patch_audit.subcloud_patch_audit.assert_not_called()

distributedcloud/dcmanager/tests/unit/manager/test_service.py

@@ -33,6 +33,12 @@ FAKE_USER = utils.UUID1
 FAKE_JOB = utils.UUID2
 
 
+class FakeDCManagerAuditAPI(object):
+
+    def __init__(self):
+        self.trigger_patch_audit = mock.MagicMock()
+
+
 class TestDCManagerService(base.DCManagerTestCase):
     def setUp(self):
         super(TestDCManagerService, self).setUp()
@@ -46,20 +52,22 @@ class TestDCManagerService(base.DCManagerTestCase):
         self.user_id = FAKE_USER
         self.job_id = FAKE_JOB
 
+        # Mock the DCManager Audit API
+        self.fake_dcmanager_audit_api = FakeDCManagerAuditAPI()
+        p = mock.patch('dcmanager.audit.rpcapi.ManagerAuditClient')
+        self.mock_dcmanager_audit_api = p.start()
+        self.mock_dcmanager_audit_api.return_value = \
+            self.fake_dcmanager_audit_api
+        self.addCleanup(p.stop)
+
     def test_init(self):
         self.assertEqual(self.service_obj.host, 'localhost')
         self.assertEqual(self.service_obj.topic, 'dcmanager')
-        self.assertEqual(self.service_obj.periodic_enable,
-                         CONF.scheduler.periodic_enable)
 
     def test_init_tgm(self):
         self.service_obj.init_tgm()
         self.assertIsNotNone(self.service_obj.TG)
 
-    def test_init_audit_managers(self):
-        self.service_obj.init_audit_managers()
-        self.assertIsNotNone(self.service_obj.patch_audit_manager)
-
     @mock.patch.object(service, 'SwUpdateManager')
     @mock.patch.object(service, 'SubcloudManager')
     def test_init_managers(self, mock_subcloud_manager,
@@ -78,14 +86,6 @@ class TestDCManagerService(base.DCManagerTestCase):
             self.service_obj.target, self.service_obj)
         mock_rpc.get_rpc_server().start.assert_called_once_with()
 
-    @mock.patch.object(service, 'PatchAuditManager')
-    def test_periodic_audit_patches(self, mock_patch_audit_manager):
-        self.service_obj.init_tgm()
-        self.service_obj.init_audit_managers()
-        self.service_obj.patch_audit()
-        mock_patch_audit_manager().periodic_patch_audit.\
-            assert_called_once_with()
-
     @mock.patch.object(service, 'SwUpdateManager')
     @mock.patch.object(service, 'SubcloudManager')
     def test_add_subcloud(self, mock_subcloud_manager, mock_sw_update_manager):

distributedcloud/dcmanager/tests/unit/manager/test_sw_update_manager.py

@@ -346,6 +346,12 @@ class FakeOrchThread(object):
         self.start = mock.MagicMock()
 
 
+class FakeDCManagerAuditAPI(object):
+
+    def __init__(self):
+        self.trigger_patch_audit = mock.MagicMock()
+
+
 class TestSwUpdateManager(base.DCManagerTestCase):
     def setUp(self):
         super(TestSwUpdateManager, self).setUp()
@@ -364,6 +370,14 @@ class TestSwUpdateManager(base.DCManagerTestCase):
             self.fake_sw_upgrade_orch_thread
         self.addCleanup(p.stop)
 
+        # Mock the dcmanager audit API
+        self.fake_dcmanager_audit_api = FakeDCManagerAuditAPI()
+        p = mock.patch('dcmanager.audit.rpcapi.ManagerAuditClient')
+        self.mock_dcmanager_audit_api = p.start()
+        self.mock_dcmanager_audit_api.return_value = \
+            self.fake_dcmanager_audit_api
+        self.addCleanup(p.stop)
+
     @mock.patch.object(sw_update_manager, 'PatchOrchThread')
     def test_init(self, mock_patch_orch_thread):
         um = sw_update_manager.SwUpdateManager()
@@ -694,7 +708,8 @@ class TestSwUpdateManager(base.DCManagerTestCase):
         FakePatchingClientOutOfSync.upload = mock.Mock()
         sw_update_manager.PatchOrchThread.stopped = lambda x: False
         mock_strategy_lock = mock.Mock()
-        pot = sw_update_manager.PatchOrchThread(mock_strategy_lock)
+        pot = sw_update_manager.PatchOrchThread(mock_strategy_lock,
+                                                self.fake_dcmanager_audit_api)
         pot.get_ks_client = mock.Mock()
         pot.update_subcloud_patches(fake_strategy_step)
 
@@ -740,7 +755,8 @@ class TestSwUpdateManager(base.DCManagerTestCase):
         FakePatchingClientOutOfSync.upload = mock.Mock()
         sw_update_manager.PatchOrchThread.stopped = lambda x: False
         mock_strategy_lock = mock.Mock()
-        pot = sw_update_manager.PatchOrchThread(mock_strategy_lock)
+        pot = sw_update_manager.PatchOrchThread(mock_strategy_lock,
+                                                self.fake_dcmanager_audit_api)
         pot.get_ks_client = mock.Mock()
         pot.update_subcloud_patches(fake_strategy_step)
 
@@ -776,7 +792,8 @@ class TestSwUpdateManager(base.DCManagerTestCase):
         FakePatchingClientOutOfSync.upload = mock.Mock()
         sw_update_manager.PatchOrchThread.stopped = lambda x: False
         mock_strategy_lock = mock.Mock()
-        pot = sw_update_manager.PatchOrchThread(mock_strategy_lock)
+        pot = sw_update_manager.PatchOrchThread(mock_strategy_lock,
+                                                self.fake_dcmanager_audit_api)
         pot.get_ks_client = mock.Mock()
         pot.update_subcloud_patches(fake_strategy_step)
 
@@ -809,7 +826,8 @@ class TestSwUpdateManager(base.DCManagerTestCase):
         FakePatchingClientFinish.commit = mock.Mock()
         sw_update_manager.PatchOrchThread.stopped = lambda x: False
         mock_strategy_lock = mock.Mock()
-        pot = sw_update_manager.PatchOrchThread(mock_strategy_lock)
+        pot = sw_update_manager.PatchOrchThread(mock_strategy_lock,
+                                                self.fake_dcmanager_audit_api)
         pot.get_ks_client = mock.Mock()
         pot.finish(fake_strategy_step)
 

distributedcloud/dcmanager/tests/unit/manager/test_sw_upgrade.py

@@ -115,7 +115,9 @@ class TestSwUpgrade(base.DCManagerTestCase):
     def setup_upgrade_worker(self):
         sw_update_manager.SwUpgradeOrchThread.stopped = lambda x: False
         mock_strategy_lock = mock.Mock()
-        worker = sw_update_manager.SwUpgradeOrchThread(mock_strategy_lock)
+        mock_dcmanager_audit_api = mock.Mock()
+        worker = sw_update_manager.SwUpgradeOrchThread(mock_strategy_lock,
+                                                       mock_dcmanager_audit_api)
         worker.get_ks_client = mock.Mock()
         return worker