Enhance logging for dcaudit and dcorch

To improve the logged information during dcmanager audit and avoid
unnecessary logging in dcorch and audit, this commit adjusts log
levels between DEBUG and INFO.
Additionally, a specific rule is added for state.log in logrotate to
retain larger files compared to the rest of the dcmanager logs.

It was also seen that at the end of the dcorch audit process, the
sync_request was being updated to 'requested' for the subcloud and
endpoint being audited. This commit enhances this logic by updating
this state just for the cases in which there are sync jobs to be run.

Test Plan:
PASS: Verify that during dcorch audit, if there are no sync jobs to be
      run, the sync_request field is not updated.
PASS: Verify that during dcorch audit, if there are sync jobs, the
      sync_request field is updated to 'requested'. Also verify that
      in the next sync round, that subcloud/enpdoint is synched.
PASS: Verify that state.log rotates when it reaches the configured
      size of 40MB.
PASS: Verif that other dcmanager logs follow the previous rotation
      policy.

Story: 2011106
Task: 51424

Change-Id: Icfa3eb9bd692e1e22f8940c904e7ef373260589d
Signed-off-by: Enzo Candotti <enzo.candotti@windriver.com>

distributedcloud/dcmanager/audit/patch_audit.py

@@ -43,12 +43,14 @@ class PatchAudit(object):
         return PatchAuditData()
 
     def subcloud_patch_audit(self, keystone_client, subcloud):
-        LOG.info("Triggered patch audit for: %s." % subcloud.name)
 
         # NOTE(nicodemos): Patch audit not supported for 24.09 subcloud
         if subcloud.software_version == SW_VERSION:
             return consts.SYNC_STATUS_NOT_AVAILABLE
 
+        # Log this just for N-1 subclouds, before the has_usm_service external call
+        LOG.info("Triggered patch audit for: %s." % subcloud.name)
+
         # NOTE(nicodemos): If the subcloud is on the 22.12 release with USM enabled,
         # skip the patch audit.
         if utils.has_usm_service(subcloud.region_name, keystone_client):

distributedcloud/dcmanager/audit/subcloud_audit_manager.py

@@ -608,7 +608,7 @@ class SubcloudAuditManager(manager.Manager):
                     software_audit_data,
                     use_cache,
                 )
-                LOG.debug(
+                LOG.info(
                     "Sent subcloud audit request message for subclouds: %s"
                     % subcloud_ids
                 )
@@ -626,7 +626,7 @@ class SubcloudAuditManager(manager.Manager):
                 software_audit_data,
                 use_cache,
             )
-            LOG.debug(
+            LOG.info(
                 "Sent final subcloud audit request message for subclouds: %s"
                 % subcloud_ids
             )

distributedcloud/dcorch/engine/sync_services/identity.py

@@ -115,7 +115,7 @@ class IdentitySyncThread(SyncThread):
         self.log_extra = {
             "instance": "{}/{}: ".format(self.region_name, self.endpoint_type)
         }
-        LOG.info("IdentitySyncThread initialized", extra=self.log_extra)
+        LOG.debug("IdentitySyncThread initialized", extra=self.log_extra)
 
     def initialize_sc_clients(self):
         super().initialize_sc_clients()

distributedcloud/dcorch/engine/sync_services/sysinv.py

@@ -101,7 +101,7 @@ class SysinvSyncThread(SyncThread):
 
         self.sc_sysinv_client = None
 
-        LOG.info("SysinvSyncThread initialized", extra=self.log_extra)
+        LOG.debug("SysinvSyncThread initialized", extra=self.log_extra)
 
     def initialize_sc_clients(self):
         super().initialize_sc_clients()

distributedcloud/dcorch/engine/sync_thread.py

@@ -769,17 +769,18 @@ class SyncThread(object):
                 # shouldn't get here
                 LOG.exception("Unexpected error!")
 
-        if not total_num_of_audit_jobs:
+        if total_num_of_audit_jobs == 0:
             # todo: if we had an "unable to sync this
             # subcloud/endpoint" alarm raised, then clear it
             pass
+        else:
+            db_api.subcloud_sync_update(
+                self.ctxt,
+                self.subcloud_name,
+                self.endpoint_type,
+                values={"sync_request": consts.SYNC_STATUS_REQUESTED},
+            )
 
-        db_api.subcloud_sync_update(
-            self.ctxt,
-            self.subcloud_name,
-            self.endpoint_type,
-            values={"sync_request": consts.SYNC_STATUS_REQUESTED},
-        )
         LOG.debug(
             "{}: done sync audit".format(threading.currentThread().getName()),
             extra=self.log_extra,

distributedcloud/dcorch/tests/unit/engine/sync_services/test_identity.py

@@ -594,7 +594,7 @@ class TestIdentitySyncThreadProjectRoleAssignmentsPut(
 
         self._execute()
 
-        self._assert_log("info", "IdentitySyncThread initialized")
+        self._assert_log("debug", "IdentitySyncThread initialized")
         self.log.error.assert_not_called()
 
 

distributedcloud/files/distcloud-logrotate.conf

@@ -1,4 +1,23 @@
-/var/log/dcmanager/*.log
+/var/log/dcmanager/state.log
+{
+        nodateext
+        size 40M
+        start 1
+        rotate 20
+        missingok
+        notifempty
+        compress
+        sharedscripts
+        postrotate
+                 systemctl reload syslog-ng > /dev/null 2>&1 || true
+        endscript
+
+}
+
+/var/log/dcmanager/audit.log
+/var/log/dcmanager/dcmanager-api.log
+/var/log/dcmanager/dcmanager.log
+/var/log/dcmanager/orchestrator.log
 {
         nodateext
         size 20M