Merge "Add handling of logs for fuel statistics collectors and sender"

nailgun/nailgun/settings.yaml

@@ -105,6 +105,8 @@ REMOVE_IMAGES_TIMEOUT: 60
 # statistics send interval
 STATS_SEND_INTERVAL: 3600
 
+STATS_LOGS_PATH: "/var/log/nailgun/"
+
 # statistics send enabled check interval
 STATS_ENABLE_CHECK_INTERVAL: 300
 

nailgun/nailgun/statistics/oswl/collector.py

@@ -14,13 +14,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import six
 import sys
 import time
 
 from nailgun import consts
 from nailgun.db import db
-from nailgun.logger import logger
 from nailgun.objects import ClusterCollection
 from nailgun.objects import MasterNodeSettings
 from nailgun.objects import OpenStackWorkloadStatsCollection
@@ -31,6 +31,9 @@ from nailgun.statistics.oswl.saver import oswl_statistics_save
 from nailgun.statistics import utils
 
 
+logger = logging.getLogger('statistics')
+
+
 def collect(resource_type):
     try:
         operational_clusters = ClusterCollection.filter_by(
@@ -94,6 +97,11 @@ def collect(resource_type):
 
 def run():
     resource_type = sys.argv[1]
+
+    # add file handler to log collecting for particular resource
+    log_file = "oswl_{0}_collectord.log".format(resource_type)
+    utils.prepare_logger(logger, log_file)
+
     poll_interval = settings.OSWL_COLLECTORS_POLLING_INTERVAL[resource_type]
     logger.info("Starting OSWL collector for {0} resource"
                 .format(resource_type))

nailgun/nailgun/statistics/oswl/helpers.py

@@ -11,6 +11,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import six
 
 from cinderclient import client as cinder_client
@@ -21,7 +22,6 @@ from novaclient import client as nova_client
 
 from nailgun import consts
 from nailgun.db import db
-from nailgun.logger import logger
 from nailgun import objects
 from nailgun.settings import settings
 from nailgun.statistics.oswl.resources_description \
@@ -29,6 +29,9 @@ from nailgun.statistics.oswl.resources_description \
 from nailgun.statistics import utils
 
 
+logger = logging.getLogger('statistics')
+
+
 class ClientProvider(object):
     """Initialize clients for OpenStack component and expose them as attrs"""
 

nailgun/nailgun/statistics/statsenderd.py

@@ -14,6 +14,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import requests
 import six
 import time
@@ -24,12 +25,15 @@ from oslo_serialization import jsonutils
 from nailgun import consts
 from nailgun.db import db
 from nailgun.db.sqlalchemy import models
-from nailgun.logger import logger
 from nailgun import objects
 from nailgun.settings import settings
 from nailgun.statistics.fuel_statistics.installation_info \
     import InstallationInfo
 from nailgun.statistics.utils import dithered
+from nailgun.statistics.utils import prepare_logger
+
+
+logger = logging.getLogger('statistics')
 
 
 class StatsSender(object):
@@ -250,6 +254,9 @@ class StatsSender(object):
 
 
 def run():
+    # add file handler to logger
+    prepare_logger(logger, file_name='statsenderd.log')
+
     logger.info("Starting standalone stats sender...")
     try:
         while True:

nailgun/nailgun/statistics/utils.py

@@ -12,18 +12,22 @@
 #    under the License.
 
 from collections import namedtuple
+import logging
 import os
 import random
 
 from contextlib import contextmanager
 
 from nailgun import consts
-from nailgun.logger import logger
+from nailgun.logger import set_logger
 from nailgun.network import manager
 from nailgun.settings import settings
 from nailgun.statistics import errors
 
 
+logger = logging.getLogger('statistics')
+
+
 WhiteListRule = namedtuple(
     'WhiteListItem', ['path', 'map_to_name', 'transform_func'])
 
@@ -159,3 +163,12 @@ def get_version_info(cluster):
     except Exception:
         logger.exception("Fetching version info for cluster '%s' failed",
                          cluster)
+
+
+def prepare_logger(logger, file_name):
+    handler = logging.FileHandler(
+        os.path.join(settings.STATS_LOGS_PATH, file_name)
+    )
+    set_logger(logger, handler)
+
+    return logger

nailgun/tools/env.sh

@@ -63,6 +63,7 @@ API_LOG: "${NAILGUN_LOGS}/api.log"
 APP_LOG: "${NAILGUN_LOGS}/app.log"
 RPC_CONSUMER_LOG_PATH: "${NAILGUN_LOGS}/receiverd.log"
 ASSASSIN_LOG_PATH: "${NAILGUN_LOGS}/assassind.log"
+STATS_LOGS_PATH: ${NAILGUN_LOGS}
 EOL
 }