Added fluentd support

Change-Id: Ie5ce47d28ede62092b53d8c448abae66c9e4d995

requirements.txt

@@ -18,6 +18,7 @@ python-openstackclient>=0.4.1
 python-keystoneclient>=3.10.0
 scp>=0.8.0
 tabulate>=0.7.3
+fluent-logger>=0.5.2
 
 # Workaround for pip install failed on RHEL/CentOS
 functools32>=3.2.3

test-requirements.txt

@@ -14,4 +14,4 @@ oslotest>=1.10.0  # Apache-2.0
 testrepository>=0.0.18
 testscenarios>=0.4
 testtools>=1.4.0
-
+pytest>=3.0.2

test/test_vmtp.py

@@ -19,10 +19,23 @@ test_vmtp
 Tests for `vmtp` module.
 """
 
-from vmtp.tests import base
+import logging
+from vmtp.fluentd import FluentLogHandler
+import vmtp.log
 
 
-class TestVmtp(base.TestCase):
+def setup_module(module):
+    vmtp.log.setup(product_name="test")
 
-    def test_something(self):
-        pass
+
+def test_fluentd():
+    logger = logging.getLogger('fluent-logger')
+    handler = FluentLogHandler('vmtp', fluentd_port=7081)
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+    logger.info('test')
+    logger.warning('test %d', 100)
+    try:
+        raise Exception("test")
+    except Exception:
+        logger.exception("got exception")

tox.ini

@@ -10,10 +10,10 @@ setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-commands = python setup.py testr --slowest --testr-args='{posargs}'
+commands = py.test -q -s --basetemp={envtmpdir} {posargs}
 
 [testenv:pep8]
-commands = flake8
+commands = flake8 {toxinidir}
 
 [testenv:venv]
 commands = {posargs}

vmtp/cfg.default.yaml

@@ -211,3 +211,17 @@ vmtp_db: "client_db"
 ########################################
 vmtp_collection: "pns_web_entry"
 
+# When enabled, all logs will be sent to a fluentd server at the requested IP and port
+# The fluentd "tag" and "label" fields for every message will be set to "nfvbench"
+fluentd:
+    # by default (logging_tag is empty) nfvbench log messages are not sent to fluentd
+    # to enable logging to fluents, specify a valid fluentd tag name to be used for the
+    # log records
+    logging_tag:
+
+    # IP address of the server, defaults to loopback
+    ip: 127.0.0.1
+
+    # port # to use, by default, use the default fluentd forward port
+    port: 24224
+

vmtp/fluentd.py

@@ -0,0 +1,49 @@
+# Copyright 2017 Cisco Systems, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from datetime import datetime
+from fluent import sender
+import logging
+
+
+class FluentLogHandler(logging.Handler):
+    '''This is a minimalist log handler for use with Fluentd
+
+    Needs to be attached to a logger using the addHandler method.
+    It only picks up from every record:
+    - the formatted message (no timestamp and no level)
+    - the level name
+    - the runlogdate (to tie multiple run-related logs together)
+    The timestamp is retrieved by the fluentd library.
+    '''
+
+    def __init__(self, tag, fluentd_ip='127.0.0.1', fluentd_port=24224):
+        logging.Handler.__init__(self)
+        self.tag = tag
+        self.formatter = logging.Formatter('%(message)s')
+        self.sender = sender.FluentSender(self.tag, port=fluentd_port)
+        self.start_new_run()
+
+    def start_new_run(self):
+        '''Delimitate a new run in the stream of records with a new timestamp
+        '''
+        self.runlogdate = str(datetime.now())
+
+    def emit(self, record):
+        data = {
+            "runlogdate": self.runlogdate,
+            "loglevel": record.levelname,
+            "message": self.formatter.format(record)
+        }
+        self.sender.emit(None, data)

vmtp/tests/base.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2010-2011 OpenStack Foundation
-# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from oslotest import base
-
-
-class TestCase(base.BaseTestCase):
-
-    """Test case base class for all unit tests."""

vmtp/vmtp.py

@@ -30,6 +30,7 @@ import compute
 from config import config_load
 from config import config_loads
 import credentials
+from fluentd import FluentLogHandler
 from glanceclient.v2 import client as glanceclient
 import iperf_tool
 from keystoneclient import client as keystoneclient
@@ -49,7 +50,7 @@ import sshutils
 
 flow_num = 0
 return_code = 0
-
+fluent_logger = None
 
 class FlowPrinter(object):
     @staticmethod
@@ -189,7 +190,7 @@ class VmtpTest(object):
                 self.instance_access.public_key_file = pub_key
                 self.instance_access.private_key_file = priv_key
             else:
-                LOG.error('Default keypair ~/.ssh/id_rsa[.pub] does not exist. Please '
+                LOG.error('Default id ~/.ssh/id_rsa[.pub] does not exist. Please '
                           'either create one in your home directory, or specify your '
                           'keypair information in the config file before running VMTP.')
                 sys.exit(1)
@@ -458,7 +459,10 @@ class VmtpTest(object):
 
     def run(self):
         error_flag = False
-
+        if fluent_logger:
+            # take a snapshot of the current time for this new run
+            # so that all subsequent logs can relate to this run
+            fluent_logger.start_new_run()
         try:
             self.setup()
             self.measure_vm_flows()
@@ -1141,6 +1145,7 @@ def merge_opts_to_configs(opts):
 
 
 def run_vmtp(opts):
+    global fluent_logger
     '''Run VMTP
     :param opts: Parameters that to be passed to VMTP in type argparse.Namespace(). See:
                  http://vmtp.readthedocs.org/en/latest/usage.html#running-vmtp-as-a-library
@@ -1156,6 +1161,14 @@ def run_vmtp(opts):
                 opts.__setattr__(key, value)
 
     config = merge_opts_to_configs(opts)
+    # setup the fluent logger as soon as possible right after the config plugin is called
+    if config.fluentd.logging_tag:
+        fluent_logger = FluentLogHandler(config.fluentd.logging_tag,
+                                         fluentd_ip=config.fluentd.ip,
+                                         fluentd_port=config.fluentd.port)
+        LOG.addHandler(fluent_logger)
+    else:
+        fluent_logger = None
     rescol = ResultsCollector()
 
     # Run the native host tests if specified by user