Add config options for log rotation

On Windows, in-use files cannot be moved or deleted. For this reason,
we need the service itself to take care of rotating logs.

For convenience reasons, we're exposing the built-in rotating log
handlers through a set of config options.

More specifically, we're adding the following new config options:
- log_rotate_interval
- log_rotate_interval_type
- max_logfile_count
- max_logfile_size_mb

Change-Id: I01db4efc08e2cb64db9cbf793f3a159f54859fe7
Closes-Bug: #1802262

doc/source/admin/index.rst

@@ -7,3 +7,4 @@
 
    advanced_config
    journal
+   log_rotation

doc/source/admin/log_rotation.rst

@@ -0,0 +1,45 @@
+=============
+ Log rotation
+=============
+
+oslo.log can work with ``logrotate``, picking up file changes once log files
+are rotated. Make sure to set the ``watch-log-file`` config option.
+
+Log rotation on Windows
+-----------------------
+
+On Windows, in-use files cannot be renamed or moved. For this reason,
+oslo.log allows setting maximum log file sizes or log rotation interval,
+in which case the service itself will take care of the log rotation (as
+opposed to having an external daemon).
+
+Configuring log rotation
+------------------------
+
+Use the following options to set a maximum log file size. In this sample,
+log files will be rotated when reaching 1GB, having at most 30 log files.
+
+.. code-block:: ini
+
+    [DEFAULT]
+    log_rotation_type = size
+    max_logfile_size_mb = 1024  # MB
+    max_logfile_count = 30
+
+The following sample configures log rotation to be performed every 12 hours.
+
+.. code-block:: ini
+
+    [DEFAULT]
+    log_rotation_type = interval
+    log_rotate_interval = 12
+    log_rotate_interval_type = H
+    max_logfile_count = 60
+
+.. note::
+
+    The time of the next rotation is computed when the service starts or when a
+    log rotation is performed, using the time of the last file modification or
+    the service start time, to which the configured log rotation interval is
+    added. This means that service restarts may delay periodic log file
+    rotations.

oslo_log/_options.py

@@ -111,6 +111,35 @@ generic_log_opts = [
     cfg.BoolOpt('use_eventlog',
                 default=False,
                 help='Log output to Windows Event Log.'),
+    cfg.IntOpt('log_rotate_interval',
+               default=1,
+               help='The amount of time before the log files are rotated. '
+                    'This option is ignored unless log_rotation_type is set'
+                    'to "interval".'),
+    cfg.StrOpt('log_rotate_interval_type',
+               choices=['Seconds', 'Minutes', 'Hours', 'Days', 'Weekday',
+                        'Midnight'],
+               ignore_case=True,
+               default='days',
+               help='Rotation interval type. The time of the last file '
+                    'change (or the time when the service was started) is '
+                    'used when scheduling the next rotation.'),
+    cfg.IntOpt('max_logfile_count',
+               default=30,
+               help='Maximum number of rotated log files.'),
+    cfg.IntOpt('max_logfile_size_mb',
+               default=200,
+               help='Log file maximum size in MB. This option is ignored if '
+                    '"log_rotation_type" is not set to "size".'),
+    cfg.StrOpt('log_rotation_type',
+               default='none',
+               choices=[('interval',
+                         'Rotate logs at predefined time intervals.'),
+                        ('size',
+                         'Rotate logs once they reach a predefined size.'),
+                        ('none', 'Do not rotate log files.')],
+               ignore_case=True,
+               help='Log rotation type.')
 ]
 
 log_opts = [

oslo_log/log.py

@@ -40,6 +40,7 @@ except ImportError:
 
 from oslo_config import cfg
 from oslo_utils import importutils
+from oslo_utils import units
 import six
 from six import moves
 
@@ -60,6 +61,15 @@ TRACE = handlers._TRACE
 
 logging.addLevelName(TRACE, 'TRACE')
 
+LOG_ROTATE_INTERVAL_MAPPING = {
+    'seconds': 's',
+    'minutes': 'm',
+    'hours': 'h',
+    'days': 'd',
+    'weekday': 'w',
+    'midnight': 'midnight'
+}
+
 
 def _get_log_file_path(conf, binary=None):
     logfile = conf.log_file
@@ -344,13 +354,33 @@ def _setup_logging_from_conf(conf, project, version):
 
     logpath = _get_log_file_path(conf)
     if logpath:
+        # On Windows, in-use files cannot be moved or deleted.
         if conf.watch_log_file and platform.system() == 'Linux':
             from oslo_log import watchers
             file_handler = watchers.FastWatchedFileHandler
+            filelog = file_handler(logpath)
+        elif conf.log_rotation_type.lower() == "interval":
+            file_handler = logging.handlers.TimedRotatingFileHandler
+            when = conf.log_rotate_interval_type.lower()
+            interval_type = LOG_ROTATE_INTERVAL_MAPPING[when]
+            # When weekday is configured, "when" has to be a value between
+            # 'w0'-'w6' (w0 for Monday, w1 for Tuesday, and so on)'
+            if interval_type == 'w':
+                interval_type = interval_type + str(conf.log_rotate_interval)
+            filelog = file_handler(logpath,
+                                   when=interval_type,
+                                   interval=conf.log_rotate_interval,
+                                   backupCount=conf.max_logfile_count)
+        elif conf.log_rotation_type.lower() == "size":
+            file_handler = logging.handlers.RotatingFileHandler
+            maxBytes = conf.max_logfile_size_mb * units.Mi
+            filelog = file_handler(logpath,
+                                   maxBytes=maxBytes,
+                                   backupCount=conf.max_logfile_count)
         else:
             file_handler = logging.handlers.WatchedFileHandler
+            filelog = file_handler(logpath)
 
-        filelog = file_handler(logpath)
         log_root.addHandler(filelog)
 
     if conf.use_stderr:

oslo_log/tests/unit/test_log.py

@@ -47,6 +47,7 @@ from oslo_log import _options
 from oslo_log import formatters
 from oslo_log import handlers
 from oslo_log import log
+from oslo_utils import units
 
 
 MIN_LOG_INI = b"""[loggers]
@@ -107,6 +108,7 @@ class CommonLoggerTestsMixIn(object):
                                                   '%(message)s')
         self.log = None
         log._setup_logging_from_conf(self.config_fixture.conf, 'test', 'test')
+        self.log_handlers = log.getLogger(None).logger.handlers
 
     def test_handlers_have_context_formatter(self):
         formatters_list = []
@@ -159,6 +161,58 @@ class CommonLoggerTestsMixIn(object):
         mock_logger = loggers_mock.return_value.logger
         mock_logger.addHandler.assert_any_call(handler_mock.return_value)
 
+    @mock.patch('oslo_log.watchers.FastWatchedFileHandler')
+    @mock.patch('oslo_log.log._get_log_file_path', return_value='test.conf')
+    @mock.patch('platform.system', return_value='Linux')
+    def test_watchlog_on_linux(self, platfotm_mock, path_mock, handler_mock):
+        self.config(watch_log_file=True)
+        log._setup_logging_from_conf(self.CONF, 'test', 'test')
+        handler_mock.assert_called_once_with(path_mock.return_value)
+        self.assertEqual(self.log_handlers[0], handler_mock.return_value)
+
+    @mock.patch('logging.handlers.WatchedFileHandler')
+    @mock.patch('oslo_log.log._get_log_file_path', return_value='test.conf')
+    @mock.patch('platform.system', return_value='Windows')
+    def test_watchlog_on_windows(self, platform_mock, path_mock, handler_mock):
+        self.config(watch_log_file=True)
+        log._setup_logging_from_conf(self.CONF, 'test', 'test')
+        handler_mock.assert_called_once_with(path_mock.return_value)
+        self.assertEqual(self.log_handlers[0], handler_mock.return_value)
+
+    @mock.patch('logging.handlers.TimedRotatingFileHandler')
+    @mock.patch('oslo_log.log._get_log_file_path', return_value='test.conf')
+    def test_timed_rotate_log(self, path_mock, handler_mock):
+        rotation_type = 'interval'
+        when = 'weekday'
+        interval = 2
+        backup_count = 2
+        self.config(log_rotation_type=rotation_type,
+                    log_rotate_interval=interval,
+                    log_rotate_interval_type=when,
+                    max_logfile_count=backup_count)
+        log._setup_logging_from_conf(self.CONF, 'test', 'test')
+        handler_mock.assert_called_once_with(path_mock.return_value,
+                                             when='w2',
+                                             interval=interval,
+                                             backupCount=backup_count)
+        self.assertEqual(self.log_handlers[0], handler_mock.return_value)
+
+    @mock.patch('logging.handlers.RotatingFileHandler')
+    @mock.patch('oslo_log.log._get_log_file_path', return_value='test.conf')
+    def test_rotate_log(self, path_mock, handler_mock):
+        rotation_type = 'size'
+        max_logfile_size_mb = 100
+        maxBytes = max_logfile_size_mb * units.Mi
+        backup_count = 2
+        self.config(log_rotation_type=rotation_type,
+                    max_logfile_size_mb=max_logfile_size_mb,
+                    max_logfile_count=backup_count)
+        log._setup_logging_from_conf(self.CONF, 'test', 'test')
+        handler_mock.assert_called_once_with(path_mock.return_value,
+                                             maxBytes=maxBytes,
+                                             backupCount=backup_count)
+        self.assertEqual(self.log_handlers[0], handler_mock.return_value)
+
 
 class LoggerTestCase(CommonLoggerTestsMixIn, test_base.BaseTestCase):
     def setUp(self):

releasenotes/notes/log-rotation-595f8232cd987a6d.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    The following new config options will allow rotating log files,
+    especially useful on Windows:
+    * ``log_rotate_interval``
+    * ``log_rotate_interval_type``
+    * ``max_logfile_count``
+    * ``max_logfile_size_mb``