Implement negative start for logs, set def. logs start to 30 days, show log size on screen

timmy/cli.py

@@ -242,7 +242,8 @@ def main(argv=None):
         enough = pretty_run(args.quiet, 'Checking free space',
                             nm.is_enough_space)
         if enough:
-            pretty_run(args.quiet, 'Collecting and packing logs', nm.get_logs,
+            msg = 'Collecting and packing %dMB of logs' % (nm.alogsize / 1024)
+            pretty_run(args.quiet, msg, nm.get_logs,
                        args=(conf['compress_timeout'],),
                        kwargs={'maxthreads': args.logs_maxthreads,
                                'fake': args.fake_logs})

timmy/conf.py

@@ -60,7 +60,8 @@ def load_conf(filename):
     conf['files'] = []
     conf['filelists'] = []
     conf['logs'] = {'path': '/var/log',
-                    'exclude': '\.[^12]\.gz$|\.\d{2,}\.gz$'}
+                    'exclude': '\.[^12]\.gz$|\.\d{2,}\.gz$',
+                    'start': '-30'}
     '''Shell mode - only run what was specified via command line.
     Skip actionable conf fields (see timmy/nodes.py -> Node.conf_actionable);
     Skip rqfile import;

timmy/nodes.py

@@ -25,7 +25,7 @@ import shutil
 import logging
 import sys
 import re
-from datetime import datetime
+from datetime import datetime, date, timedelta
 import tools
 from tools import w_list, run_with_lock
 from copy import deepcopy
@@ -323,12 +323,30 @@ class Node(object):
                      re.search(item['exclude'], string)))
 
         for item in self.logs:
+            start_str = ''
             if 'start' in item:
-                start = ' -newermt \\"$(date -d \'%s\')\\"' % item['start']
+                if item['start'].startswith('-'):
+                    days = int(item['start'][1:])
+                    start_str = str(date.today() - timedelta(days=days))
+                else:
+                    for format in ['%Y-%m-%d', '%Y-%m-%d %H:%M:%S']:
+                        try:
+                            if datetime.strptime(start_str, format):
+                                start_str = item['start']
+                                break
+                        except ValueError:
+                            pass
+                    if not start_str:
+                        self.logger.warning(('incorrect value of "start"'
+                                             ' parameter in "logs": "%s" -'
+                                             ' ignoring...')
+                                            % item['start'])
+            if start_str:
+                start_param = ' -newermt "$(date -d \'%s\')"' % start_str
             else:
-                start = ''
+                start_param = ''
             cmd = ("find '%s' -type f%s -exec du -b {} +" % (item['path'],
-                                                             start))
+                                                             start_param))
             self.logger.info('node: %s, logs du-cmd: %s' %
                              (self.id, cmd))
             outs, errs, code = tools.ssh_node(ip=self.ip,
@@ -349,6 +367,8 @@ class Node(object):
                         size, f = line.split('\t')
                         if filter_by_re(item, f):
                             item['files'][f] = int(size)
+                        else:
+                            self.logger.debug('log file "%s" excluded' % f)
                 self.logger.debug('logs: %s' % (item['files']))
         return self.logs