Merge from trunk

bin/swift-bench

@@ -22,7 +22,7 @@ import uuid
 from optparse import OptionParser
 
 from swift.common.bench import BenchController
-from swift.common.utils import readconf, NamedLogger
+from swift.common.utils import readconf, LogAdapter, NamedFormatter
 
 # The defaults should be sufficient to run swift-bench on a SAIO
 CONF_DEFAULTS = {
@@ -124,10 +124,11 @@ if __name__ == '__main__':
         'critical': logging.CRITICAL}.get(
             options.log_level.lower(), logging.INFO))
     loghandler = logging.StreamHandler()
-    logformat = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
-    loghandler.setFormatter(logformat)
     logger.addHandler(loghandler)
-    logger = NamedLogger(logger, 'swift-bench')
+    logger = LogAdapter(logger)
+    logformat = NamedFormatter('swift-bench', logger,
+                   fmt='%(server)s %(asctime)s %(levelname)s %(message)s')
+    loghandler.setFormatter(logformat)
 
     controller = BenchController(logger, options)
     controller.run()

bin/swift-ring-builder

@@ -20,20 +20,40 @@ from gzip import GzipFile
 from os import mkdir
 from os.path import basename, dirname, exists, join as pathjoin
 from sys import argv, exit
+from textwrap import wrap
 from time import time
 
 from swift.common.ring import RingBuilder
 
 
 MAJOR_VERSION = 1
-MINOR_VERSION = 1
+MINOR_VERSION = 2
 EXIT_RING_CHANGED = 0
 EXIT_RING_UNCHANGED = 1
 EXIT_ERROR = 2
 
 
 def search_devs(builder, search_value):
-    # d<device_id>z<zone>-<ip>:<port>/<device_name>_<meta>
+    """
+The <search-value> can be of the form:
+    d<device_id>z<zone>-<ip>:<port>/<device_name>_<meta>
+    Any part is optional, but you must include at least one part.
+    Examples:
+        d74              Matches the device id 74
+        z1               Matches devices in zone 1
+        z1-1.2.3.4       Matches devices in zone 1 with the ip 1.2.3.4
+        1.2.3.4          Matches devices in any zone with the ip 1.2.3.4
+        z1:5678          Matches devices in zone 1 using port 5678
+        :5678            Matches devices that use port 5678
+        /sdb1            Matches devices with the device name sdb1
+        _shiny           Matches devices with shiny in the meta data
+        _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data
+    Most specific example:
+        d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
+    Nerd explanation:
+        All items require their single character prefix except the ip, in which
+        case the - is optional unless the device id or zone is also included.
+    """
     orig_search_value = search_value
     match = []
     if search_value.startswith('d'):
@@ -72,7 +92,8 @@ def search_devs(builder, search_value):
         match.append(('meta', search_value[1:]))
         search_value = ''
     if search_value:
-        raise ValueError('Invalid <search-value>: %s' % repr(orig_search_value))
+        raise ValueError('Invalid <search-value>: %s' %
+                         repr(orig_search_value))
     devs = []
     for dev in builder.devs:
         if not dev:
@@ -89,142 +110,22 @@ def search_devs(builder, search_value):
     return devs
 
 
-SEARCH_VALUE_HELP = '''
+class Commands:
-    The <search-value> can be of the form:
-    d<device_id>z<zone>-<ip>:<port>/<device_name>_<meta>
-    Any part is optional, but you must include at least one part.
-    Examples:
-        d74              Matches the device id 74
-        z1               Matches devices in zone 1
-        z1-1.2.3.4       Matches devices in zone 1 with the ip 1.2.3.4
-        1.2.3.4          Matches devices in any zone with the ip 1.2.3.4
-        z1:5678          Matches devices in zone 1 using port 5678
-        :5678            Matches devices that use port 5678
-        /sdb1            Matches devices with the device name sdb1
-        _shiny           Matches devices with shiny in the meta data
-        _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data
-    Most specific example:
-        d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
-    Nerd explanation:
-        All items require their single character prefix except the ip, in which
-        case the - is optional unless the device id or zone is also included.
-'''.strip()
 
-CREATE_HELP = '''
+    def unknown():
-swift-ring-builder <builder_file> create <part_power> <replicas> <min_part_hours>
+        print 'Unknown command: %s' % argv[2]
+        exit(EXIT_ERROR)
+
+    def create():
+        """
+swift-ring-builder <builder_file> create <part_power> <replicas>
+                                         <min_part_hours>
     Creates <builder_file> with 2^<part_power> partitions and <replicas>.
     <min_part_hours> is number of hours to restrict moving a partition more
     than once.
-'''.strip()
+        """
-
-SEARCH_HELP = '''
-swift-ring-builder <builder_file> search <search-value>
-    Shows information about matching devices.
-
-    %(SEARCH_VALUE_HELP)s
-'''.strip() % globals()
-
-ADD_HELP = '''
-swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta> <wght>
-    Adds a device to the ring with the given information. No partitions will be
-    assigned to the new device until after running 'rebalance'. This is so you
-    can make multiple device changes and rebalance them all just once.
-'''.strip()
-
-SET_WEIGHT_HELP = '''
-swift-ring-builder <builder_file> set_weight <search-value> <weight>
-    Resets the device's weight. No partitions will be reassigned to or from the
-    device until after running 'rebalance'. This is so you can make multiple
-    device changes and rebalance them all just once.
-
-    %(SEARCH_VALUE_HELP)s
-'''.strip() % globals()
-
-SET_INFO_HELP = '''
-swift-ring-builder <builder_file> set_info <search-value>
-                                     <ip>:<port>/<device_name>_<meta>
-    Resets the device's information. This information isn't used to assign
-    partitions, so you can use 'write_ring' afterward to rewrite the current
-    ring with the newer device information. Any of the parts are optional
-    in the final <ip>:<port>/<device_name>_<meta> parameter; just give what you
-    want to change. For instance set_info d74 _"snet: 5.6.7.8" would just
-    update the meta data for device id 74.
-
-    %(SEARCH_VALUE_HELP)s
-'''.strip() % globals()
-
-REMOVE_HELP = '''
-swift-ring-builder <builder_file> remove <search-value>
-    Removes the device(s) from the ring. This should normally just be used for
-    a device that has failed. For a device you wish to decommission, it's best
-    to set its weight to 0, wait for it to drain all its data, then use this
-    remove command. This will not take effect until after running 'rebalance'.
-    This is so you can make multiple device changes and rebalance them all just
-    once.
-
-    %(SEARCH_VALUE_HELP)s
-'''.strip() % globals()
-
-SET_MIN_PART_HOURS_HELP = '''
-swift-ring-builder <builder_file> set_min_part_hours <hours>
-    Changes the <min_part_hours> to the given <hours>. This should be set to
-    however long a full replication/update cycle takes. We're working on a way
-    to determine this more easily than scanning logs.
-'''.strip()
-
-
-if __name__ == '__main__':
-    if len(argv) < 2:
-        print '''
-swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s
-
-%(CREATE_HELP)s
-
-swift-ring-builder <builder_file>
-    Shows information about the ring and the devices within.
-
-%(SEARCH_HELP)s
-
-%(ADD_HELP)s
-
-%(SET_WEIGHT_HELP)s
-
-%(SET_INFO_HELP)s
-
-%(REMOVE_HELP)s
-
-swift-ring-builder <builder_file> rebalance
-    Attempts to rebalance the ring by reassigning partitions that haven't been
-    recently reassigned.
-
-swift-ring-builder <builder_file> validate
-    Just runs the validation routines on the ring.
-
-swift-ring-builder <builder_file> write_ring
-    Just rewrites the distributable ring file. This is done automatically after
-    a successful rebalance, so really this is only useful after one or more
-    'set_info' calls when no rebalance is needed but you want to send out the
-    new device information.
-
-%(SET_MIN_PART_HOURS_HELP)s
-
-Quick list: create search add set_weight set_info remove rebalance write_ring
-            set_min_part_hours
-Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
-'''.strip() % globals()
-        exit(EXIT_RING_UNCHANGED)
-
-    if exists(argv[1]):
-        builder = pickle.load(open(argv[1], 'rb'))
-        for dev in builder.devs:
-            if dev and 'meta' not in dev:
-                dev['meta'] = ''
-    elif len(argv) < 3 or argv[2] != 'create':
-        print 'Ring Builder file does not exist: %s' % argv[1]
-        exit(EXIT_ERROR)
-    elif argv[2] == 'create':
         if len(argv) < 6:
-            print CREATE_HELP
+            print Commands.create.__doc__.strip()
             exit(EXIT_RING_UNCHANGED)
         builder = RingBuilder(int(argv[3]), int(argv[4]), int(argv[5]))
         backup_dir = pathjoin(dirname(argv[1]), 'backups')
@@ -238,19 +139,11 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_CHANGED)
 
-    backup_dir = pathjoin(dirname(argv[1]), 'backups')
+    def default():
-    try:
+        """
-        mkdir(backup_dir)
+swift-ring-builder <builder_file>
-    except OSError, err:
+    Shows information about the ring and the devices within.
-        if err.errno != EEXIST:
+        """
-            raise
-
-    ring_file = argv[1]
-    if ring_file.endswith('.builder'):
-        ring_file = ring_file[:-len('.builder')]
-    ring_file += '.ring.gz'
-
-    if len(argv) == 2:
         print '%s, build version %d' % (argv[1], builder.version)
         zones = 0
         balance = 0
@@ -284,9 +177,15 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
                      dev['meta'])
         exit(EXIT_RING_UNCHANGED)
 
-    if argv[2] == 'search':
+    def search():
+        """
+swift-ring-builder <builder_file> search <search-value>
+    Shows information about matching devices.
+        """
         if len(argv) < 4:
-            print SEARCH_HELP
+            print Commands.search.__doc__.strip()
+            print
+            print search_devs.__doc__.strip()
             exit(EXIT_RING_UNCHANGED)
         devs = search_devs(builder, argv[3])
         if not devs:
@@ -311,10 +210,16 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
                  dev['meta'])
         exit(EXIT_RING_UNCHANGED)
 
-    elif argv[2] == 'add':
+    def add():
-        # add z<zone>-<ip>:<port>/<device_name>_<meta> <wght>
+        """
+swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
+                                      <wght>
+    Adds a device to the ring with the given information. No partitions will be
+    assigned to the new device until after running 'rebalance'. This is so you
+    can make multiple device changes and rebalance them all just once.
+        """
         if len(argv) < 5:
-            print ADD_HELP
+            print Commands.add.__doc__.strip()
             exit(EXIT_RING_UNCHANGED)
 
         if not argv[3].startswith('z'):
@@ -379,9 +284,17 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
-    elif argv[2] == 'set_weight':
+    def set_weight():
+        """
+swift-ring-builder <builder_file> set_weight <search-value> <weight>
+    Resets the device's weight. No partitions will be reassigned to or from the
+    device until after running 'rebalance'. This is so you can make multiple
+    device changes and rebalance them all just once.
+        """
         if len(argv) != 5:
-            print SET_WEIGHT_HELP
+            print Commands.set_weight.__doc__.strip()
+            print
+            print search_devs.__doc__.strip()
             exit(EXIT_RING_UNCHANGED)
         devs = search_devs(builder, argv[3])
         weight = float(argv[4])
@@ -404,9 +317,21 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
-    elif argv[2] == 'set_info':
+    def set_info():
+        """
+swift-ring-builder <builder_file> set_info <search-value>
+                                           <ip>:<port>/<device_name>_<meta>
+    Resets the device's information. This information isn't used to assign
+    partitions, so you can use 'write_ring' afterward to rewrite the current
+    ring with the newer device information. Any of the parts are optional
+    in the final <ip>:<port>/<device_name>_<meta> parameter; just give what you
+    want to change. For instance set_info d74 _"snet: 5.6.7.8" would just
+    update the meta data for device id 74.
+        """
         if len(argv) != 5:
-            print SET_INFO_HELP
+            print Commands.set_info.__doc__.strip()
+            print
+            print search_devs.__doc__.strip()
             exit(EXIT_RING_UNCHANGED)
         devs = search_devs(builder, argv[3])
         change_value = argv[4]
@@ -471,9 +396,20 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
-    elif argv[2] == 'remove':
+    def remove():
+        """
+swift-ring-builder <builder_file> remove <search-value>
+    Removes the device(s) from the ring. This should normally just be used for
+    a device that has failed. For a device you wish to decommission, it's best
+    to set its weight to 0, wait for it to drain all its data, then use this
+    remove command. This will not take effect until after running 'rebalance'.
+    This is so you can make multiple device changes and rebalance them all just
+    once.
+        """
         if len(argv) < 4:
-            print REMOVE_HELP
+            print Commands.remove.__doc__.strip()
+            print
+            print search_devs.__doc__.strip()
             exit(EXIT_RING_UNCHANGED)
         devs = search_devs(builder, argv[3])
         if not devs:
@@ -491,11 +427,17 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
         for dev in devs:
             builder.remove_dev(dev['id'])
             print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \
-                  'marked for removal and will be removed next rebalance.' % dev
+                  'marked for removal and will be removed next rebalance.' \
+                  % dev
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
-    elif argv[2] == 'rebalance':
+    def rebalance():
+        """
+swift-ring-builder <builder_file> rebalance
+    Attempts to rebalance the ring by reassigning partitions that haven't been
+    recently reassigned.
+        """
         devs_changed = builder.devs_changed
         last_balance = builder.get_balance()
         parts, balance = builder.rebalance()
@@ -528,15 +470,28 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_CHANGED)
 
-    elif argv[2] == 'validate':
+    def validate():
+        """
+swift-ring-builder <builder_file> validate
+    Just runs the validation routines on the ring.
+        """
         builder.validate()
         exit(EXIT_RING_UNCHANGED)
 
-    elif argv[2] == 'write_ring':
+    def write_ring():
+        """
+swift-ring-builder <builder_file> write_ring
+    Just rewrites the distributable ring file. This is done automatically after
+    a successful rebalance, so really this is only useful after one or more
+    'set_info' calls when no rebalance is needed but you want to send out the
+    new device information.
+        """
         ring_data = builder.get_ring()
         if not ring_data._replica2part2dev_id:
             if ring_data.devs:
-               print 'Warning: Writing a ring with no partition assignments but with devices; did you forget to run "rebalance"?'
+                print 'Warning: Writing a ring with no partition ' \
+                      'assignments but with devices; did you forget to run ' \
+                      '"rebalance"?'
             else:
                 print 'Warning: Writing an empty ring'
         pickle.dump(ring_data,
@@ -545,14 +500,20 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
         pickle.dump(ring_data, GzipFile(ring_file, 'wb'), protocol=2)
         exit(EXIT_RING_CHANGED)
 
-    elif argv[2] == 'pretend_min_part_hours_passed':
+    def pretend_min_part_hours_passed():
         builder.pretend_min_part_hours_passed()
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
-    elif argv[2] == 'set_min_part_hours':
+    def set_min_part_hours():
+        """
+swift-ring-builder <builder_file> set_min_part_hours <hours>
+    Changes the <min_part_hours> to the given <hours>. This should be set to
+    however long a full replication/update cycle takes. We're working on a way
+    to determine this more easily than scanning logs.
+        """
         if len(argv) < 4:
-            print SET_MIN_PART_HOURS_HELP
+            print Commands.set_min_part_hours.__doc__.strip()
             exit(EXIT_RING_UNCHANGED)
         builder.change_min_part_hours(int(argv[3]))
         print 'The minimum number of hours before a partition can be ' \
@@ -560,5 +521,51 @@ Exit codes: 0 = ring changed, 1 = ring did not change, 2 = error
         pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
-    print 'Unknown command: %s' % argv[2]
+
+if __name__ == '__main__':
+    if len(argv) < 2:
+        print "swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % \
+              globals()
+        print Commands.default.__doc__.strip()
+        print
+        cmds = [c for c, f in Commands.__dict__.iteritems()
+                    if f.__doc__ and c[0] != '_' and c != 'default']
+        cmds.sort()
+        for cmd in cmds:
+            print Commands.__dict__[cmd].__doc__.strip()
+            print
+        print search_devs.__doc__.strip()
+        print
+        for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ',
+                         subsequent_indent='            '):
+            print line
+        print 'Exit codes: 0 = ring changed, 1 = ring did not change, ' \
+              '2 = error'
+        exit(EXIT_RING_UNCHANGED)
+
+    if exists(argv[1]):
+        builder = pickle.load(open(argv[1], 'rb'))
+        for dev in builder.devs:
+            if dev and 'meta' not in dev:
+                dev['meta'] = ''
+    elif len(argv) < 3 or argv[2] != 'create':
+        print 'Ring Builder file does not exist: %s' % argv[1]
         exit(EXIT_ERROR)
+
+    backup_dir = pathjoin(dirname(argv[1]), 'backups')
+    try:
+        mkdir(backup_dir)
+    except OSError, err:
+        if err.errno != EEXIST:
+            raise
+
+    ring_file = argv[1]
+    if ring_file.endswith('.builder'):
+        ring_file = ring_file[:-len('.builder')]
+    ring_file += '.ring.gz'
+
+    if len(argv) == 2:
+        command = "default"
+    else:
+        command = argv[2]
+    Commands.__dict__.get(command, Commands.unknown)()

swift/common/middleware/catch_errors.py

@@ -26,6 +26,10 @@ class CatchErrorMiddleware(object):
 
     def __init__(self, app, conf):
         self.app = app
+        # if the application already has a logger we should use that one
+        self.logger = getattr(app, 'logger', None)
+        if not self.logger:
+            # and only call get_logger if we have to
             self.logger = get_logger(conf)
 
     def __call__(self, env, start_response):

swift/common/utils.py

@@ -284,11 +284,15 @@ class LoggerFileObject(object):
 
 
 class LogAdapter(object):
-    """Cheesy version of the LoggerAdapter available in Python 3"""
+    """
+    A Logger like object which performs some reformatting on calls to
+    :meth:`exception`.  Can be used to store a threadlocal transaction id.
+    """
+
+    _txn_id = threading.local()
 
     def __init__(self, logger):
         self.logger = logger
-        self._txn_id = threading.local()
         for proxied_method in ('debug', 'log', 'warn', 'warning', 'error',
                                'critical', 'info'):
             setattr(self, proxied_method, getattr(logger, proxied_method))
@@ -334,18 +338,45 @@ class LogAdapter(object):
 
 
 class NamedFormatter(logging.Formatter):
-    def __init__(self, server, logger):
+    """
-        logging.Formatter.__init__(self)
+    NamedFormatter is used to add additional information to log messages.
+    Normally it will simply add the server name as an attribute on the
+    LogRecord and the default format string will include it at the
+    begining of the log message.  Additionally, if the transaction id is
+    available and not already included in the message, NamedFormatter will
+    add it.
+
+    NamedFormatter may be initialized with a format string which makes use
+    of the standard LogRecord attributes.  In addition the format string
+    may include the following mapping key:
+
+    +----------------+---------------------------------------------+
+    | Format         | Description                                 |
+    +================+=============================================+
+    | %(server)s     | Name of the swift server doing logging      |
+    +----------------+---------------------------------------------+
+
+    :param server: the swift server name, a string.
+    :param logger: a Logger or :class:`LogAdapter` instance, additional
+                   context may be pulled from attributes on this logger if
+                   available.
+    :param fmt: the format string used to construct the message, if none is
+                supplied it defaults to ``"%(server)s %(message)s"``
+    """
+
+    def __init__(self, server, logger,
+                 fmt="%(server)s %(message)s"):
+        logging.Formatter.__init__(self, fmt)
         self.server = server
         self.logger = logger
 
     def format(self, record):
+        record.server = self.server
         msg = logging.Formatter.format(self, record)
         if self.logger.txn_id and (record.levelno != logging.INFO or
                                    self.logger.txn_id not in msg):
-            return '%s %s (txn: %s)' % (self.server, msg, self.logger.txn_id)
+            msg = "%s (txn: %s)" % (msg, self.logger.txn_id)
-        else:
+        return msg
-            return '%s %s' % (self.server, msg)
 
 
 def get_logger(conf, name=None, log_to_console=False):
@@ -386,7 +417,10 @@ def get_logger(conf, name=None, log_to_console=False):
     root_logger.setLevel(
         getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO))
     adapted_logger = LogAdapter(root_logger)
-    get_logger.handler.setFormatter(NamedFormatter(name, adapted_logger))
+    formatter = NamedFormatter(name, adapted_logger)
+    get_logger.handler.setFormatter(formatter)
+    if hasattr(get_logger, 'console'):
+        get_logger.console.setFormatter(formatter)
     return adapted_logger