openstack/swift
Code Issues Proposed changes

Can run swift-bench across multiple cores/servers.

Browse Source 
You run one or more swift-bench-client processes like this:
    $ swift-bench-client 127.0.0.1 20001
    $ swift-bench-client 127.0.0.1 20002

Then you run swift-bench with a new option, --bench-clients (-b), which is
specified once for each swift-bench-client:
    $ swift-bench -b 127.0.0.1:20001 -b 127.0.0.1:20002

You get log lines from each client (interleaved) along with a final report
for all clients:
    127.0.0.1:20002 swift-bench-server 2012-08-25 22:44:06,148 INFO Auth version: 1.0
    127.0.0.1:20001 swift-bench-server 2012-08-25 22:44:06,148 INFO Auth version: 1.0
    127.0.0.1:20001 swift-bench-server 2012-08-25 22:44:12,249 INFO 83 PUTS [0 failures], 41.5/s
    127.0.0.1:20002 swift-bench-server 2012-08-25 22:44:14,430 INFO 74 PUTS [0 failures], 34.3/s
    ...
    127.0.0.1:20002 swift-bench-server 2012-08-25 22:45:18,942 INFO Auth version: 1.0
    127.0.0.1:20002 swift-bench-server 2012-08-25 22:45:20,946 INFO 238 DEL [2 failures], 118.9/s
    swift-bench 2012-08-25 22:45:27,549 INFO 2000 PUTS **FINAL** [0 failures], 56.8/s
    swift-bench 2012-08-25 22:45:27,550 INFO 30000 GETS **FINAL** [50 failures], 974.6/s
    swift-bench 2012-08-25 22:45:27,550 INFO 2000 DEL **FINAL** [20 failures], 237.1/s

The concurrency, PUT count, and GET count config settings are divided by
the number of bench_clients.  In other words, the same volume of work is
attempted (vs. not specifying --bench-clients), but it can now span
servers and CPU cores.

Benchmark containers are created (if use_proxy = yes) and deleted (if
delete = yes), with appropriate concurrency, in the initiating
swift-bench process, not any of the swift-bench-client processes.

Change-Id: Idbf31a23093244ab357a9bf77e6031257774f24a
This commit is contained in:
Darrell Bishop
2012-08-25 23:02:53 -07:00
committed by Gerrit Code Review
parent c509ac2371
commit ed3b12d05c
4 changed files with 262 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
bin/swift-bench
View File

@@ -21,7 +21,8 @@ import signal
import uuid
from optparse import OptionParser
from swift.common.bench import BenchController
from swift.common.bench import (BenchController, DistributedBenchController,
create_containers, delete_containers)
from swift.common.utils import readconf, LogAdapter
# The defaults should be sufficient to run swift-bench on a SAIO
@@ -49,6 +50,8 @@ CONF_DEFAULTS = {
'devices': 'sdb1', # space-sep list
'log_level': 'INFO',
'timeout': '10',
'auth_version': '1.0',
'bench_clients': [],
}
SAIO_DEFAULTS = {
@@ -81,6 +84,13 @@ if __name__ == '__main__':
help='User name for obtaining an auth token')
parser.add_option('-K', '--key', dest='key',
help='Key for obtaining an auth token')
parser.add_option('-b', '--bench-clients', action='append',
metavar='<ip>:<port>',
help=('A string of the form "<ip>:<port>" which matches '
'the arguments supplied to a swift-bench-client '
'process. This argument must be specified '
'once per swift-bench-client you want to '
'utilize.'))
parser.add_option('-u', '--url', dest='url',
help='Storage URL')
parser.add_option('-c', '--concurrency', dest='concurrency',
@@ -125,6 +135,8 @@ if __name__ == '__main__':
options.put_concurrency = options.concurrency
options.get_concurrency = options.concurrency
options.del_concurrency = options.concurrency
options.containers = ['%s_%d' % (options.container_name, i)
for i in xrange(int(options.num_containers))]
def sigterm(signum, frame):
sys.exit('Termination signal received.')
@@ -145,5 +157,13 @@ if __name__ == '__main__':
'%(message)s')
loghandler.setFormatter(logformat)
controller = BenchController(logger, options)
if options.use_proxy:
create_containers(logger, options)
controller_class = DistributedBenchController if options.bench_clients \
else BenchController
controller = controller_class(logger, options)
controller.run()
if options.delete:
delete_containers(logger, options)

59
bin/swift-bench-client Executable file
View File

@@ -0,0 +1,59 @@
#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import sys
import signal
from optparse import OptionParser
from swift.common.bench import BenchServer
from swift.common.utils import LogAdapter
if __name__ == '__main__':
usage = "usage: %prog <ip> <port>"
usage += "\n\nRun a client for distributed swift-bench runs."
parser = OptionParser(usage=usage)
parser.add_option('-o', '--log-level', dest='log_level',
default='info',
help='Logging level (debug, info, etc)')
if len(sys.argv) != 3:
parser.print_help()
sys.exit(1)
options, args = parser.parse_args()
logger = logging.getLogger()
logger.setLevel({
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL}.get(
options.log_level.lower(), logging.INFO))
loghandler = logging.StreamHandler()
logger.addHandler(loghandler)
logger = LogAdapter(logger, 'swift-bench-client')
logformat = logging.Formatter('%(server)s %(asctime)s %(levelname)s '
'%(message)s')
loghandler.setFormatter(logformat)
def sigterm(signum, frame):
sys.exit('Termination signal received.')
signal.signal(signal.SIGTERM, sigterm)
signal.signal(signal.SIGINT, sigterm)
server = BenchServer(logger, args[0], args[1])
server.run()