openstack/swift-bench
Code Issues Proposed changes

refactor bin/bench into swiftbench/cli for testing

Browse Source 
The test infrastructure for swiftbench is pretty bad. In order to start
cleaning it up this patch starts with some initial scaffolding for
test_cli.py. Sd such it refactors the code out of bin/swiftbench and
puts it into swiftbench/cli/__init__.py. Leaving the cli namespace free
for future expansions.

A very basic tests/test_cli.py has been added which initialises some test
scaffolding. Additionally, fix some bugs this uncovered:

- The --delete-concurrency option now actually has an effect.
- Stop mutating global state on every run.

Partial-Bug: #1263290
Change-Id: Ibb2c3bb17522b6302697e2d2b01df3a6aa62800e
This commit is contained in:
Matthew Oliver 2022-12-07 17:55:13 +11:00 committed by Tim Burke
parent 427a16cb08
commit 5f803ae8fd
3 changed files with 363 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
bin/swift-bench
View File

@ -13,181 +13,9 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import sys
import signal
import uuid
from optparse import OptionParser
from six.moves import range
from swiftbench.bench import (BenchController, DistributedBenchController,
create_containers, delete_containers)
from swiftbench.utils import readconf, config_true_value
# The defaults should be sufficient to run swift-bench on a SAIO
CONF_DEFAULTS = {
'auth': os.environ.get('ST_AUTH', ''),
'user': os.environ.get('ST_USER', ''),
'key': os.environ.get('ST_KEY', ''),
'auth_version': '1.0',
'use_proxy': 'yes',
'put_concurrency': '10',
'get_concurrency': '10',
'del_concurrency': '10',
'concurrency': '', # set all 3 in one shot
'object_sources': '', # set of file contents to read and use for PUTs
'lower_object_size': '10', # bounded random size used if these differ
'upper_object_size': '10',
'object_size': '1', # only if not object_sources and lower == upper
'num_objects': '1000',
'num_gets': '10000',
'delete': 'yes',
'container_name': uuid.uuid4().hex, # really "container name base"
'num_containers': '20',
'url': '', # used when use_proxy = no or overrides auth X-Storage-Url
'account': '', # used when use_proxy = no
'devices': 'sdb1', # space-sep list
'log_level': 'INFO',
'timeout': '10',
'delay': '0',
'bench_clients': [],
}
SAIO_DEFAULTS = {
'auth': 'http://localhost:8080/auth/v1.0',
'user': 'test:tester',
'key': 'testing',
}
from swiftbench.cli import main
if __name__ == '__main__':
usage = "usage: %prog [OPTIONS] [CONF_FILE]"
usage += """\n\nConf file with SAIO defaults:
[bench]
auth = http://localhost:8080/auth/v1.0
user = test:tester
key = testing
concurrency = 10
object_size = 1
num_objects = 1000
num_gets = 10000
delete = yes
auth_version = 1.0
policy_name = gold
"""
parser = OptionParser(usage=usage)
parser.add_option('', '--saio', dest='saio', action='store_true',
default=False, help='Run benchmark with SAIO defaults')
parser.add_option('-A', '--auth', dest='auth',
help='URL for obtaining an auth token')
parser.add_option('-U', '--user', dest='user',
help='User name for obtaining an auth token')
parser.add_option('-K', '--key', dest='key',
help='Key for obtaining an auth token')
parser.add_option('-b', '--bench-clients', action='append',
metavar='<ip>:<port>',
help=('A string of the form "<ip>:<port>" which matches '
'the arguments supplied to a swift-bench-client '
'process. This argument must be specified '
'once per swift-bench-client you want to '
'utilize.'))
parser.add_option('-u', '--url', dest='url',
help='Storage URL')
parser.add_option('-c', '--concurrency', dest='concurrency',
help=('Number of concurrent connections to use. For '
'finer-grained control, see --get-concurrency, '
'--put-concurrency, and --delete-concurrency.'))
parser.add_option('--get-concurrency', dest='get_concurrency',
help='Number of concurrent GET requests')
parser.add_option('--put-concurrency', dest='put_concurrency',
help='Number of concurrent PUT requests')
parser.add_option('--delete-concurrency', dest='delete_concurrency',
help='Number of concurrent DELETE requests')
parser.add_option('-s', '--object-size', dest='object_size',
help='Size of objects to PUT (in bytes)')
parser.add_option('-l', '--lower-object-size', dest='lower_object_size',
help=('Lower size of objects (in bytes); '
'--object-size will be upper-object-size'))
parser.add_option('-n', '--num-objects', dest='num_objects',
help='Number of objects to PUT')
parser.add_option('-g', '--num-gets', dest='num_gets',
help='Number of GET operations to perform')
parser.add_option('-C', '--num-containers', dest='num_containers',
help='Number of containers to distribute objects among')
parser.add_option('-x', '--no-delete', dest='delete', action='store_false',
help='If set, will not delete the objects created')
parser.add_option('-V', '--auth_version', dest='auth_version',
help='Authentication version')
parser.add_option('-d', '--delay', dest='delay',
help='Delay before delete requests in seconds')
parser.add_option('-P', '--policy-name', dest='policy_name',
help='Specify which policy to use when creating '
'containers')
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
options, args = parser.parse_args()
if options.saio:
CONF_DEFAULTS.update(SAIO_DEFAULTS)
if getattr(options, 'lower_object_size', None):
if options.object_size <= options.lower_object_size:
raise ValueError('--lower-object-size (%s) must be '
'< --object-size (%s)' %
(options.lower_object_size, options.object_size))
CONF_DEFAULTS['upper_object_size'] = options.object_size
if args:
conf = args[0]
if not os.path.exists(conf):
sys.exit("No such conf file: %s" % conf)
conf = readconf(conf, 'bench', log_name='swift-bench',
defaults=CONF_DEFAULTS)
conf['bench_clients'] = []
else:
conf = CONF_DEFAULTS
parser.set_defaults(**conf)
options, _junk = parser.parse_args()
if options.concurrency != '':
options.put_concurrency = options.concurrency
options.get_concurrency = options.concurrency
options.del_concurrency = options.concurrency
options.containers = ['%s_%d' % (options.container_name, i)
for i in range(int(options.num_containers))]
# Turn "yes"/"no"/etc. strings to booleans
options.use_proxy = config_true_value(options.use_proxy)
options.delete = config_true_value(options.delete)
def sigterm(signum, frame):
sys.exit('Termination signal received.')
signal.signal(signal.SIGTERM, sigterm)
logger = logging.getLogger('swift-bench')
logger.propagate = False
logger.setLevel({
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL}.get(
options.log_level.lower(), logging.INFO))
loghandler = logging.StreamHandler()
logger.addHandler(loghandler)
logformat = logging.Formatter(
'swift-bench %(asctime)s %(levelname)s %(message)s')
loghandler.setFormatter(logformat)
if options.use_proxy:
create_containers(logger, options)
controller_class = DistributedBenchController if options.bench_clients \
else BenchController
controller = controller_class(logger, options)
controller.run()
if options.use_proxy and options.delete:
delete_containers(logger, options)
sys.exit(main(sys.argv[1:]))

192
swiftbench/cli/__init__.py Normal file
View File

@ -0,0 +1,192 @@
# Copyright (c) 2022 NVIDIA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import logging
import os
import sys
import signal
import uuid
from optparse import OptionParser
from six.moves import range
from swiftbench.bench import (BenchController, DistributedBenchController,
create_containers, delete_containers)
from swiftbench.utils import readconf, config_true_value
# The defaults should be sufficient to run swift-bench on a SAIO
CONF_DEFAULTS = {
'auth': os.environ.get('ST_AUTH', ''),
'user': os.environ.get('ST_USER', ''),
'key': os.environ.get('ST_KEY', ''),
'auth_version': '1.0',
'use_proxy': 'yes',
'put_concurrency': '10',
'get_concurrency': '10',
'del_concurrency': '10',
'concurrency': '', # set all 3 in one shot
'object_sources': '', # set of file contents to read and use for PUTs
'lower_object_size': '10', # bounded random size used if these differ
'upper_object_size': '10',
'object_size': '1', # only if not object_sources and lower == upper
'num_objects': '1000',
'num_gets': '10000',
'delete': 'yes',
'container_name': uuid.uuid4().hex, # really "container name base"
'num_containers': '20',
'url': '', # used when use_proxy = no or overrides auth X-Storage-Url
'account': '', # used when use_proxy = no
'devices': 'sdb1', # space-sep list
'log_level': 'INFO',
'timeout': '10',
'delay': '0',
'bench_clients': [],
}
SAIO_DEFAULTS = {
'auth': 'http://localhost:8080/auth/v1.0',
'user': 'test:tester',
'key': 'testing',
}
def main(argv):
usage = "usage: %prog [OPTIONS] [CONF_FILE]"
usage += """\n\nConf file with SAIO defaults:
[bench]
auth = http://localhost:8080/auth/v1.0
user = test:tester
key = testing
concurrency = 10
object_size = 1
num_objects = 1000
num_gets = 10000
delete = yes
auth_version = 1.0
policy_name = gold
"""
parser = OptionParser(usage=usage)
parser.add_option('', '--saio', dest='saio', action='store_true',
default=False, help='Run benchmark with SAIO defaults')
parser.add_option('-A', '--auth', dest='auth',
help='URL for obtaining an auth token')
parser.add_option('-U', '--user', dest='user',
help='User name for obtaining an auth token')
parser.add_option('-K', '--key', dest='key',
help='Key for obtaining an auth token')
parser.add_option('-b', '--bench-clients', action='append',
metavar='<ip>:<port>',
help=('A string of the form "<ip>:<port>" which matches '
'the arguments supplied to a swift-bench-client '
'process. This argument must be specified '
'once per swift-bench-client you want to '
'utilize.'))
parser.add_option('-u', '--url', dest='url',
help='Storage URL')
parser.add_option('-c', '--concurrency', dest='concurrency',
help=('Number of concurrent connections to use. For '
'finer-grained control, see --get-concurrency, '
'--put-concurrency, and --delete-concurrency.'))
parser.add_option('--get-concurrency', dest='get_concurrency',
help='Number of concurrent GET requests')
parser.add_option('--put-concurrency', dest='put_concurrency',
help='Number of concurrent PUT requests')
parser.add_option('--delete-concurrency', dest='del_concurrency',
help='Number of concurrent DELETE requests')
parser.add_option('-s', '--object-size', dest='object_size',
help='Size of objects to PUT (in bytes)')
parser.add_option('-l', '--lower-object-size', dest='lower_object_size',
help=('Lower size of objects (in bytes); '
'--object-size will be upper-object-size'))
parser.add_option('-n', '--num-objects', dest='num_objects',
help='Number of objects to PUT')
parser.add_option('-g', '--num-gets', dest='num_gets',
help='Number of GET operations to perform')
parser.add_option('-C', '--num-containers', dest='num_containers',
help='Number of containers to distribute objects among')
parser.add_option('-x', '--no-delete', dest='delete', action='store_false',
help='If set, will not delete the objects created')
parser.add_option('-V', '--auth_version', dest='auth_version',
help='Authentication version')
parser.add_option('-d', '--delay', dest='delay',
help='Delay before delete requests in seconds')
parser.add_option('-P', '--policy-name', dest='policy_name',
help='Specify which policy to use when creating '
'containers')
options, args = parser.parse_args(argv)
parser_defaults = copy.deepcopy(CONF_DEFAULTS)
if options.saio:
parser_defaults.update(SAIO_DEFAULTS)
if getattr(options, 'lower_object_size', None):
if options.object_size <= options.lower_object_size:
raise ValueError('--lower-object-size (%s) must be '
'< --object-size (%s)' %
(options.lower_object_size, options.object_size))
parser_defaults['upper_object_size'] = options.object_size
if args:
conf = args[0]
if not os.path.exists(conf):
sys.exit("No such conf file: %s" % conf)
conf = readconf(conf, 'bench', log_name='swift-bench',
defaults=parser_defaults)
conf['bench_clients'] = []
else:
conf = parser_defaults
parser.set_defaults(**conf)
options, _junk = parser.parse_args(argv)
if options.concurrency != '':
options.put_concurrency = options.concurrency
options.get_concurrency = options.concurrency
options.del_concurrency = options.concurrency
options.containers = ['%s_%d' % (options.container_name, i)
for i in range(int(options.num_containers))]
# Turn "yes"/"no"/etc. strings to booleans
options.use_proxy = config_true_value(options.use_proxy)
options.delete = config_true_value(options.delete)
def sigterm(signum, frame):
sys.exit('Termination signal received.')
signal.signal(signal.SIGTERM, sigterm)
logger = logging.getLogger('swift-bench')
logger.propagate = False
logger.setLevel({
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL}.get(
options.log_level.lower(), logging.INFO))
loghandler = logging.StreamHandler()
logger.addHandler(loghandler)
logformat = logging.Formatter(
'swift-bench %(asctime)s %(levelname)s %(message)s')
loghandler.setFormatter(logformat)
if options.use_proxy:
create_containers(logger, options)
controller_class = DistributedBenchController if options.bench_clients \
else BenchController
controller = controller_class(logger, options)
controller.run()
if options.use_proxy and options.delete:
delete_containers(logger, options)

169
tests/test_cli.py Normal file
View File

@ -0,0 +1,169 @@
# Copyright (c) 2022 NVIDIA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import mock
from swiftbench import cli
class TestCli(unittest.TestCase):
def setUp(self):
self.container_options = None
self.delete_options = None
def fake_create_containers(self, logger, options):
self.container_options = options
def fake_delete_containers(self, logger, options):
self.delete_options = options
def run_main(self, args):
mock_controller = mock.Mock()
with mock.patch('swiftbench.cli.create_containers',
self.fake_create_containers), \
mock.patch('swiftbench.cli.delete_containers',
self.fake_delete_containers), \
mock.patch('swiftbench.cli.BenchController',
mock_controller), \
mock.patch('swiftbench.cli.DistributedBenchController',
mock_controller):
cli.main(args)
return (mock_controller.call_args[0][-1], self.container_options,
self.delete_options)
def test_defaults(self):
controller_opts, container_opts, del_opts = self.run_main([])
self.assertFalse(controller_opts.saio)
self.assertEqual(controller_opts.auth, '')
self.assertEqual(controller_opts.user, '')
self.assertEqual(controller_opts.key, '')
self.assertEqual(controller_opts.url, '')
self.assertEqual(controller_opts.concurrency, '')
self.assertEqual(controller_opts.get_concurrency, '10')
self.assertEqual(controller_opts.put_concurrency, '10')
self.assertEqual(controller_opts.lower_object_size, '10')
self.assertEqual(controller_opts.upper_object_size, '10')
self.assertEqual(controller_opts.num_objects, '1000')
self.assertEqual(controller_opts.num_gets, '10000')
self.assertEqual(controller_opts.num_containers, '20')
self.assertEqual(len(controller_opts.containers), 20)
self.assertTrue(controller_opts.delete)
self.assertEqual(controller_opts.auth_version, '1.0')
self.assertEqual(controller_opts.delay, '0')
self.assertIsNone(controller_opts.policy_name)
self.assertTrue(controller_opts.use_proxy)
self.assertEqual(controller_opts.del_concurrency, '10')
self.assertEqual(controller_opts.object_sources, '')
self.assertEqual(controller_opts.account, '')
self.assertEqual(controller_opts.container_name, mock.ANY)
self.assertEqual(controller_opts.devices, 'sdb1')
self.assertEqual(controller_opts.log_level, 'INFO')
self.assertEqual(controller_opts.timeout, '10')
self.assertEqual(controller_opts.bench_clients, [])
self.assertTrue(controller_opts.containers)
def test_defaults_with_saio(self):
controller_opts, container_opts, del_opts = self.run_main(['--saio'])
self.assertTrue(controller_opts.saio)
self.assertEqual(controller_opts.auth,
'http://localhost:8080/auth/v1.0')
self.assertEqual(controller_opts.user, 'test:tester')
self.assertEqual(controller_opts.key, 'testing')
self.assertEqual(controller_opts.url, '')
self.assertEqual(controller_opts.concurrency, '')
self.assertEqual(controller_opts.get_concurrency, '10')
self.assertEqual(controller_opts.put_concurrency, '10')
self.assertEqual(controller_opts.lower_object_size, '10')
self.assertEqual(controller_opts.upper_object_size, '10')
self.assertEqual(controller_opts.num_objects, '1000')
self.assertEqual(controller_opts.num_gets, '10000')
self.assertEqual(controller_opts.num_containers, '20')
self.assertEqual(len(controller_opts.containers), 20)
self.assertTrue(controller_opts.delete)
self.assertEqual(controller_opts.auth_version, '1.0')
self.assertEqual(controller_opts.delay, '0')
self.assertIsNone(controller_opts.policy_name)
self.assertTrue(controller_opts.use_proxy)
self.assertEqual(controller_opts.del_concurrency, '10')
self.assertEqual(controller_opts.object_sources, '')
self.assertEqual(controller_opts.account, '')
self.assertEqual(controller_opts.container_name, mock.ANY)
self.assertEqual(controller_opts.devices, 'sdb1')
self.assertEqual(controller_opts.log_level, 'INFO')
self.assertEqual(controller_opts.timeout, '10')
self.assertEqual(controller_opts.bench_clients, [])
self.assertTrue(controller_opts.containers)
def test_cli_options(self):
controller_opts, container_opts, del_opts = self.run_main(
[
'--auth', 'http://some_url/auth/v1.0',
'-U', 'other:user',
'--key', 'my_key',
'--bench-clients', '1.2.3.4:1234',
'--url', 'http://storage.url/v1/AUTH_user',
'--get-concurrency', '9',
'--put-concurrency', '8',
'--delete-concurrency', '7',
'--object-size', '999',
'--lower-object-size', '1',
'--num-objects', '5000',
'--num-gets', '4000',
'--num-containers', '4',
'-x',
'--auth_version', '2.0',
'--delay', '10',
'--policy-name', 'gold'])
self.assertFalse(controller_opts.saio)
self.assertEqual(controller_opts.auth, 'http://some_url/auth/v1.0')
self.assertEqual(controller_opts.user, 'other:user')
self.assertEqual(controller_opts.key, 'my_key')
self.assertEqual(controller_opts.bench_clients, ['1.2.3.4:1234'])
self.assertEqual(controller_opts.url,
'http://storage.url/v1/AUTH_user')
self.assertEqual(controller_opts.get_concurrency, '9')
self.assertEqual(controller_opts.put_concurrency, '8')
self.assertEqual(controller_opts.del_concurrency, '7')
self.assertEqual(controller_opts.object_size, '999')
self.assertEqual(controller_opts.lower_object_size, '1')
self.assertEqual(controller_opts.num_objects, '5000')
self.assertEqual(controller_opts.num_gets, '4000')
self.assertEqual(controller_opts.num_containers, '4')
self.assertEqual(len(controller_opts.containers), 4)
self.assertFalse(controller_opts.delete)
self.assertEqual(controller_opts.auth_version, '2.0')
self.assertEqual(controller_opts.delay, '10')
self.assertEqual(controller_opts.policy_name, 'gold')
self.assertTrue(controller_opts.use_proxy)
self.assertEqual(controller_opts.object_sources, '')
self.assertEqual(controller_opts.account, '')
self.assertEqual(controller_opts.container_name, mock.ANY)
self.assertEqual(controller_opts.devices, 'sdb1')
self.assertEqual(controller_opts.log_level, 'INFO')
self.assertEqual(controller_opts.timeout, '10')
self.assertTrue(controller_opts.containers)
def test_concurrency_overrides_get_put_delete(self):
controller_opts, container_opts, del_opts = self.run_main(
[
'--concurrency', '5',
'--get-concurrency', '9',
'--put-concurrency', '8',
'--delete-concurrency', '7'])
self.assertEqual(controller_opts.concurrency, '5')
self.assertEqual(controller_opts.get_concurrency, '5')
self.assertEqual(controller_opts.put_concurrency, '5')
self.assertEqual(controller_opts.del_concurrency, '5')