From 4614f846090da2606fd9b539353bde8cab9ffe35 Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Wed, 7 Dec 2022 17:55:13 +1100 Subject: [PATCH] refactor bin/bench into swiftbench/cli for testing The test infrastructure for swiftbench is pretty bad. In order to start cleaning it up this patch starts with some initial scaffolding for test_cli.py. Sd such it refactors the code out of bin/swiftbench and puts it into swiftbench/cli/__init__.py. Leaving the cli namespace free for future expansions. A verybasic tests/test_cli.py has been added which initialises some test scaffolding and a first test_default has been added. This is at least a start that we can extend. Change-Id: Ibb2c3bb17522b6302697e2d2b01df3a6aa62800e --- bin/swift-bench | 176 +--------------------------------- swiftbench/cli/__init__.py | 190 +++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 169 +++++++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+), 174 deletions(-) create mode 100644 swiftbench/cli/__init__.py create mode 100644 tests/test_cli.py diff --git a/bin/swift-bench b/bin/swift-bench index c1b8997..6e75638 100755 --- a/bin/swift-bench +++ b/bin/swift-bench @@ -13,181 +13,9 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - -import logging -import os import sys -import signal -import uuid -from optparse import OptionParser -from six.moves import range - -from swiftbench.bench import (BenchController, DistributedBenchController, - create_containers, delete_containers) -from swiftbench.utils import readconf, config_true_value - -# The defaults should be sufficient to run swift-bench on a SAIO -CONF_DEFAULTS = { - 'auth': os.environ.get('ST_AUTH', ''), - 'user': os.environ.get('ST_USER', ''), - 'key': os.environ.get('ST_KEY', ''), - 'auth_version': '1.0', - 'use_proxy': 'yes', - 'put_concurrency': '10', - 'get_concurrency': '10', - 'del_concurrency': '10', - 'concurrency': '', # set all 3 in one shot - 'object_sources': '', # set of file contents to read and use for PUTs - 'lower_object_size': '10', # bounded random size used if these differ - 'upper_object_size': '10', - 'object_size': '1', # only if not object_sources and lower == upper - 'num_objects': '1000', - 'num_gets': '10000', - 'delete': 'yes', - 'container_name': uuid.uuid4().hex, # really "container name base" - 'num_containers': '20', - 'url': '', # used when use_proxy = no or overrides auth X-Storage-Url - 'account': '', # used when use_proxy = no - 'devices': 'sdb1', # space-sep list - 'log_level': 'INFO', - 'timeout': '10', - 'delay': '0', - 'bench_clients': [], -} - -SAIO_DEFAULTS = { - 'auth': 'http://localhost:8080/auth/v1.0', - 'user': 'test:tester', - 'key': 'testing', -} +from swiftbench import main if __name__ == '__main__': - usage = "usage: %prog [OPTIONS] [CONF_FILE]" - usage += """\n\nConf file with SAIO defaults: - - [bench] - auth = http://localhost:8080/auth/v1.0 - user = test:tester - key = testing - concurrency = 10 - object_size = 1 - num_objects = 1000 - num_gets = 10000 - delete = yes - auth_version = 1.0 - policy_name = gold - """ - parser = OptionParser(usage=usage) - parser.add_option('', '--saio', dest='saio', action='store_true', - default=False, help='Run benchmark with SAIO defaults') - parser.add_option('-A', '--auth', dest='auth', - help='URL for obtaining an auth token') - parser.add_option('-U', '--user', dest='user', - help='User name for obtaining an auth token') - parser.add_option('-K', '--key', dest='key', - help='Key for obtaining an auth token') - parser.add_option('-b', '--bench-clients', action='append', - metavar=':', - help=('A string of the form ":" which matches ' - 'the arguments supplied to a swift-bench-client ' - 'process. This argument must be specified ' - 'once per swift-bench-client you want to ' - 'utilize.')) - parser.add_option('-u', '--url', dest='url', - help='Storage URL') - parser.add_option('-c', '--concurrency', dest='concurrency', - help=('Number of concurrent connections to use. For ' - 'finer-grained control, see --get-concurrency, ' - '--put-concurrency, and --delete-concurrency.')) - parser.add_option('--get-concurrency', dest='get_concurrency', - help='Number of concurrent GET requests') - parser.add_option('--put-concurrency', dest='put_concurrency', - help='Number of concurrent PUT requests') - parser.add_option('--delete-concurrency', dest='delete_concurrency', - help='Number of concurrent DELETE requests') - parser.add_option('-s', '--object-size', dest='object_size', - help='Size of objects to PUT (in bytes)') - parser.add_option('-l', '--lower-object-size', dest='lower_object_size', - help=('Lower size of objects (in bytes); ' - '--object-size will be upper-object-size')) - parser.add_option('-n', '--num-objects', dest='num_objects', - help='Number of objects to PUT') - parser.add_option('-g', '--num-gets', dest='num_gets', - help='Number of GET operations to perform') - parser.add_option('-C', '--num-containers', dest='num_containers', - help='Number of containers to distribute objects among') - parser.add_option('-x', '--no-delete', dest='delete', action='store_false', - help='If set, will not delete the objects created') - parser.add_option('-V', '--auth_version', dest='auth_version', - help='Authentication version') - parser.add_option('-d', '--delay', dest='delay', - help='Delay before delete requests in seconds') - parser.add_option('-P', '--policy-name', dest='policy_name', - help='Specify which policy to use when creating ' - 'containers') - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - options, args = parser.parse_args() - if options.saio: - CONF_DEFAULTS.update(SAIO_DEFAULTS) - if getattr(options, 'lower_object_size', None): - if options.object_size <= options.lower_object_size: - raise ValueError('--lower-object-size (%s) must be ' - '< --object-size (%s)' % - (options.lower_object_size, options.object_size)) - CONF_DEFAULTS['upper_object_size'] = options.object_size - if args: - conf = args[0] - if not os.path.exists(conf): - sys.exit("No such conf file: %s" % conf) - conf = readconf(conf, 'bench', log_name='swift-bench', - defaults=CONF_DEFAULTS) - conf['bench_clients'] = [] - else: - conf = CONF_DEFAULTS - parser.set_defaults(**conf) - options, _junk = parser.parse_args() - - if options.concurrency is not '': - options.put_concurrency = options.concurrency - options.get_concurrency = options.concurrency - options.del_concurrency = options.concurrency - options.containers = ['%s_%d' % (options.container_name, i) - for i in range(int(options.num_containers))] - - # Turn "yes"/"no"/etc. strings to booleans - options.use_proxy = config_true_value(options.use_proxy) - options.delete = config_true_value(options.delete) - - def sigterm(signum, frame): - sys.exit('Termination signal received.') - signal.signal(signal.SIGTERM, sigterm) - - logger = logging.getLogger('swift-bench') - logger.propagate = False - logger.setLevel({ - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warning': logging.WARNING, - 'error': logging.ERROR, - 'critical': logging.CRITICAL}.get( - options.log_level.lower(), logging.INFO)) - loghandler = logging.StreamHandler() - logger.addHandler(loghandler) - logformat = logging.Formatter( - 'swift-bench %(asctime)s %(levelname)s %(message)s') - loghandler.setFormatter(logformat) - - if options.use_proxy: - create_containers(logger, options) - - controller_class = DistributedBenchController if options.bench_clients \ - else BenchController - controller = controller_class(logger, options) - controller.run() - - if options.use_proxy and options.delete: - delete_containers(logger, options) + sys.exit(main(sys.argv[1:])) diff --git a/swiftbench/cli/__init__.py b/swiftbench/cli/__init__.py new file mode 100644 index 0000000..44e8bef --- /dev/null +++ b/swiftbench/cli/__init__.py @@ -0,0 +1,190 @@ +# Copyright (c) 2022 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import sys +import signal +import uuid +from optparse import OptionParser + +from six.moves import range + +from swiftbench.bench import (BenchController, DistributedBenchController, + create_containers, delete_containers) +from swiftbench.utils import readconf, config_true_value + +# The defaults should be sufficient to run swift-bench on a SAIO +CONF_DEFAULTS = { + 'auth': os.environ.get('ST_AUTH', ''), + 'user': os.environ.get('ST_USER', ''), + 'key': os.environ.get('ST_KEY', ''), + 'auth_version': '1.0', + 'use_proxy': 'yes', + 'put_concurrency': '10', + 'get_concurrency': '10', + 'del_concurrency': '10', + 'concurrency': '', # set all 3 in one shot + 'object_sources': '', # set of file contents to read and use for PUTs + 'lower_object_size': '10', # bounded random size used if these differ + 'upper_object_size': '10', + 'object_size': '1', # only if not object_sources and lower == upper + 'num_objects': '1000', + 'num_gets': '10000', + 'delete': 'yes', + 'container_name': uuid.uuid4().hex, # really "container name base" + 'num_containers': '20', + 'url': '', # used when use_proxy = no or overrides auth X-Storage-Url + 'account': '', # used when use_proxy = no + 'devices': 'sdb1', # space-sep list + 'log_level': 'INFO', + 'timeout': '10', + 'delay': '0', + 'bench_clients': [], +} + +SAIO_DEFAULTS = { + 'auth': 'http://localhost:8080/auth/v1.0', + 'user': 'test:tester', + 'key': 'testing', +} + + +def main(argv): + usage = "usage: %prog [OPTIONS] [CONF_FILE]" + usage += """\n\nConf file with SAIO defaults: + + [bench] + auth = http://localhost:8080/auth/v1.0 + user = test:tester + key = testing + concurrency = 10 + object_size = 1 + num_objects = 1000 + num_gets = 10000 + delete = yes + auth_version = 1.0 + policy_name = gold + """ + parser = OptionParser(usage=usage) + parser.add_option('', '--saio', dest='saio', action='store_true', + default=False, help='Run benchmark with SAIO defaults') + parser.add_option('-A', '--auth', dest='auth', + help='URL for obtaining an auth token') + parser.add_option('-U', '--user', dest='user', + help='User name for obtaining an auth token') + parser.add_option('-K', '--key', dest='key', + help='Key for obtaining an auth token') + parser.add_option('-b', '--bench-clients', action='append', + metavar=':', + help=('A string of the form ":" which matches ' + 'the arguments supplied to a swift-bench-client ' + 'process. This argument must be specified ' + 'once per swift-bench-client you want to ' + 'utilize.')) + parser.add_option('-u', '--url', dest='url', + help='Storage URL') + parser.add_option('-c', '--concurrency', dest='concurrency', + help=('Number of concurrent connections to use. For ' + 'finer-grained control, see --get-concurrency, ' + '--put-concurrency, and --delete-concurrency.')) + parser.add_option('--get-concurrency', dest='get_concurrency', + help='Number of concurrent GET requests') + parser.add_option('--put-concurrency', dest='put_concurrency', + help='Number of concurrent PUT requests') + parser.add_option('--delete-concurrency', dest='delete_concurrency', + help='Number of concurrent DELETE requests') + parser.add_option('-s', '--object-size', dest='object_size', + help='Size of objects to PUT (in bytes)') + parser.add_option('-l', '--lower-object-size', dest='lower_object_size', + help=('Lower size of objects (in bytes); ' + '--object-size will be upper-object-size')) + parser.add_option('-n', '--num-objects', dest='num_objects', + help='Number of objects to PUT') + parser.add_option('-g', '--num-gets', dest='num_gets', + help='Number of GET operations to perform') + parser.add_option('-C', '--num-containers', dest='num_containers', + help='Number of containers to distribute objects among') + parser.add_option('-x', '--no-delete', dest='delete', action='store_false', + help='If set, will not delete the objects created') + parser.add_option('-V', '--auth_version', dest='auth_version', + help='Authentication version') + parser.add_option('-d', '--delay', dest='delay', + help='Delay before delete requests in seconds') + parser.add_option('-P', '--policy-name', dest='policy_name', + help='Specify which policy to use when creating ' + 'containers') + + options, args = parser.parse_args(argv) + if options.saio: + CONF_DEFAULTS.update(SAIO_DEFAULTS) + if getattr(options, 'lower_object_size', None): + if options.object_size <= options.lower_object_size: + raise ValueError('--lower-object-size (%s) must be ' + '< --object-size (%s)' % + (options.lower_object_size, options.object_size)) + CONF_DEFAULTS['upper_object_size'] = options.object_size + if args: + conf = args[0] + if not os.path.exists(conf): + sys.exit("No such conf file: %s" % conf) + conf = readconf(conf, 'bench', log_name='swift-bench', + defaults=CONF_DEFAULTS) + conf['bench_clients'] = [] + else: + conf = CONF_DEFAULTS + parser.set_defaults(**conf) + options, _junk = parser.parse_args(argv) + + if options.concurrency != '': + options.put_concurrency = options.concurrency + options.get_concurrency = options.concurrency + options.delete_concurrency = options.concurrency + options.containers = ['%s_%d' % (options.container_name, i) + for i in range(int(options.num_containers))] + + # Turn "yes"/"no"/etc. strings to booleans + options.use_proxy = config_true_value(options.use_proxy) + options.delete = config_true_value(options.delete) + + def sigterm(signum, frame): + sys.exit('Termination signal received.') + signal.signal(signal.SIGTERM, sigterm) + + logger = logging.getLogger('swift-bench') + logger.propagate = False + logger.setLevel({ + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL}.get( + options.log_level.lower(), logging.INFO)) + loghandler = logging.StreamHandler() + logger.addHandler(loghandler) + logformat = logging.Formatter( + 'swift-bench %(asctime)s %(levelname)s %(message)s') + loghandler.setFormatter(logformat) + + if options.use_proxy: + create_containers(logger, options) + + controller_class = DistributedBenchController if options.bench_clients \ + else BenchController + controller = controller_class(logger, options) + controller.run() + + if options.use_proxy and options.delete: + delete_containers(logger, options) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..84c65a4 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,169 @@ +# Copyright (c) 2022 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import mock +from swiftbench import cli + + +class TestCli(unittest.TestCase): + + def setUp(self): + self.container_options = None + self.delete_options = None + + def fake_create_containers(self, logger, options): + self.container_options = options + + def fake_delete_containers(self, logger, options): + self.delete_options = options + + def run_main(self, args): + mock_controller = mock.Mock() + + with mock.patch('swiftbench.cli.create_containers', + self.fake_create_containers), \ + mock.patch('swiftbench.cli.delete_containers', + self.fake_delete_containers), \ + mock.patch('swiftbench.cli.BenchController', + mock_controller), \ + mock.patch('swiftbench.cli.DistributedBenchController', + mock_controller): + cli.main(args) + return (mock_controller.call_args[0][-1], self.container_options, + self.delete_options) + + def test_defaults(self): + controller_opts, container_opts, del_opts = self.run_main([]) + self.assertFalse(controller_opts.saio) + self.assertEqual(controller_opts.auth, '') + self.assertEqual(controller_opts.user, '') + self.assertEqual(controller_opts.key, '') + self.assertEqual(controller_opts.url, '') + self.assertEqual(controller_opts.concurrency, '') + self.assertEqual(controller_opts.get_concurrency, '10') + self.assertEqual(controller_opts.put_concurrency, '10') + self.assertEqual(controller_opts.lower_object_size, '10') + self.assertEqual(controller_opts.upper_object_size, '10') + self.assertEqual(controller_opts.num_objects, '1000') + self.assertEqual(controller_opts.num_gets, '10000') + self.assertEqual(controller_opts.num_containers, '20') + self.assertEqual(len(controller_opts.containers), 20) + self.assertTrue(controller_opts.delete) + self.assertEqual(controller_opts.auth_version, '1.0') + self.assertEqual(controller_opts.delay, '0') + self.assertIsNone(controller_opts.policy_name) + self.assertTrue(controller_opts.use_proxy) + self.assertEqual(controller_opts.delete_concurrency, '10') + self.assertEqual(controller_opts.object_sources, '') + self.assertEqual(controller_opts.account, '') + self.assertEqual(controller_opts.container_name, mock.ANY) + self.assertEqual(controller_opts.devices, 'sdb1') + self.assertEqual(controller_opts.log_level, 'INFO') + self.assertEqual(controller_opts.timeout, '10') + self.assertEqual(controller_opts.bench_clients, []) + self.assertTrue(controller_opts.containers) + + def test_defaults_with_saio(self): + controller_opts, container_opts, del_opts = self.run_main(['--saio']) + self.assertFalse(controller_opts.saio) + self.assertEqual(controller_opts.auth, + 'http://localhost:8080/auth/v1.0') + self.assertEqual(controller_opts.user, 'test:tester') + self.assertEqual(controller_opts.key, 'testing') + self.assertEqual(controller_opts.url, '') + self.assertEqual(controller_opts.concurrency, '') + self.assertEqual(controller_opts.get_concurrency, '10') + self.assertEqual(controller_opts.put_concurrency, '10') + self.assertEqual(controller_opts.lower_object_size, '10') + self.assertEqual(controller_opts.upper_object_size, '10') + self.assertEqual(controller_opts.num_objects, '1000') + self.assertEqual(controller_opts.num_gets, '10000') + self.assertEqual(controller_opts.num_containers, '20') + self.assertEqual(len(controller_opts.containers), 20) + self.assertTrue(controller_opts.delete) + self.assertEqual(controller_opts.auth_version, '1.0') + self.assertEqual(controller_opts.delay, '0') + self.assertIsNone(controller_opts.policy_name) + self.assertTrue(controller_opts.use_proxy) + self.assertEqual(controller_opts.delete_concurrency, '10') + self.assertEqual(controller_opts.object_sources, '') + self.assertEqual(controller_opts.account, '') + self.assertEqual(controller_opts.container_name, mock.ANY) + self.assertEqual(controller_opts.devices, 'sdb1') + self.assertEqual(controller_opts.log_level, 'INFO') + self.assertEqual(controller_opts.timeout, '10') + self.assertEqual(controller_opts.bench_clients, []) + self.assertTrue(controller_opts.containers) + + def test_cli_options(self): + controller_opts, container_opts, del_opts = self.run_main( + [ + '--auth', 'http://some_url/auth/v1.0', + '-U', 'other:user', + '--key', 'my_key', + '--bench-clients', '1.2.3.4:1234', + '--url', 'http://storage.url/v1/AUTH_user', + '--get-concurrency', '9', + '--put-concurrency', '8', + '--delete-concurrency', '7', + '--object-size', '999', + '--lower-object-size', '1', + '--num-objects', '5000', + '--num-gets', '4000', + '--num-containers', '4', + '-x', + '--auth_version', '2.0', + '--delay', '10', + '--policy-name', 'gold']) + self.assertFalse(controller_opts.saio) + self.assertEqual(controller_opts.auth, 'http://some_url/auth/v1.0') + self.assertEqual(controller_opts.user, 'other:user') + self.assertEqual(controller_opts.key, 'my_key') + self.assertEqual(controller_opts.bench_clients, ['1.2.3.4:1234']) + self.assertEqual(controller_opts.url, + 'http://storage.url/v1/AUTH_user') + self.assertEqual(controller_opts.get_concurrency, '9') + self.assertEqual(controller_opts.put_concurrency, '8') + self.assertEqual(controller_opts.delete_concurrency, '7') + self.assertEqual(controller_opts.object_size, '999') + self.assertEqual(controller_opts.lower_object_size, '1') + self.assertEqual(controller_opts.num_objects, '5000') + self.assertEqual(controller_opts.num_gets, '4000') + self.assertEqual(controller_opts.num_containers, '4') + self.assertEqual(len(controller_opts.containers), 4) + self.assertFalse(controller_opts.delete) + self.assertEqual(controller_opts.auth_version, '2.0') + self.assertEqual(controller_opts.delay, '10') + self.assertEqual(controller_opts.policy_name, 'gold') + self.assertTrue(controller_opts.use_proxy) + self.assertEqual(controller_opts.object_sources, '') + self.assertEqual(controller_opts.account, '') + self.assertEqual(controller_opts.container_name, mock.ANY) + self.assertEqual(controller_opts.devices, 'sdb1') + self.assertEqual(controller_opts.log_level, 'INFO') + self.assertEqual(controller_opts.timeout, '10') + self.assertTrue(controller_opts.containers) + + def test_concurrency_overrides_get_put_delete(self): + controller_opts, container_opts, del_opts = self.run_main( + [ + '--concurrency', '5', + '--get-concurrency', '9', + '--put-concurrency', '8', + '--delete-concurrency', '7']) + self.assertEqual(controller_opts.concurrency, '5') + self.assertEqual(controller_opts.get_concurrency, '5') + self.assertEqual(controller_opts.put_concurrency, '5') + self.assertEqual(controller_opts.delete_concurrency, '5')