From 1b8334b35a0d01b828ba975ab7d778f403cc7932 Mon Sep 17 00:00:00 2001 From: Eva Balycheva Date: Sat, 28 Nov 2015 07:42:09 +0300 Subject: [PATCH] Make zaqar-bench use credentials from os_client_config Currently zaqar-bench can't benchmark Zaqar while Zaqar is using keystone authentication backend. This patch makes it possible by using os_client_config library. The library gets authentication parameters from "clouds.yaml". If keystone authentication environment variables present, they override values from "clouds.yaml". The aquired parameters then passed to constructors of each python-zaqarclient's Client object used in zaqar-bench. The patch also makes benchmark queue names reusable across all zaqar-bench parts and by this fixes the old DRY princible bug. To use zaqar-bench with keystone authentication the user must explicitly set OS_AUTH_STRATEGY=keystone as environment variable before running the tool. Otherwise the default 'noauth' auth strategy will be used. This allows the user to run zaqar-bench as usual. This patch also adds option "--api-version" with it's short version "api" to zaqar-bench which defaults to Zaqar API v2. Change-Id: I0a7aaeaeac6da1b2c9f08fbfdddd467de5747a28 Closes-Bug: 1523752 --- bench-requirements.txt | 1 + zaqar/bench/conductor.py | 11 ++-- zaqar/bench/config.py | 5 +- zaqar/bench/consumer.py | 5 +- zaqar/bench/helpers.py | 123 ++++++++++++++++++++++++++++++++++++--- zaqar/bench/observer.py | 5 +- zaqar/bench/producer.py | 5 +- 7 files changed, 134 insertions(+), 21 deletions(-) diff --git a/bench-requirements.txt b/bench-requirements.txt index 7b0d1dd2..d2507dbd 100644 --- a/bench-requirements.txt +++ b/bench-requirements.txt @@ -2,3 +2,4 @@ argparse>=1.2.1 gevent>=1.0.1 marktime>=0.2.0 python-zaqarclient>=0.0.2 +os-client-config!=1.6.2,>=1.4.0 \ No newline at end of file diff --git a/zaqar/bench/conductor.py b/zaqar/bench/conductor.py index 274c6b10..6feeb227 100644 --- a/zaqar/bench/conductor.py +++ b/zaqar/bench/conductor.py @@ -16,6 +16,10 @@ from __future__ import print_function import json import multiprocessing as mp +import os +# NOTE(Eva-i): See https://github.com/gevent/gevent/issues/349. Let's keep +# it until the new stable version of gevent(>=1.1) will be released. +os.environ["GEVENT_RESOLVER"] = "ares" from zaqar.bench import config from zaqar.bench import consumer @@ -39,11 +43,8 @@ def _print_verbose_stats(name, stats): def _reset_queues(): cli = helpers.get_new_client() - - for i in range(CONF.num_queues): - # TODO(kgriffs): DRY up name generation so it is done - # in a helper, vs. being copy-pasted everywhere. - queue = cli.queue(CONF.queue_prefix + '-' + str(i)) + for queue_name in helpers.queue_names: + queue = cli.queue(queue_name) queue.delete() diff --git a/zaqar/bench/config.py b/zaqar/bench/config.py index b3a476d0..49cc28d2 100644 --- a/zaqar/bench/config.py +++ b/zaqar/bench/config.py @@ -49,11 +49,14 @@ _CLI_OPTIONS = ( default=5, help='Number of Observer Workers'), + cfg.FloatOpt('api_version', short='api', default='2', + help='Zaqar API version to use'), + cfg.IntOpt('messages_per_claim', short='cno', default=5, help=('Number of messages the consumer will attempt to ' 'claim at a time')), cfg.IntOpt('messages_per_list', short='lno', default=5, - help=('Number of messages the obserer will attempt to ' + help=('Number of messages the observer will attempt to ' 'list at a time')), cfg.IntOpt('time', short='t', default=5, diff --git a/zaqar/bench/consumer.py b/zaqar/bench/consumer.py index bd2bb18a..06f7dcdd 100644 --- a/zaqar/bench/consumer.py +++ b/zaqar/bench/consumer.py @@ -96,8 +96,9 @@ def load_generator(stats, num_workers, num_queues, test_duration, url, ttl, grace, limit): cli = helpers.get_new_client() - queues = [cli.queue(CONF.queue_prefix + '-' + str(i)) - for i in range(num_queues)] + queues = [] + for queue_name in helpers.queue_names: + queues.append(cli.queue(queue_name)) gevent.joinall([ gevent.spawn(claim_delete, diff --git a/zaqar/bench/helpers.py b/zaqar/bench/helpers.py index a31499c2..8cc3ede6 100644 --- a/zaqar/bench/helpers.py +++ b/zaqar/bench/helpers.py @@ -11,21 +11,126 @@ # See the License for the specific language governing permissions and # limitations under the License. +# NOTE(Eva-i): Some code was taken from python-zaqarclient. + +import os +import sys + +import os_client_config +from six.moves import urllib_parse from zaqarclient.queues import client from zaqar.bench import config CONF = config.conf -client_conf = { - 'auth_opts': { - 'backend': 'noauth', - 'options': { - 'os_project_id': 'my-lovely-benchmark', - }, - }, -} + +def _get_credential_args(): + """Retrieves credential arguments for keystone + + Credentials are either read via os-client-config from the environment + or from a config file ('clouds.yaml'). Config file variables override those + from the environment variables. + + devstack produces a clouds.yaml with two named clouds - one named + 'devstack' which has user privs and one named 'devstack-admin' which + has admin privs. This function will default to getting the credentials from + environment variables. If not all required credentials present in + environment variables, it tries to get credentials for 'devstack-admin' + cloud in clouds.yaml. If no 'devstack-admin' cloud found, it tried to get + credentials for 'devstack' cloud. If no 'devstack' cloud found, throws + an error and stops the application. + """ + os_cfg = os_client_config.OpenStackConfig() + + cloud = os_cfg.get_one_cloud() + cred_args = cloud.get_auth_args() + + required_options = ['username', 'password', 'auth_url', 'project_name'] + if not all(arg in cred_args for arg in required_options): + try: + cloud = os_cfg.get_one_cloud(cloud='devstack-admin') + except Exception: + try: + cloud = os_cfg.get_one_cloud(cloud='devstack') + except Exception: + print("Insufficient amount of credentials found for keystone " + "authentication. Credentials should reside either in " + "environment variables or in 'clouds.yaml' file. If " + "both present, the ones in environment variables will " + "be preferred. Exiting.") + sys.exit() + cred_args = cloud.get_auth_args() + + print("Using '{}' credentials".format(cloud.name)) + return cred_args + + +def _generate_client_conf(): + auth_strategy = os.environ.get + if auth_strategy == 'keystone': + args = _get_credential_args() + # FIXME(flwang): Now we're hardcode the keystone auth version, since + # there is a 'bug' with the osc-config which is returning the auth_url + # without version. This should be fixed as long as the bug is fixed. + parsed_url = urllib_parse.urlparse(args['auth_url']) + auth_url = args['auth_url'] + if not parsed_url.path or parsed_url.path == '/': + auth_url = urllib_parse.urljoin(args['auth_url'], 'v2.0') + conf = { + 'auth_opts': { + 'backend': 'keystone', + 'options': { + 'os_username': args['username'], + 'os_password': args['password'], + 'os_project_name': args['project_name'], + 'os_auth_url': auth_url, + 'insecure': '', + }, + }, + } + else: + conf = { + 'auth_opts': { + 'backend': 'noauth', + 'options': { + 'os_project_id': 'my-lovely-benchmark', + }, + }, + } + print("Using '{0}' authentication method".format(conf['auth_opts'] + ['backend'])) + return conf + + +class LazyAPIVersion(object): + def __init__(self): + self.api_version = None + + @property + def get(self): + if self.api_version is None: + conversion_map = { + 1.0: 1, + 1.1: 1.1, + 2.0: 2, + } + try: + self.api_version = conversion_map[CONF.api_version] + except KeyError: + print("Unknown Zaqar API version: '{}'. Exiting...".format( + CONF.api_version)) + sys.exit() + print("Benchmarking Zaqar API v{0}...".format(self.api_version)) + return self.api_version + + +client_conf = _generate_client_conf() +client_api = LazyAPIVersion() +queue_names = [] +for i in range(CONF.num_queues): + queue_names.append((CONF.queue_prefix + '-' + str(i))) def get_new_client(): - return client.Client(CONF.server_url, 1.1, conf=client_conf) + return client.Client(CONF.server_url, client_api.get, conf=client_conf) diff --git a/zaqar/bench/observer.py b/zaqar/bench/observer.py index 9a2e5f4e..db8752e7 100644 --- a/zaqar/bench/observer.py +++ b/zaqar/bench/observer.py @@ -100,8 +100,9 @@ def load_generator(stats, num_workers, num_queues, test_duration, limit): cli = helpers.get_new_client() - queues = [cli.queue(CONF.queue_prefix + '-' + str(i)) - for i in range(num_queues)] + queues = [] + for queue_name in helpers.queue_names: + queues.append(cli.queue(queue_name)) gevent.joinall([ gevent.spawn(observer, diff --git a/zaqar/bench/producer.py b/zaqar/bench/producer.py index 8bcb9422..248ee52f 100644 --- a/zaqar/bench/producer.py +++ b/zaqar/bench/producer.py @@ -105,8 +105,9 @@ def producer(queues, message_pool, stats, test_duration): def load_generator(stats, num_workers, num_queues, test_duration): cli = helpers.get_new_client() - queues = [cli.queue(CONF.queue_prefix + '-' + str(i)) - for i in range(num_queues)] + queues = [] + for queue_name in helpers.queue_names: + queues.append(cli.queue(queue_name)) message_pool = load_messages()