Merge "Make zaqar-bench use credentials from os_client_config"

bench-requirements.txt

@@ -2,3 +2,4 @@ argparse>=1.2.1
 gevent>=1.0.1
 marktime>=0.2.0
 python-zaqarclient>=0.0.2
+os-client-config!=1.6.2,>=1.4.0

zaqar/bench/conductor.py

@@ -16,6 +16,10 @@ from __future__ import print_function
 
 import json
 import multiprocessing as mp
+import os
+# NOTE(Eva-i): See https://github.com/gevent/gevent/issues/349. Let's keep
+# it until the new stable version of gevent(>=1.1) will be released.
+os.environ["GEVENT_RESOLVER"] = "ares"
 
 from zaqar.bench import config
 from zaqar.bench import consumer
@@ -39,11 +43,8 @@ def _print_verbose_stats(name, stats):
 
 def _reset_queues():
     cli = helpers.get_new_client()
-
+    for queue_name in helpers.queue_names:
-    for i in range(CONF.num_queues):
+        queue = cli.queue(queue_name)
-        # TODO(kgriffs): DRY up name generation so it is done
-        # in a helper, vs. being copy-pasted everywhere.
-        queue = cli.queue(CONF.queue_prefix + '-' + str(i))
         queue.delete()
 
 

zaqar/bench/config.py

@@ -49,11 +49,14 @@ _CLI_OPTIONS = (
         default=5,
         help='Number of Observer Workers'),
 
+    cfg.FloatOpt('api_version', short='api', default='2',
+                 help='Zaqar API version to use'),
+
     cfg.IntOpt('messages_per_claim', short='cno', default=5,
                help=('Number of messages the consumer will attempt to '
                      'claim at a time')),
     cfg.IntOpt('messages_per_list', short='lno', default=5,
-               help=('Number of messages the obserer will attempt to '
+               help=('Number of messages the observer will attempt to '
                      'list at a time')),
 
     cfg.IntOpt('time', short='t', default=5,

zaqar/bench/consumer.py

@@ -96,8 +96,9 @@ def load_generator(stats, num_workers, num_queues,
                    test_duration, url, ttl, grace, limit):
 
     cli = helpers.get_new_client()
-    queues = [cli.queue(CONF.queue_prefix + '-' + str(i))
+    queues = []
-              for i in range(num_queues)]
+    for queue_name in helpers.queue_names:
+        queues.append(cli.queue(queue_name))
 
     gevent.joinall([
         gevent.spawn(claim_delete,

zaqar/bench/helpers.py

@@ -11,21 +11,126 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# NOTE(Eva-i): Some code was taken from python-zaqarclient.
+
+import os
+import sys
+
+import os_client_config
+from six.moves import urllib_parse
 from zaqarclient.queues import client
 
 from zaqar.bench import config
 
 CONF = config.conf
 
-client_conf = {
+
+def _get_credential_args():
+    """Retrieves credential arguments for keystone
+
+    Credentials are either read via os-client-config from the environment
+    or from a config file ('clouds.yaml'). Config file variables override those
+    from the environment variables.
+
+    devstack produces a clouds.yaml with two named clouds - one named
+    'devstack' which has user privs and one named 'devstack-admin' which
+    has admin privs. This function will default to getting the credentials from
+    environment variables. If not all required credentials present in
+    environment variables, it tries to get credentials for 'devstack-admin'
+    cloud in clouds.yaml. If no 'devstack-admin' cloud found, it tried to get
+    credentials for 'devstack' cloud. If no 'devstack' cloud found, throws
+    an error and stops the application.
+    """
+    os_cfg = os_client_config.OpenStackConfig()
+
+    cloud = os_cfg.get_one_cloud()
+    cred_args = cloud.get_auth_args()
+
+    required_options = ['username', 'password', 'auth_url', 'project_name']
+    if not all(arg in cred_args for arg in required_options):
+        try:
+            cloud = os_cfg.get_one_cloud(cloud='devstack-admin')
+        except Exception:
+            try:
+                cloud = os_cfg.get_one_cloud(cloud='devstack')
+            except Exception:
+                print("Insufficient amount of credentials found for keystone "
+                      "authentication. Credentials should reside either in "
+                      "environment variables or in 'clouds.yaml' file. If "
+                      "both present, the ones in environment variables will "
+                      "be preferred. Exiting.")
+                sys.exit()
+        cred_args = cloud.get_auth_args()
+
+    print("Using '{}' credentials".format(cloud.name))
+    return cred_args
+
+
+def _generate_client_conf():
+    auth_strategy = os.environ.get
+    if auth_strategy == 'keystone':
+        args = _get_credential_args()
+        # FIXME(flwang): Now we're hardcode the keystone auth version, since
+        # there is a 'bug' with the osc-config which is returning the auth_url
+        # without version. This should be fixed as long as the bug is fixed.
+        parsed_url = urllib_parse.urlparse(args['auth_url'])
+        auth_url = args['auth_url']
+        if not parsed_url.path or parsed_url.path == '/':
+            auth_url = urllib_parse.urljoin(args['auth_url'], 'v2.0')
+        conf = {
+            'auth_opts': {
+                'backend': 'keystone',
+                'options': {
+                    'os_username': args['username'],
+                    'os_password': args['password'],
+                    'os_project_name': args['project_name'],
+                    'os_auth_url': auth_url,
+                    'insecure': '',
+                },
+            },
+        }
+    else:
+        conf = {
             'auth_opts': {
                 'backend': 'noauth',
                 'options': {
                     'os_project_id': 'my-lovely-benchmark',
                 },
             },
-}
+        }
+    print("Using '{0}' authentication method".format(conf['auth_opts']
+                                                     ['backend']))
+    return conf
+
+
+class LazyAPIVersion(object):
+    def __init__(self):
+        self.api_version = None
+
+    @property
+    def get(self):
+        if self.api_version is None:
+            conversion_map = {
+                1.0: 1,
+                1.1: 1.1,
+                2.0: 2,
+            }
+            try:
+                self.api_version = conversion_map[CONF.api_version]
+            except KeyError:
+                print("Unknown Zaqar API version: '{}'. Exiting...".format(
+                      CONF.api_version))
+                sys.exit()
+            print("Benchmarking Zaqar API v{0}...".format(self.api_version))
+        return self.api_version
+
+
+client_conf = _generate_client_conf()
+client_api = LazyAPIVersion()
+queue_names = []
+for i in range(CONF.num_queues):
+    queue_names.append((CONF.queue_prefix + '-' + str(i)))
 
 
 def get_new_client():
-    return client.Client(CONF.server_url, 1.1, conf=client_conf)
+    return client.Client(CONF.server_url, client_api.get, conf=client_conf)

zaqar/bench/observer.py

@@ -100,8 +100,9 @@ def load_generator(stats, num_workers, num_queues,
                    test_duration, limit):
 
     cli = helpers.get_new_client()
-    queues = [cli.queue(CONF.queue_prefix + '-' + str(i))
+    queues = []
-              for i in range(num_queues)]
+    for queue_name in helpers.queue_names:
+        queues.append(cli.queue(queue_name))
 
     gevent.joinall([
         gevent.spawn(observer,

zaqar/bench/producer.py

@@ -105,8 +105,9 @@ def producer(queues, message_pool, stats, test_duration):
 def load_generator(stats, num_workers, num_queues, test_duration):
 
     cli = helpers.get_new_client()
-    queues = [cli.queue(CONF.queue_prefix + '-' + str(i))
+    queues = []
-              for i in range(num_queues)]
+    for queue_name in helpers.queue_names:
+        queues.append(cli.queue(queue_name))
 
     message_pool = load_messages()