Finish support of benchmarking with existing users

Now one can use Rally for benchmarking OpenStack clouds that are using
LDAP, AD or any other read only keystone backend where you are not able
to create any users.

To do this one can specify "users" section in ExistingCloud plugin
input file.

Rally will use these "users" to generate load instead of creating new.
As well if you specify both "users" section in deployment input file
and "users" context it will use temporary created by Rally users.

Restructurize samples/deployments directory:

  - Deploying OpenStack clouds with Rally is not popular functionallity
    so hide all samples deeper.
  - Some fixes in readme.rst
  - Add new sample for existing cloud with predefined users

Change-Id: If3d31960ee317e0770abceacc62b3acf0947d269

rally/benchmark/context/existing_users.py

@@ -0,0 +1,70 @@
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from rally.benchmark.context import base
+from rally.common.i18n import _
+from rally.common import log as logging
+from rally.common import utils as rutils
+from rally import objects
+from rally import osclients
+
+
+LOG = logging.getLogger(__name__)
+
+
+# NOTE(boris-42): This context should be hidden for now and used only by
+#                 benchmark engine.  In future during various refactoring of
+#                 validation system and rally CI testing we will make it public
+@base.context(name="existing_users", order=99, hidden=True)
+class ExistingUsers(base.Context):
+    """This context supports using existing users in Rally.
+
+       It uses information about deployment to properly
+       initialize context["users"] and context["tenants"]
+
+       So there won't be big difference between usage of "users" and
+       "existing_users" context.
+    """
+
+    # NOTE(boris-42): We don't need to check config schema because
+    #                 this is used only by benchmark engine
+    CONFIG_SCHEMA = {}
+
+    def __init__(self, context):
+        super(ExistingUsers, self).__init__(context)
+        self.context["users"] = []
+        self.context["tenants"] = {}
+
+    @rutils.log_task_wrapper(LOG.info, _("Enter context: `existing_users`"))
+    def setup(self):
+        for user in self.config:
+            user_endpoint = objects.Endpoint(**user)
+            user_kclient = osclients.Clients(user_endpoint).keystone()
+
+            if user_kclient.tenant_id not in self.context["tenants"]:
+                self.context["tenants"][user_kclient.tenant_id] = {
+                    "id": user_kclient.tenant_id,
+                    "name": user_kclient.tenant_name
+                }
+
+            self.context["users"].append({
+                "endpoint": user_endpoint,
+                "id": user_kclient.user_id,
+                "tenant_id": user_kclient.tenant_id
+            })
+
+    @rutils.log_task_wrapper(LOG.info, _("Exit context: `existing_users`"))
+    def cleanup(self):
+        """These users are not managed by Rally, so don't touch them."""

rally/benchmark/engine.py

@@ -22,6 +22,7 @@ import jsonschema
 import six
 
 from rally.benchmark.context import base as base_ctx
+from rally.benchmark.context import existing_users as existing_users_ctx
 from rally.benchmark.context import users as users_ctx
 from rally.benchmark.runners import base as base_runner
 from rally.benchmark.scenarios import base as base_scenario
@@ -107,7 +108,7 @@ class BenchmarkEngine(object):
         self.config = config
         self.task = task
         self.admin = admin and objects.Endpoint(**admin) or None
-        self.users = map(lambda u: objects.Endpoint(**u), users or [])
+        self.existing_users = users or []
         self.abort_on_sla_failure = abort_on_sla_failure
 
     @rutils.log_task_wrapper(LOG.info, _("Task validation check cloud."))
@@ -153,25 +154,38 @@ class BenchmarkEngine(object):
                   "config": kwargs, "reason": six.text_type(e)}
             raise exceptions.InvalidBenchmarkConfig(**kw)
 
+    def _get_user_ctx_for_validation(self, context):
+        if self.existing_users:
+            context["config"] = {"existing_users": self.existing_users}
+            user_context = existing_users_ctx.ExistingUsers(context)
+        else:
+            user_context = users_ctx.UserGenerator(context)
+
+        return user_context
+
     @rutils.log_task_wrapper(LOG.info, _("Task validation of semantic."))
     def _validate_config_semantic(self, config):
         self._check_cloud()
 
-        # NOTE(boris-42): In future we will have more complex context, because
-        #                 we will have pre-created users mode as well.
         context = {"task": self.task, "admin": {"endpoint": self.admin}}
         deployment = objects.Deployment.get(self.task["deployment_uuid"])
 
-        with users_ctx.UserGenerator(context) as ctx:
+        # TODO(boris-42): It's quite hard at the moment to validate case
+        #                 when both user context and existing_users are
+        #                 specified. So after switching to plugin base
+        #                 and refactoring validation mechanism this place
+        #                 will be replaced
+        with self._get_user_ctx_for_validation(context) as ctx:
             ctx.setup()
             admin = osclients.Clients(self.admin)
             user = osclients.Clients(context["users"][0]["endpoint"])
 
-            for name, values in six.iteritems(config):
-                for pos, kwargs in enumerate(values):
-                    self._validate_config_semantic_helper(admin, user, name,
-                                                          pos, deployment,
-                                                          kwargs)
+            for u in context["users"]:
+                user = osclients.Clients(u["endpoint"])
+                for name, values in six.iteritems(config):
+                    for pos, kwargs in enumerate(values):
+                        self._validate_config_semantic_helper(
+                            admin, user, name, pos, deployment, kwargs)
 
     @rutils.log_task_wrapper(LOG.info, _("Task validation."))
     def validate(self):
@@ -195,7 +209,11 @@ class BenchmarkEngine(object):
 
     def _prepare_context(self, context, name, endpoint):
         scenario_context = base_scenario.Scenario.meta(name, "context")
-        scenario_context.setdefault("users", {})
+        if self.existing_users and "users" not in context:
+            scenario_context.setdefault("existing_users", self.existing_users)
+        elif "users" not in context:
+            scenario_context.setdefault("users", {})
+
         scenario_context.update(context)
         context_obj = {
             "task": self.task,

samples/deployments/README.rst

@@ -1,12 +1,21 @@
 Rally Deployments
 =================
 
-Before starting cluster benchmarking, its connection parameters
-should be saved in Rally database (deployment record).
+Rally needs to be configured to use an OpenStack Cloud deployment before it
+can benchmark the deployment.
 
-If there is no cluster, rally also can create it.
+To configure Rally to use an OpenStack Cloud deployment, you need create a
+deployment configuration by supplying the endpoint and credentials, as follows:
+
+.. code-block::
+
+    rally deployment create --file <one_of_files_from_this_dir> --name my_cloud
+
+
+If you don't have OpenStack deployments, Rally can deploy it for you.
+For samples of various deployments take a look at samples from
+**for_deploying_openstack_with_rally** directory.
 
-There are examples of deployment configurations:
 
 existing.json
 -------------
@@ -18,6 +27,15 @@ existing-keystone-v3.json
 
 Register existing OpenStack cluster that uses Keystone v3.
 
+existing-with-predefined-users.json
+--------------------------------------
+
+If you are using read-only backend in Keystone like LDAP, AD then
+you need this sample. If you don't specify "users" rally will use already
+existing users that you provide.
+
+
+
 existing-with-given-endpoint.json
 ---------------------------------
 
@@ -26,33 +44,3 @@ to explicitly set keystone management_url. Use this parameter if
 keystone fails to setup management_url correctly.
 For example, this parameter must be specified for FUEL cluster
 and has value "http://<identity-public-url-ip>:35357/v2.0/"
-
-devstack-in-existing-servers.json
----------------------------------
-
-Register existing DevStack cluster.
-
-devstack-in-lxc.json
---------------------
-
-Deploy DevStack cluster on LXC and register it by Rally.
-
-devstack-in-openstack.json
---------------------------
-
-Deploy DevStack cluster on OpenStack and register it by Rally.
-
-devstack-lxc-engine-in-existing-servers.json
---------------------------------------------
-
-See *devstack-lxc-engine-in-existing-servers.rst* for details
-
-fuel-ha.json
-------------
-
-Deploy High Availability FUEL cluster and register it by Rally.
-
-fuel-multinode.json
--------------------
-
-Deploy Multinode FUEL cluster and register it by Rally.

samples/deployments/existing-with-predefined-users.json

@@ -0,0 +1,23 @@
+{
+    "type": "ExistingCloud",
+    "auth_url": "http://example.net:5000/v2.0/",
+    "region_name": "RegionOne",
+    "endpoint_type": "public",
+    "admin": {
+        "username": "admin",
+        "password": "pa55word",
+        "tenant_name": "demo"
+    },
+    "users": [
+        {
+            "username": "not_an_admin1",
+            "password": "password",
+            "tenant_name": "some_tenant"
+        },
+        {
+            "username": "not_an_admin2",
+            "password": "password2",
+            "tenant_name": "some_tenant2"
+        }
+    ]
+}

samples/deployments/for_deploying_openstack_with_rally/README.rst

@@ -0,0 +1,40 @@
+Using Rally for OpenStack deployment
+====================================
+
+This directory contains various input files for **rally deployment create**
+command. This command will deploy OpenStack and register it to Rally DB.
+
+
+devstack-in-existing-servers.json
+---------------------------------
+
+Deploy with DevStack OpenStack cloud on specified servers.
+
+
+devstack-in-lxc.json
+--------------------
+
+Deploy DevStack cluster in LXC containers
+
+
+devstack-in-openstack.json
+--------------------------
+
+Create require VM in specified OpenStack cloud and deploy using DevStack
+OpenStack on them.
+
+
+devstack-lxc-engine-in-existing-servers.json
+--------------------------------------------
+
+See *devstack-lxc-engine-in-existing-servers.rst* for details
+
+fuel-ha.json
+------------
+
+Deploy High Availability FUEL cluster.
+
+fuel-multinode.json
+-------------------
+
+Deploy Multinode FUEL cluster.

tests/unit/benchmark/context/test_existing_users.py

@@ -0,0 +1,62 @@
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+
+from rally.benchmark.context import existing_users
+from tests.unit import test
+
+
+class ExistingUserTestCase(test.TestCase):
+
+    @mock.patch("rally.benchmark.context.existing_users.osclients.Clients")
+    @mock.patch("rally.benchmark.context.existing_users.objects.Endpoint")
+    def test_setup(self, mock_endpoint, mock_osclients):
+        user1 = mock.MagicMock(tenant_id="1")
+        user2 = mock.MagicMock(tenant_id="1")
+        user3 = mock.MagicMock(tenant_id="2")
+
+        mock_osclients.return_value.keystone.side_effect = [
+            user1, user2, user3
+        ]
+
+        context = {
+            "task": mock.MagicMock(),
+            "config": {
+                "existing_users": [user1, user2, user3]
+            }
+        }
+        existing_users.ExistingUsers(context).setup()
+
+        self.assertIn("users", context)
+        self.assertIn("tenants", context)
+        self.assertEqual(3, len(context["users"]))
+        self.assertEqual(
+            {
+                "id": user1.user_id,
+                "endpoint": mock_endpoint.return_value,
+                "tenant_id": user1.tenant_id
+            },
+            context["users"][0]
+        )
+        self.assertEqual(["1", "2"], sorted(context["tenants"].keys()))
+        self.assertEqual({"id": "1", "name": user1.tenant_name},
+                         context["tenants"]["1"])
+        self.assertEqual({"id": "2", "name": user3.tenant_name},
+                         context["tenants"]["2"])
+
+    def test_cleanup(self):
+        # NOTE(boris-42): Test that cleanup is not abstract
+        existing_users.ExistingUsers({"task": mock.MagicMock()}).cleanup()

tests/unit/benchmark/test_engine.py

@@ -178,6 +178,22 @@ class BenchmarkEngineTestCase(test.TestCase):
                           eng._validate_config_semantic_helper, "a", "u", "n",
                           "p", mock.MagicMock(), {})
 
+    @mock.patch("rally.benchmark.engine.existing_users_ctx.ExistingUsers")
+    def test_get_user_ctx_for_validation_existing_users(self, mock_users_ctx):
+
+        context = {"a": 10}
+        users = [mock.MagicMock(), mock.MagicMock()]
+
+        eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock(),
+                                     users=users)
+
+        result = eng._get_user_ctx_for_validation(context)
+
+        self.assertEqual(context["config"]["existing_users"], users)
+        mock_users_ctx.assert_called_once_with(context)
+
+        self.assertEqual(mock_users_ctx.return_value, result)
+
     @mock.patch("rally.benchmark.engine.osclients.Clients")
     @mock.patch("rally.benchmark.engine.users_ctx")
     @mock.patch("rally.benchmark.engine.BenchmarkEngine"
@@ -278,6 +294,28 @@ class BenchmarkEngineTestCase(test.TestCase):
         eng = engine.BenchmarkEngine(config, task)
         eng.run()
 
+    @mock.patch("rally.benchmark.engine.LOG")
+    @mock.patch("rally.benchmark.engine.BenchmarkEngine.consume_results")
+    @mock.patch("rally.benchmark.engine.base_scenario.Scenario")
+    @mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner")
+    @mock.patch("rally.benchmark.engine.base_ctx.ContextManager.cleanup")
+    @mock.patch("rally.benchmark.engine.base_ctx.ContextManager.setup")
+    def test_run_exception_is_logged(self, mock_ctx_setup, mock_ctx_cleanup,
+                                     mock_runner, mock_scenario, mock_consume,
+                                     mock_log):
+
+        mock_ctx_setup.side_effect = Exception
+
+        config = {
+            "a.benchmark": [{"context": {"context_a": {"a": 1}}}],
+            "b.benchmark": [{"context": {"context_b": {"b": 2}}}]
+        }
+        task = mock.MagicMock()
+        eng = engine.BenchmarkEngine(config, task)
+        eng.run()
+
+        self.assertEqual(2, mock_log.exception.call_count)
+
     @mock.patch("rally.benchmark.engine.base_scenario.Scenario.meta")
     def test__prepare_context(self, mock_meta):
         default_context = {"a": 1, "b": 2}
@@ -303,6 +341,30 @@ class BenchmarkEngineTestCase(test.TestCase):
         self.assertEqual(result, expected_result)
         mock_meta.assert_called_once_with(name, "context")
 
+    @mock.patch("rally.benchmark.engine.base_scenario.Scenario.meta")
+    def test__prepare_context_with_existing_users(self, mock_meta):
+        mock_meta.return_value = {}
+        task = mock.MagicMock()
+        name = "a.benchmark"
+        context = {"b": 3, "c": 4}
+        endpoint = mock.MagicMock()
+        config = {
+            "a.benchmark": [{"context": {"context_a": {"a": 1}}}],
+        }
+        existing_users = [mock.MagicMock()]
+        eng = engine.BenchmarkEngine(config, task, users=existing_users)
+        result = eng._prepare_context(context, name, endpoint)
+        expected_context = {"existing_users": existing_users}
+        expected_context.update(context)
+        expected_result = {
+            "task": task,
+            "admin": {"endpoint": endpoint},
+            "scenario_name": name,
+            "config": expected_context
+        }
+        self.assertEqual(result, expected_result)
+        mock_meta.assert_called_once_with(name, "context")
+
     @mock.patch("rally.benchmark.sla.base.SLAChecker")
     def test_consume_results(self, mock_sla):
         mock_sla_instance = mock.MagicMock()