From d366987c87ffc9eba5c92dd67adc0ad08771097c Mon Sep 17 00:00:00 2001 From: Boris Pavlovic Date: Sat, 28 Mar 2015 03:52:12 +0300 Subject: [PATCH] Finish support of benchmarking with existing users Now one can use Rally for benchmarking OpenStack clouds that are using LDAP, AD or any other read only keystone backend where you are not able to create any users. To do this one can specify "users" section in ExistingCloud plugin input file. Rally will use these "users" to generate load instead of creating new. As well if you specify both "users" section in deployment input file and "users" context it will use temporary created by Rally users. Restructurize samples/deployments directory: - Deploying OpenStack clouds with Rally is not popular functionallity so hide all samples deeper. - Some fixes in readme.rst - Add new sample for existing cloud with predefined users Change-Id: If3d31960ee317e0770abceacc62b3acf0947d269 --- rally/benchmark/context/existing_users.py | 70 +++++++++++++++++++ rally/benchmark/engine.py | 38 +++++++--- samples/deployments/README.rst | 56 ++++++--------- .../existing-with-predefined-users.json | 23 ++++++ .../README.rst | 40 +++++++++++ .../devstack-in-existing-servers.json | 0 .../devstack-in-lxc.json | 0 .../devstack-in-openstack.json | 0 ...vstack-lxc-engine-in-existing-servers.json | 0 ...evstack-lxc-engine-in-existing-servers.rst | 0 .../fuel-ha.json | 0 .../fuel-multinode.json | 0 .../multihost.json | 0 .../multihost.rst | 0 .../benchmark/context/test_existing_users.py | 62 ++++++++++++++++ tests/unit/benchmark/test_engine.py | 62 ++++++++++++++++ 16 files changed, 307 insertions(+), 44 deletions(-) create mode 100644 rally/benchmark/context/existing_users.py create mode 100644 samples/deployments/existing-with-predefined-users.json create mode 100644 samples/deployments/for_deploying_openstack_with_rally/README.rst rename samples/deployments/{ => for_deploying_openstack_with_rally}/devstack-in-existing-servers.json (100%) rename samples/deployments/{ => for_deploying_openstack_with_rally}/devstack-in-lxc.json (100%) rename samples/deployments/{ => for_deploying_openstack_with_rally}/devstack-in-openstack.json (100%) rename samples/deployments/{ => for_deploying_openstack_with_rally}/devstack-lxc-engine-in-existing-servers.json (100%) rename samples/deployments/{ => for_deploying_openstack_with_rally}/devstack-lxc-engine-in-existing-servers.rst (100%) rename samples/deployments/{ => for_deploying_openstack_with_rally}/fuel-ha.json (100%) rename samples/deployments/{ => for_deploying_openstack_with_rally}/fuel-multinode.json (100%) rename samples/deployments/{ => for_deploying_openstack_with_rally}/multihost.json (100%) rename samples/deployments/{ => for_deploying_openstack_with_rally}/multihost.rst (100%) create mode 100644 tests/unit/benchmark/context/test_existing_users.py diff --git a/rally/benchmark/context/existing_users.py b/rally/benchmark/context/existing_users.py new file mode 100644 index 0000000000..ed2464b56b --- /dev/null +++ b/rally/benchmark/context/existing_users.py @@ -0,0 +1,70 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from rally.benchmark.context import base +from rally.common.i18n import _ +from rally.common import log as logging +from rally.common import utils as rutils +from rally import objects +from rally import osclients + + +LOG = logging.getLogger(__name__) + + +# NOTE(boris-42): This context should be hidden for now and used only by +# benchmark engine. In future during various refactoring of +# validation system and rally CI testing we will make it public +@base.context(name="existing_users", order=99, hidden=True) +class ExistingUsers(base.Context): + """This context supports using existing users in Rally. + + It uses information about deployment to properly + initialize context["users"] and context["tenants"] + + So there won't be big difference between usage of "users" and + "existing_users" context. + """ + + # NOTE(boris-42): We don't need to check config schema because + # this is used only by benchmark engine + CONFIG_SCHEMA = {} + + def __init__(self, context): + super(ExistingUsers, self).__init__(context) + self.context["users"] = [] + self.context["tenants"] = {} + + @rutils.log_task_wrapper(LOG.info, _("Enter context: `existing_users`")) + def setup(self): + for user in self.config: + user_endpoint = objects.Endpoint(**user) + user_kclient = osclients.Clients(user_endpoint).keystone() + + if user_kclient.tenant_id not in self.context["tenants"]: + self.context["tenants"][user_kclient.tenant_id] = { + "id": user_kclient.tenant_id, + "name": user_kclient.tenant_name + } + + self.context["users"].append({ + "endpoint": user_endpoint, + "id": user_kclient.user_id, + "tenant_id": user_kclient.tenant_id + }) + + @rutils.log_task_wrapper(LOG.info, _("Exit context: `existing_users`")) + def cleanup(self): + """These users are not managed by Rally, so don't touch them.""" diff --git a/rally/benchmark/engine.py b/rally/benchmark/engine.py index 1e164f380e..8b0a3498fa 100644 --- a/rally/benchmark/engine.py +++ b/rally/benchmark/engine.py @@ -22,6 +22,7 @@ import jsonschema import six from rally.benchmark.context import base as base_ctx +from rally.benchmark.context import existing_users as existing_users_ctx from rally.benchmark.context import users as users_ctx from rally.benchmark.runners import base as base_runner from rally.benchmark.scenarios import base as base_scenario @@ -107,7 +108,7 @@ class BenchmarkEngine(object): self.config = config self.task = task self.admin = admin and objects.Endpoint(**admin) or None - self.users = map(lambda u: objects.Endpoint(**u), users or []) + self.existing_users = users or [] self.abort_on_sla_failure = abort_on_sla_failure @rutils.log_task_wrapper(LOG.info, _("Task validation check cloud.")) @@ -153,25 +154,38 @@ class BenchmarkEngine(object): "config": kwargs, "reason": six.text_type(e)} raise exceptions.InvalidBenchmarkConfig(**kw) + def _get_user_ctx_for_validation(self, context): + if self.existing_users: + context["config"] = {"existing_users": self.existing_users} + user_context = existing_users_ctx.ExistingUsers(context) + else: + user_context = users_ctx.UserGenerator(context) + + return user_context + @rutils.log_task_wrapper(LOG.info, _("Task validation of semantic.")) def _validate_config_semantic(self, config): self._check_cloud() - # NOTE(boris-42): In future we will have more complex context, because - # we will have pre-created users mode as well. context = {"task": self.task, "admin": {"endpoint": self.admin}} deployment = objects.Deployment.get(self.task["deployment_uuid"]) - with users_ctx.UserGenerator(context) as ctx: + # TODO(boris-42): It's quite hard at the moment to validate case + # when both user context and existing_users are + # specified. So after switching to plugin base + # and refactoring validation mechanism this place + # will be replaced + with self._get_user_ctx_for_validation(context) as ctx: ctx.setup() admin = osclients.Clients(self.admin) user = osclients.Clients(context["users"][0]["endpoint"]) - for name, values in six.iteritems(config): - for pos, kwargs in enumerate(values): - self._validate_config_semantic_helper(admin, user, name, - pos, deployment, - kwargs) + for u in context["users"]: + user = osclients.Clients(u["endpoint"]) + for name, values in six.iteritems(config): + for pos, kwargs in enumerate(values): + self._validate_config_semantic_helper( + admin, user, name, pos, deployment, kwargs) @rutils.log_task_wrapper(LOG.info, _("Task validation.")) def validate(self): @@ -195,7 +209,11 @@ class BenchmarkEngine(object): def _prepare_context(self, context, name, endpoint): scenario_context = base_scenario.Scenario.meta(name, "context") - scenario_context.setdefault("users", {}) + if self.existing_users and "users" not in context: + scenario_context.setdefault("existing_users", self.existing_users) + elif "users" not in context: + scenario_context.setdefault("users", {}) + scenario_context.update(context) context_obj = { "task": self.task, diff --git a/samples/deployments/README.rst b/samples/deployments/README.rst index 2f79858937..dca1f1f82f 100644 --- a/samples/deployments/README.rst +++ b/samples/deployments/README.rst @@ -1,12 +1,21 @@ Rally Deployments ================= -Before starting cluster benchmarking, its connection parameters -should be saved in Rally database (deployment record). +Rally needs to be configured to use an OpenStack Cloud deployment before it +can benchmark the deployment. -If there is no cluster, rally also can create it. +To configure Rally to use an OpenStack Cloud deployment, you need create a +deployment configuration by supplying the endpoint and credentials, as follows: + +.. code-block:: + + rally deployment create --file --name my_cloud + + +If you don't have OpenStack deployments, Rally can deploy it for you. +For samples of various deployments take a look at samples from +**for_deploying_openstack_with_rally** directory. -There are examples of deployment configurations: existing.json ------------- @@ -18,6 +27,15 @@ existing-keystone-v3.json Register existing OpenStack cluster that uses Keystone v3. +existing-with-predefined-users.json +-------------------------------------- + +If you are using read-only backend in Keystone like LDAP, AD then +you need this sample. If you don't specify "users" rally will use already +existing users that you provide. + + + existing-with-given-endpoint.json --------------------------------- @@ -26,33 +44,3 @@ to explicitly set keystone management_url. Use this parameter if keystone fails to setup management_url correctly. For example, this parameter must be specified for FUEL cluster and has value "http://:35357/v2.0/" - -devstack-in-existing-servers.json ---------------------------------- - -Register existing DevStack cluster. - -devstack-in-lxc.json --------------------- - -Deploy DevStack cluster on LXC and register it by Rally. - -devstack-in-openstack.json --------------------------- - -Deploy DevStack cluster on OpenStack and register it by Rally. - -devstack-lxc-engine-in-existing-servers.json --------------------------------------------- - -See *devstack-lxc-engine-in-existing-servers.rst* for details - -fuel-ha.json ------------- - -Deploy High Availability FUEL cluster and register it by Rally. - -fuel-multinode.json -------------------- - -Deploy Multinode FUEL cluster and register it by Rally. diff --git a/samples/deployments/existing-with-predefined-users.json b/samples/deployments/existing-with-predefined-users.json new file mode 100644 index 0000000000..f1002b5d88 --- /dev/null +++ b/samples/deployments/existing-with-predefined-users.json @@ -0,0 +1,23 @@ +{ + "type": "ExistingCloud", + "auth_url": "http://example.net:5000/v2.0/", + "region_name": "RegionOne", + "endpoint_type": "public", + "admin": { + "username": "admin", + "password": "pa55word", + "tenant_name": "demo" + }, + "users": [ + { + "username": "not_an_admin1", + "password": "password", + "tenant_name": "some_tenant" + }, + { + "username": "not_an_admin2", + "password": "password2", + "tenant_name": "some_tenant2" + } + ] +} diff --git a/samples/deployments/for_deploying_openstack_with_rally/README.rst b/samples/deployments/for_deploying_openstack_with_rally/README.rst new file mode 100644 index 0000000000..a7f6b6d27d --- /dev/null +++ b/samples/deployments/for_deploying_openstack_with_rally/README.rst @@ -0,0 +1,40 @@ +Using Rally for OpenStack deployment +==================================== + +This directory contains various input files for **rally deployment create** +command. This command will deploy OpenStack and register it to Rally DB. + + +devstack-in-existing-servers.json +--------------------------------- + +Deploy with DevStack OpenStack cloud on specified servers. + + +devstack-in-lxc.json +-------------------- + +Deploy DevStack cluster in LXC containers + + +devstack-in-openstack.json +-------------------------- + +Create require VM in specified OpenStack cloud and deploy using DevStack +OpenStack on them. + + +devstack-lxc-engine-in-existing-servers.json +-------------------------------------------- + +See *devstack-lxc-engine-in-existing-servers.rst* for details + +fuel-ha.json +------------ + +Deploy High Availability FUEL cluster. + +fuel-multinode.json +------------------- + +Deploy Multinode FUEL cluster. diff --git a/samples/deployments/devstack-in-existing-servers.json b/samples/deployments/for_deploying_openstack_with_rally/devstack-in-existing-servers.json similarity index 100% rename from samples/deployments/devstack-in-existing-servers.json rename to samples/deployments/for_deploying_openstack_with_rally/devstack-in-existing-servers.json diff --git a/samples/deployments/devstack-in-lxc.json b/samples/deployments/for_deploying_openstack_with_rally/devstack-in-lxc.json similarity index 100% rename from samples/deployments/devstack-in-lxc.json rename to samples/deployments/for_deploying_openstack_with_rally/devstack-in-lxc.json diff --git a/samples/deployments/devstack-in-openstack.json b/samples/deployments/for_deploying_openstack_with_rally/devstack-in-openstack.json similarity index 100% rename from samples/deployments/devstack-in-openstack.json rename to samples/deployments/for_deploying_openstack_with_rally/devstack-in-openstack.json diff --git a/samples/deployments/devstack-lxc-engine-in-existing-servers.json b/samples/deployments/for_deploying_openstack_with_rally/devstack-lxc-engine-in-existing-servers.json similarity index 100% rename from samples/deployments/devstack-lxc-engine-in-existing-servers.json rename to samples/deployments/for_deploying_openstack_with_rally/devstack-lxc-engine-in-existing-servers.json diff --git a/samples/deployments/devstack-lxc-engine-in-existing-servers.rst b/samples/deployments/for_deploying_openstack_with_rally/devstack-lxc-engine-in-existing-servers.rst similarity index 100% rename from samples/deployments/devstack-lxc-engine-in-existing-servers.rst rename to samples/deployments/for_deploying_openstack_with_rally/devstack-lxc-engine-in-existing-servers.rst diff --git a/samples/deployments/fuel-ha.json b/samples/deployments/for_deploying_openstack_with_rally/fuel-ha.json similarity index 100% rename from samples/deployments/fuel-ha.json rename to samples/deployments/for_deploying_openstack_with_rally/fuel-ha.json diff --git a/samples/deployments/fuel-multinode.json b/samples/deployments/for_deploying_openstack_with_rally/fuel-multinode.json similarity index 100% rename from samples/deployments/fuel-multinode.json rename to samples/deployments/for_deploying_openstack_with_rally/fuel-multinode.json diff --git a/samples/deployments/multihost.json b/samples/deployments/for_deploying_openstack_with_rally/multihost.json similarity index 100% rename from samples/deployments/multihost.json rename to samples/deployments/for_deploying_openstack_with_rally/multihost.json diff --git a/samples/deployments/multihost.rst b/samples/deployments/for_deploying_openstack_with_rally/multihost.rst similarity index 100% rename from samples/deployments/multihost.rst rename to samples/deployments/for_deploying_openstack_with_rally/multihost.rst diff --git a/tests/unit/benchmark/context/test_existing_users.py b/tests/unit/benchmark/context/test_existing_users.py new file mode 100644 index 0000000000..7167f504aa --- /dev/null +++ b/tests/unit/benchmark/context/test_existing_users.py @@ -0,0 +1,62 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + + +from rally.benchmark.context import existing_users +from tests.unit import test + + +class ExistingUserTestCase(test.TestCase): + + @mock.patch("rally.benchmark.context.existing_users.osclients.Clients") + @mock.patch("rally.benchmark.context.existing_users.objects.Endpoint") + def test_setup(self, mock_endpoint, mock_osclients): + user1 = mock.MagicMock(tenant_id="1") + user2 = mock.MagicMock(tenant_id="1") + user3 = mock.MagicMock(tenant_id="2") + + mock_osclients.return_value.keystone.side_effect = [ + user1, user2, user3 + ] + + context = { + "task": mock.MagicMock(), + "config": { + "existing_users": [user1, user2, user3] + } + } + existing_users.ExistingUsers(context).setup() + + self.assertIn("users", context) + self.assertIn("tenants", context) + self.assertEqual(3, len(context["users"])) + self.assertEqual( + { + "id": user1.user_id, + "endpoint": mock_endpoint.return_value, + "tenant_id": user1.tenant_id + }, + context["users"][0] + ) + self.assertEqual(["1", "2"], sorted(context["tenants"].keys())) + self.assertEqual({"id": "1", "name": user1.tenant_name}, + context["tenants"]["1"]) + self.assertEqual({"id": "2", "name": user3.tenant_name}, + context["tenants"]["2"]) + + def test_cleanup(self): + # NOTE(boris-42): Test that cleanup is not abstract + existing_users.ExistingUsers({"task": mock.MagicMock()}).cleanup() diff --git a/tests/unit/benchmark/test_engine.py b/tests/unit/benchmark/test_engine.py index bb9bd8796d..fe8f2051bc 100644 --- a/tests/unit/benchmark/test_engine.py +++ b/tests/unit/benchmark/test_engine.py @@ -178,6 +178,22 @@ class BenchmarkEngineTestCase(test.TestCase): eng._validate_config_semantic_helper, "a", "u", "n", "p", mock.MagicMock(), {}) + @mock.patch("rally.benchmark.engine.existing_users_ctx.ExistingUsers") + def test_get_user_ctx_for_validation_existing_users(self, mock_users_ctx): + + context = {"a": 10} + users = [mock.MagicMock(), mock.MagicMock()] + + eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock(), + users=users) + + result = eng._get_user_ctx_for_validation(context) + + self.assertEqual(context["config"]["existing_users"], users) + mock_users_ctx.assert_called_once_with(context) + + self.assertEqual(mock_users_ctx.return_value, result) + @mock.patch("rally.benchmark.engine.osclients.Clients") @mock.patch("rally.benchmark.engine.users_ctx") @mock.patch("rally.benchmark.engine.BenchmarkEngine" @@ -278,6 +294,28 @@ class BenchmarkEngineTestCase(test.TestCase): eng = engine.BenchmarkEngine(config, task) eng.run() + @mock.patch("rally.benchmark.engine.LOG") + @mock.patch("rally.benchmark.engine.BenchmarkEngine.consume_results") + @mock.patch("rally.benchmark.engine.base_scenario.Scenario") + @mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner") + @mock.patch("rally.benchmark.engine.base_ctx.ContextManager.cleanup") + @mock.patch("rally.benchmark.engine.base_ctx.ContextManager.setup") + def test_run_exception_is_logged(self, mock_ctx_setup, mock_ctx_cleanup, + mock_runner, mock_scenario, mock_consume, + mock_log): + + mock_ctx_setup.side_effect = Exception + + config = { + "a.benchmark": [{"context": {"context_a": {"a": 1}}}], + "b.benchmark": [{"context": {"context_b": {"b": 2}}}] + } + task = mock.MagicMock() + eng = engine.BenchmarkEngine(config, task) + eng.run() + + self.assertEqual(2, mock_log.exception.call_count) + @mock.patch("rally.benchmark.engine.base_scenario.Scenario.meta") def test__prepare_context(self, mock_meta): default_context = {"a": 1, "b": 2} @@ -303,6 +341,30 @@ class BenchmarkEngineTestCase(test.TestCase): self.assertEqual(result, expected_result) mock_meta.assert_called_once_with(name, "context") + @mock.patch("rally.benchmark.engine.base_scenario.Scenario.meta") + def test__prepare_context_with_existing_users(self, mock_meta): + mock_meta.return_value = {} + task = mock.MagicMock() + name = "a.benchmark" + context = {"b": 3, "c": 4} + endpoint = mock.MagicMock() + config = { + "a.benchmark": [{"context": {"context_a": {"a": 1}}}], + } + existing_users = [mock.MagicMock()] + eng = engine.BenchmarkEngine(config, task, users=existing_users) + result = eng._prepare_context(context, name, endpoint) + expected_context = {"existing_users": existing_users} + expected_context.update(context) + expected_result = { + "task": task, + "admin": {"endpoint": endpoint}, + "scenario_name": name, + "config": expected_context + } + self.assertEqual(result, expected_result) + mock_meta.assert_called_once_with(name, "context") + @mock.patch("rally.benchmark.sla.base.SLAChecker") def test_consume_results(self, mock_sla): mock_sla_instance = mock.MagicMock()