d7635776d7
*) Change task config format
. Split "context" & "runner" stuff
*) Refactor Verification
. Move validation to context.base, runner.base and scenario.base
. Validate whole config fully before starting any of tasks
. Optimize scenario args validation (create only one time clients)
. Optimize order of validation:
1) Validate names of benchmarks
2) Validate all static parameters, e.g. configuration of runner
and context
3) If everything is ok in all benchmarks, then start validation
of scenario args.
. Store validation result (exception) in task["verification_log"]
. Remove verification logic from BenchmarkEngine.__exit__
. Remove scenario args verification results from task["results"]
*) Fix & Swtich to new format doc/samples/tasks
. Switch to new fromat
. Add missing task configratuion
. Better formatting
. json & yaml samples
*) Refactored unit tests
. tests.rally.benchmark.test_engine
. tests.rally.benchmark.context.base
. tests.orcestrator.test_api.start_task
cover validation step as well and new change format
*) Refactor orchestrator api start task
. Remove benchmark engine context
. Call verify explicity
. Do not raise any excpetion in case of validation error
. Catch in start task any unexcepted Exceptions a set deployment
in incosistance state
*) Refactor CLI
. Properly handle new behaviour of verification
. Replace table on task start to just message
. Add HINTs to task detailed command
*) Add unit test for checking doc samples
*) Improve benchmark engine LOGing
blueprint benchmark-new-task-config
Change-Id: I23d3f6b3439fdb44946a7c2491d5a9b3559dc671
276 lines
11 KiB
Python
276 lines
11 KiB
Python
# Copyright 2013: Mirantis Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Tests for the Test engine."""
|
|
|
|
import jsonschema
|
|
import mock
|
|
|
|
from rally.benchmark import engine
|
|
from rally import consts
|
|
from rally import exceptions
|
|
from tests import fakes
|
|
from tests import test
|
|
|
|
|
|
class BenchmarkEngineTestCase(test.TestCase):
|
|
|
|
def test_init(self):
|
|
config = mock.MagicMock()
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(config, task)
|
|
self.assertEqual(eng.config, config)
|
|
self.assertEqual(eng.task, task)
|
|
|
|
@mock.patch("rally.benchmark.engine.jsonschema.validate")
|
|
def test_validate(self, mock_json_validate):
|
|
config = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(config, mock.MagicMock())
|
|
mock_validate = mock.MagicMock()
|
|
|
|
eng._validate_config_scenarios_name = mock_validate.names
|
|
eng._validate_config_syntax = mock_validate.syntax
|
|
eng._validate_config_semantic = mock_validate.semantic
|
|
|
|
eng.validate()
|
|
|
|
expected_calls = [
|
|
mock.call.names(config),
|
|
mock.call.syntax(config),
|
|
mock.call.semantic(config)
|
|
]
|
|
mock_validate.assert_has_calls(expected_calls)
|
|
|
|
def test_validate__wrong_schema(self):
|
|
config = {
|
|
"wrong": True
|
|
}
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(config, task)
|
|
self.assertRaises(exceptions.InvalidTaskException,
|
|
eng.validate)
|
|
task.set_failed.assert_called_once()
|
|
|
|
@mock.patch("rally.benchmark.engine.jsonschema.validate")
|
|
def test_validate__wrong_scenarios_name(self, mova_validate):
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(mock.MagicMock(), task)
|
|
eng._validate_config_scenarios_name = \
|
|
mock.MagicMock(side_effect=exceptions.NotFoundScenarios)
|
|
|
|
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
|
|
task.set_failed.assert_called_once()
|
|
|
|
@mock.patch("rally.benchmark.engine.jsonschema.validate")
|
|
def test_validate__wrong_syntax(self, mova_validate):
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(mock.MagicMock(), task)
|
|
eng._validate_config_scenarios_name = mock.MagicMock()
|
|
eng._validate_config_syntax = \
|
|
mock.MagicMock(side_effect=exceptions.InvalidBenchmarkConfig)
|
|
|
|
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
|
|
task.set_failed.assert_called_once()
|
|
|
|
@mock.patch("rally.benchmark.engine.jsonschema.validate")
|
|
def test_validate__wrong_semantic(self, mova_validate):
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(mock.MagicMock(), task)
|
|
eng._validate_config_scenarios_name = mock.MagicMock()
|
|
eng._validate_config_syntax = mock.MagicMock()
|
|
eng._validate_config_semantic = \
|
|
mock.MagicMock(side_effect=exceptions.InvalidBenchmarkConfig)
|
|
|
|
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
|
|
task.set_failed.assert_called_once()
|
|
|
|
@mock.patch("rally.benchmark.engine.base_scenario.Scenario")
|
|
def test__validate_config_scenarios_name(self, mock_scenario):
|
|
config = {
|
|
"a": [],
|
|
"b": []
|
|
}
|
|
mock_scenario.list_benchmark_scenarios.return_value = ["e", "b", "a"]
|
|
eng = engine.BenchmarkEngine(config, mock.MagicMock())
|
|
eng._validate_config_scenarios_name(config)
|
|
|
|
@mock.patch("rally.benchmark.engine.base_scenario.Scenario")
|
|
def test__validate_config_scenarios_name_non_exsisting(self,
|
|
mock_scenario):
|
|
config = {
|
|
"exist": [],
|
|
"nonexist1": [],
|
|
"nonexist2": []
|
|
}
|
|
mock_scenario.list_benchmark_scenarios.return_value = ["exist", "aaa"]
|
|
eng = engine.BenchmarkEngine(config, mock.MagicMock())
|
|
|
|
self.assertRaises(exceptions.NotFoundScenarios,
|
|
eng._validate_config_scenarios_name, config)
|
|
|
|
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner.validate")
|
|
@mock.patch("rally.benchmark.engine.base_ctx.Context.validate")
|
|
def test__validate_config_syntax(self, mock_context, mock_runner):
|
|
config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]}
|
|
eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock())
|
|
eng._validate_config_syntax(config)
|
|
mock_runner.assert_has_calls([mock.call({}), mock.call("b")])
|
|
mock_context.assert_has_calls([mock.call("a"), mock.call({})])
|
|
|
|
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner")
|
|
@mock.patch("rally.benchmark.engine.base_ctx.Context.validate")
|
|
def test__validate_config_syntax__wrong_runner(self, mock_context,
|
|
mock_runner):
|
|
config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]}
|
|
eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock())
|
|
|
|
mock_runner.validate = mock.MagicMock(
|
|
side_effect=jsonschema.ValidationError("a"))
|
|
self.assertRaises(exceptions.InvalidBenchmarkConfig,
|
|
eng._validate_config_syntax, config)
|
|
|
|
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner.validate")
|
|
@mock.patch("rally.benchmark.engine.base_ctx.Context")
|
|
def test__validate_config_syntax__wrong_context(self, mock_context,
|
|
mock_runner):
|
|
config = {"sca": [{"context": "a"}], "scb": [{"runner": "b"}]}
|
|
eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock())
|
|
|
|
mock_context.validate = mock.MagicMock(
|
|
side_effect=jsonschema.ValidationError("a"))
|
|
self.assertRaises(exceptions.InvalidBenchmarkConfig,
|
|
eng._validate_config_syntax, config)
|
|
|
|
@mock.patch("rally.benchmark.engine.base_scenario.Scenario.validate")
|
|
def test__validate_config_semantic_helper(self, mock_validate):
|
|
eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock())
|
|
eng._validate_config_sematic_helper("admin", "user", "name", "pos",
|
|
{"args": "args"})
|
|
mock_validate.assert_called_once_with("name", "args", admin="admin",
|
|
users=["user"])
|
|
|
|
@mock.patch("rally.benchmark.engine.base_scenario.Scenario.validate")
|
|
def test__validate_config_semanitc_helper_invalid_arg(self, mock_validate):
|
|
mock_validate.side_effect = exceptions.InvalidScenarioArgument()
|
|
eng = engine.BenchmarkEngine(mock.MagicMock(), mock.MagicMock())
|
|
|
|
self.assertRaises(exceptions.InvalidBenchmarkConfig,
|
|
eng._validate_config_sematic_helper, "a", "u", "n",
|
|
"p", {})
|
|
|
|
@mock.patch("rally.benchmark.engine.osclients.Clients")
|
|
@mock.patch("rally.benchmark.engine.users_ctx")
|
|
@mock.patch("rally.benchmark.engine.BenchmarkEngine"
|
|
"._validate_config_sematic_helper")
|
|
def test__validate_config_sematic(self, mock_helper, mock_userctx,
|
|
mock_osclients):
|
|
mock_userctx.UserGenerator = fakes.FakeUserContext
|
|
mock_osclients.return_value = mock.MagicMock()
|
|
config = {
|
|
"a": [mock.MagicMock(), mock.MagicMock()],
|
|
"b": [mock.MagicMock()]
|
|
}
|
|
|
|
eng = engine.BenchmarkEngine(config, mock.MagicMock())
|
|
eng.admin_endpoint = "admin"
|
|
|
|
eng._validate_config_semantic(config)
|
|
|
|
expected_calls = [
|
|
mock.call("admin"),
|
|
mock.call(fakes.FakeUserContext.user["endpoint"])
|
|
]
|
|
mock_osclients.assert_has_calls(expected_calls)
|
|
|
|
admin = user = mock_osclients.return_value
|
|
expected_calls = [
|
|
mock.call(admin, user, "a", 0, config["a"][0]),
|
|
mock.call(admin, user, "a", 1, config["a"][1]),
|
|
mock.call(admin, user, "b", 0, config["b"][0])
|
|
]
|
|
mock_helper.assert_has_calls(expected_calls)
|
|
|
|
def test_run__update_status(self):
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine([], task)
|
|
results = eng.run()
|
|
self.assertEqual(results, {})
|
|
task.update_status.assert_has_calls([
|
|
mock.call(consts.TaskStatus.RUNNING),
|
|
mock.call(consts.TaskStatus.FINISHED)
|
|
])
|
|
|
|
@mock.patch("rally.benchmark.engine.endpoint.Endpoint")
|
|
@mock.patch("rally.benchmark.engine.osclients")
|
|
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner")
|
|
def test_run__config_has_args(self, mock_endpoint, mock_osclients,
|
|
mock_runner):
|
|
config = {
|
|
"a.args": [{"args": {"a": "a", "b": 1}}],
|
|
"b.args": [{"args": {"a": 1}}]
|
|
}
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(config, task).bind([{}])
|
|
eng.run()
|
|
|
|
@mock.patch("rally.benchmark.engine.endpoint.Endpoint")
|
|
@mock.patch("rally.benchmark.engine.osclients")
|
|
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner")
|
|
def test_run__config_has_runner(self, mock_endpoint, mock_osclients,
|
|
mock_runner):
|
|
config = {
|
|
"a.args": [{"runner": {"type": "a", "b": 1}}],
|
|
"b.args": [{"runner": {"a": 1}}]
|
|
}
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(config, task).bind([{}])
|
|
eng.run()
|
|
|
|
@mock.patch("rally.benchmark.engine.endpoint.Endpoint")
|
|
@mock.patch("rally.benchmark.engine.osclients")
|
|
@mock.patch("rally.benchmark.engine.base_runner.ScenarioRunner")
|
|
def test_run__config_has_context(self, mock_endpoint, mock_osclients,
|
|
mock_runner):
|
|
config = {
|
|
"a.args": [{"context": {"context_a": {"a": 1}}}],
|
|
"b.args": [{"context": {"context_b": {"b": 2}}}]
|
|
}
|
|
task = mock.MagicMock()
|
|
eng = engine.BenchmarkEngine(config, task).bind([{}])
|
|
eng.run()
|
|
|
|
@mock.patch("rally.benchmark.engine.osclients")
|
|
@mock.patch("rally.benchmark.engine.endpoint.Endpoint")
|
|
def test_bind(self, mock_endpoint, mock_osclients):
|
|
mock_endpoint.return_value = mock.MagicMock()
|
|
benchmark_engine = engine.BenchmarkEngine(mock.MagicMock(),
|
|
mock.MagicMock())
|
|
endpoint = {
|
|
"auth_url": "http://valid.com",
|
|
"username": "user",
|
|
"password": "pwd",
|
|
"tenant_name": "tenant"
|
|
}
|
|
|
|
binded_benchmark_engine = benchmark_engine.bind([endpoint])
|
|
self.assertEqual([mock_endpoint.return_value],
|
|
benchmark_engine.endpoints)
|
|
self.assertEqual(benchmark_engine, binded_benchmark_engine)
|
|
expected_calls = [
|
|
mock.call.Clients(mock_endpoint.return_value),
|
|
mock.call.Clients().verified_keystone()
|
|
]
|
|
mock_osclients.assert_has_calls(expected_calls)
|