d7635776d7
*) Change task config format
. Split "context" & "runner" stuff
*) Refactor Verification
. Move validation to context.base, runner.base and scenario.base
. Validate whole config fully before starting any of tasks
. Optimize scenario args validation (create only one time clients)
. Optimize order of validation:
1) Validate names of benchmarks
2) Validate all static parameters, e.g. configuration of runner
and context
3) If everything is ok in all benchmarks, then start validation
of scenario args.
. Store validation result (exception) in task["verification_log"]
. Remove verification logic from BenchmarkEngine.__exit__
. Remove scenario args verification results from task["results"]
*) Fix & Swtich to new format doc/samples/tasks
. Switch to new fromat
. Add missing task configratuion
. Better formatting
. json & yaml samples
*) Refactored unit tests
. tests.rally.benchmark.test_engine
. tests.rally.benchmark.context.base
. tests.orcestrator.test_api.start_task
cover validation step as well and new change format
*) Refactor orchestrator api start task
. Remove benchmark engine context
. Call verify explicity
. Do not raise any excpetion in case of validation error
. Catch in start task any unexcepted Exceptions a set deployment
in incosistance state
*) Refactor CLI
. Properly handle new behaviour of verification
. Replace table on task start to just message
. Add HINTs to task detailed command
*) Add unit test for checking doc samples
*) Improve benchmark engine LOGing
blueprint benchmark-new-task-config
Change-Id: I23d3f6b3439fdb44946a7c2491d5a9b3559dc671
238 lines
8.8 KiB
Python
238 lines
8.8 KiB
Python
# Copyright 2013: Mirantis Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
""" Test for orchestrator. """
|
|
|
|
import mock
|
|
import uuid
|
|
|
|
from rally.benchmark.scenarios import base
|
|
from rally import consts
|
|
from rally.orchestrator import api
|
|
from tests import fakes
|
|
from tests import test
|
|
|
|
FAKE_DEPLOY_CONFIG = {
|
|
# TODO(akscram): A fake engine is more suitable for that.
|
|
'name': 'DummyEngine',
|
|
'endpoint': {
|
|
'auth_url': 'http://example.net:5000/v2.0/',
|
|
'username': 'admin',
|
|
'password': 'myadminpass',
|
|
'tenant_name': 'demo'
|
|
},
|
|
}
|
|
|
|
|
|
FAKE_TASK_CONFIG = {
|
|
'FakeScenario.fake': [
|
|
{
|
|
'args': {},
|
|
'runner': {
|
|
'type': 'continuous',
|
|
'timeout': 10000,
|
|
'times': 3,
|
|
'active_users': 2,
|
|
},
|
|
'context': {
|
|
'users': {
|
|
'tenants': 5,
|
|
'users_per_tenant': 6,
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
|
|
|
|
class FakeScenario(base.Scenario):
|
|
@classmethod
|
|
def fake(cls, context):
|
|
pass
|
|
|
|
|
|
# TODO(akscram): The test cases are very superficial because they test
|
|
# only database operations and actually no more. Each
|
|
# case in this test should to mock everything external.
|
|
class APITestCase(test.TestCase):
|
|
|
|
def setUp(self):
|
|
super(APITestCase, self).setUp()
|
|
self.deploy_config = FAKE_DEPLOY_CONFIG
|
|
self.task_config = FAKE_TASK_CONFIG
|
|
self.deploy_uuid = str(uuid.uuid4())
|
|
self.endpoints = [FAKE_DEPLOY_CONFIG['endpoint']]
|
|
# TODO(msdubov): Remove this as soon as DummyEngine requires permission
|
|
# on input
|
|
self.endpoints[0]["permission"] = consts.EndpointPermission.ADMIN
|
|
self.task_uuid = str(uuid.uuid4())
|
|
self.task = {
|
|
'uuid': self.task_uuid,
|
|
}
|
|
self.deployment = {
|
|
'uuid': self.deploy_uuid,
|
|
'name': 'fake_name',
|
|
'config': self.deploy_config,
|
|
'endpoints': self.endpoints,
|
|
}
|
|
self.tempest = mock.Mock()
|
|
|
|
@mock.patch('rally.objects.Task')
|
|
def test_create_task(self, mock_task):
|
|
deployment_uuid = uuid.uuid4()
|
|
tag = "a"
|
|
api.create_task(deployment_uuid, tag)
|
|
mock_task.assert_called_once_with(deployment_uuid=deployment_uuid,
|
|
tag=tag)
|
|
|
|
@mock.patch("rally.benchmark.engine.BenchmarkEngine"
|
|
"._validate_config_semantic")
|
|
@mock.patch("rally.benchmark.engine.BenchmarkEngine"
|
|
"._validate_config_syntax")
|
|
@mock.patch("rally.benchmark.engine.BenchmarkEngine"
|
|
"._validate_config_scenarios_name")
|
|
@mock.patch('rally.benchmark.engine.osclients')
|
|
@mock.patch('rally.benchmark.engine.base_runner.ScenarioRunner.get_runner')
|
|
@mock.patch('rally.objects.deploy.db.deployment_get')
|
|
@mock.patch('rally.objects.task.db.task_result_create')
|
|
@mock.patch('rally.objects.task.db.task_update')
|
|
@mock.patch('rally.objects.task.db.task_create')
|
|
def test_start_task(self, mock_task_create, mock_task_update,
|
|
mock_task_result_create, mock_deploy_get,
|
|
mock_utils_runner, mock_osclients,
|
|
mock_validate_names, mock_validate_syntax,
|
|
mock_validate_semantic):
|
|
mock_task_create.return_value = self.task
|
|
mock_task_update.return_value = self.task
|
|
mock_deploy_get.return_value = self.deployment
|
|
|
|
mock_utils_runner.return_value = mock_runner = mock.Mock()
|
|
mock_runner.run.return_value = ['fake_result']
|
|
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
|
|
api.start_task(self.deploy_uuid, self.task_config)
|
|
|
|
mock_deploy_get.assert_called_once_with(self.deploy_uuid)
|
|
mock_task_create.assert_called_once_with({
|
|
'deployment_uuid': self.deploy_uuid,
|
|
})
|
|
mock_task_update.assert_has_calls([
|
|
mock.call(self.task_uuid, {'status': consts.TaskStatus.VERIFYING}),
|
|
mock.call(self.task_uuid, {'status': consts.TaskStatus.RUNNING}),
|
|
mock.call(self.task_uuid, {'status': consts.TaskStatus.FINISHED})
|
|
])
|
|
# NOTE(akscram): It looks really awful, but checks degradation.
|
|
mock_task_result_create.assert_called_once_with(
|
|
self.task_uuid,
|
|
{
|
|
'kw': {
|
|
'args': {},
|
|
'runner': {
|
|
'type': 'continuous',
|
|
'timeout': 10000,
|
|
'times': 3,
|
|
'active_users': 2,
|
|
},
|
|
'context': {
|
|
'users': {
|
|
'tenants': 5,
|
|
'users_per_tenant': 6,
|
|
}
|
|
}
|
|
},
|
|
'name': 'FakeScenario.fake',
|
|
'pos': 0,
|
|
},
|
|
{
|
|
'raw': ['fake_result']
|
|
}
|
|
)
|
|
|
|
def test_abort_task(self):
|
|
self.assertRaises(NotImplementedError, api.abort_task,
|
|
self.task_uuid)
|
|
|
|
@mock.patch('rally.objects.task.db.task_delete')
|
|
def test_delete_task(self, mock_delete):
|
|
api.delete_task(self.task_uuid)
|
|
mock_delete.assert_called_once_with(
|
|
self.task_uuid,
|
|
status=consts.TaskStatus.FINISHED)
|
|
|
|
@mock.patch('rally.objects.task.db.task_delete')
|
|
def test_delete_task_force(self, mock_delete):
|
|
api.delete_task(self.task_uuid, force=True)
|
|
mock_delete.assert_called_once_with(self.task_uuid, status=None)
|
|
|
|
@mock.patch('rally.objects.deploy.db.deployment_update')
|
|
@mock.patch('rally.objects.deploy.db.deployment_create')
|
|
def test_create_deploy(self, mock_create, mock_update):
|
|
mock_create.return_value = self.deployment
|
|
mock_update.return_value = self.deployment
|
|
api.create_deploy(self.deploy_config, 'fake_deploy')
|
|
mock_create.assert_called_once_with({
|
|
'name': 'fake_deploy',
|
|
'config': self.deploy_config,
|
|
})
|
|
mock_update.assert_has_calls([
|
|
mock.call(self.deploy_uuid, {'endpoints': self.endpoints})
|
|
])
|
|
|
|
@mock.patch('rally.objects.deploy.db.deployment_delete')
|
|
@mock.patch('rally.objects.deploy.db.deployment_update')
|
|
@mock.patch('rally.objects.deploy.db.deployment_get')
|
|
def test_destroy_deploy(self, mock_get, mock_update, mock_delete):
|
|
mock_get.return_value = self.deployment
|
|
mock_update.return_value = self.deployment
|
|
api.destroy_deploy(self.deploy_uuid)
|
|
mock_get.assert_called_once_with(self.deploy_uuid)
|
|
mock_delete.assert_called_once_with(self.deploy_uuid)
|
|
|
|
@mock.patch('rally.objects.deploy.db.deployment_update')
|
|
@mock.patch('rally.objects.deploy.db.deployment_get')
|
|
def test_recreate_deploy(self, mock_get, mock_update):
|
|
mock_get.return_value = self.deployment
|
|
mock_update.return_value = self.deployment
|
|
api.recreate_deploy(self.deploy_uuid)
|
|
mock_get.assert_called_once_with(self.deploy_uuid)
|
|
mock_update.assert_has_calls([
|
|
mock.call(self.deploy_uuid, {'endpoints': self.endpoints})
|
|
])
|
|
|
|
@mock.patch('rally.verification.verifiers.tempest.tempest.Tempest')
|
|
@mock.patch('rally.objects.deploy.db.deployment_get')
|
|
def test_verify(self, mock_get, mock_tempest):
|
|
mock_tempest.return_value = self.tempest
|
|
self.tempest.is_installed.return_value = True
|
|
mock_get.return_value = self.deployment
|
|
api.verify(self.deploy_uuid, 'image_id', 'alt_image_id', 'flavor_id',
|
|
'alt_flavor_id', 'smoke', None)
|
|
|
|
self.tempest.is_installed.assert_called_once_with()
|
|
mock_get.assert_called_once_with(self.deploy_uuid)
|
|
self.tempest.verify.assert_called_once_with(
|
|
image_ref='image_id',
|
|
image_ref_alt='alt_image_id',
|
|
flavor_ref='flavor_id',
|
|
flavor_ref_alt='alt_flavor_id',
|
|
username='admin',
|
|
password='myadminpass',
|
|
tenant_name='demo',
|
|
uri='http://example.net:5000/v2.0/',
|
|
uri_v3='http://example.net:5000/v3/',
|
|
set_name='smoke',
|
|
regex=None)
|