diff --git a/doc/specs/implemented/sla_pd_plugin.rst b/doc/specs/implemented/sla_pd_plugin.rst new file mode 100644 index 00000000..5ee64b38 --- /dev/null +++ b/doc/specs/implemented/sla_pd_plugin.rst @@ -0,0 +1,68 @@ +.. + This work is licensed under a Creative Commons Attribution 3.0 Unported + License. + + http://creativecommons.org/licenses/by/3.0/legalcode + +================================== +SLA Performance degradation plugin +================================== + +Problem description +=================== + +During density and reliability testing of OpenStack with Rally +we observed test cases, during execution of which performance +of OpenStack cluster has been drammatically degradated. + +Proposed change +=============== + +Develop a new Rally SLA plugin: *performance_degradation* + +This SLA plugin should find minimum and maximum duration of +iterations completed without errors during Rally task execution. +Assuming that minimum duration is 100%, it should calculate +performance degradation against maximum duration. + +SLA plugin results: + - failure if performance degradation is more than value set + in plugin's max_degradation parameter; + - success if degradation is less + - performance degradation value as a percentage. + +How to enable this plugin: + +.. code:: json + + "sla": { + "performance_degradation": { + "max_degradation": 50 + } + } + +Alternatives +------------ + +None + +Implementation +============== + +Assignee(s) +----------- + +Primary assignee: + +anevenchannyy + +Work Items +---------- + + - Implement plugin + - Add non-voting job with this plugin to the most important OpenStack services + +Dependencies +============ + +None diff --git a/rally-jobs/rally.yaml b/rally-jobs/rally.yaml index 20a18244..814293fc 100644 --- a/rally-jobs/rally.yaml +++ b/rally-jobs/rally.yaml @@ -404,6 +404,8 @@ max: 1 min_iterations: 10 sigmas: 10 + performance_degradation: + max_degradation: 50 - args: diff --git a/tests/functional/test_cli_task.py b/tests/functional/test_cli_task.py index bdd6cea8..67ecc713 100644 --- a/tests/functional/test_cli_task.py +++ b/tests/functional/test_cli_task.py @@ -980,3 +980,50 @@ class SLAExtraFlagsTestCase(unittest.TestCase): "times": 5, "rps": 3, "timeout": 6}) + + +class SLAPerfDegrTestCase(unittest.TestCase): + + def _get_sample_task_config(self, max_degradation=500): + return { + "Dummy.dummy_random_action": [ + { + "args": { + "actions_num": 5, + "sleep_min": 0.5, + "sleep_max": 2 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "sla": { + "performance_degradation": { + "max_degradation": max_degradation + } + } + } + ] + } + + def test_sla_fail(self): + rally = utils.Rally() + cfg = self._get_sample_task_config(max_degradation=1) + config = utils.TaskConfig(cfg) + rally("task start --task %s" % config.filename) + self.assertRaises(utils.RallyCliError, rally, "task sla_check") + + def test_sla_success(self): + rally = utils.Rally() + config = utils.TaskConfig(self._get_sample_task_config()) + rally("task start --task %s" % config.filename) + rally("task sla_check") + expected = [ + {"benchmark": "Dummy.dummy_random_action", + "criterion": "performance_degradation", + "detail": mock.ANY, + "pos": 0, "status": "PASS"}, + ] + data = rally("task sla_check --json", getjson=True) + self.assertEqual(expected, data)