From e12e5b184f0e1d13b0479d57b0a5452baeedb4d5 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Thu, 17 Aug 2017 21:09:30 +0300 Subject: [PATCH] Make task format v2 more user-friendly - move scenario name and args into separate section - rename section 'context' to 'contexts' Change-Id: Ib683dfb81610030b4fccb119f2fb1a6356d1dce1 --- rally-jobs/rally-neutron-existing-users.yaml | 124 +++--- rally-jobs/self-rally.yaml | 405 +++++++++---------- rally/task/engine.py | 65 ++- tests/unit/task/test_engine.py | 76 ++-- 4 files changed, 334 insertions(+), 336 deletions(-) diff --git a/rally-jobs/rally-neutron-existing-users.yaml b/rally-jobs/rally-neutron-existing-users.yaml index 4782f2e6d3..5c99c3cd01 100644 --- a/rally-jobs/rally-neutron-existing-users.yaml +++ b/rally-jobs/rally-neutron-existing-users.yaml @@ -13,41 +13,41 @@ title: Test main Cinder actions workloads: - - name: CinderVolumes.create_volume - args: - size: 1 + scenario: + CinderVolumes.create_volume: + size: 1 runner: - type: "constant" - times: 2 - concurrency: 2 + constant: + times: 2 + concurrency: 2 sla: failure_rate: max: 0 - - name: CinderVolumes.create_volume - args: - size: 1 - image: - name: {{image_name}} + scenario: + CinderVolumes.create_volume: + size: 1 + image: + name: {{image_name}} runner: - type: "constant" - times: 1 - concurrency: 1 + constant: + times: 1 + concurrency: 1 sla: failure_rate: max: 0 - - name: CinderVolumes.create_snapshot_and_attach_volume - args: - volume_type: "lvmdriver-1" - size: - min: 1 - max: 1 + scenario: + CinderVolumes.create_snapshot_and_attach_volume: + volume_type: "lvmdriver-1" + size: + min: 1 + max: 1 runner: - type: "constant" - times: 2 - concurrency: 2 - context: + constant: + times: 2 + concurrency: 2 + contexts: servers: image: name: {{image_name}} @@ -61,17 +61,17 @@ title: Test main Nova actions workloads: - - name: NovaServers.boot_and_list_server - args: - flavor: - name: {{flavor_name}} - image: - name: {{image_name}} - detailed: True + scenario: + NovaServers.boot_and_list_server: + flavor: + name: {{flavor_name}} + image: + name: {{image_name}} + detailed: True runner: - type: "constant" - times: 2 - concurrency: 2 + constant: + times: 2 + concurrency: 2 sla: failure_rate: max: 0 @@ -79,15 +79,15 @@ title: Test main Glance actions workloads: - - name: GlanceImages.create_and_delete_image - args: - image_location: "{{ cirros_image_url }}" - container_format: "bare" - disk_format: "qcow2" + scenario: + GlanceImages.create_and_delete_image: + image_location: "{{ cirros_image_url }}" + container_format: "bare" + disk_format: "qcow2" runner: - type: "constant" - times: 1 - concurrency: 1 + constant: + times: 1 + concurrency: 1 sla: failure_rate: max: 100 @@ -95,37 +95,37 @@ title: Test main Neutron actions workloads: - - name: NeutronNetworks.create_and_list_networks - args: - network_create_args: + scenario: + NeutronNetworks.create_and_list_networks: + network_create_args: runner: - type: "constant" - times: 2 - concurrency: 2 + constant: + times: 2 + concurrency: 2 sla: failure_rate: max: 0 - - name: NeutronNetworks.create_and_list_subnets - args: - subnet_cidr_start: "1.1.0.0/30" - subnets_per_network: 2 + scenario: + NeutronNetworks.create_and_list_subnets: + subnet_cidr_start: "1.1.0.0/30" + subnets_per_network: 2 runner: - type: "constant" - times: 2 - concurrency: 2 + constant: + times: 2 + concurrency: 2 sla: failure_rate: max: 0 - - name: NeutronNetworks.create_and_list_floating_ips - args: - floating_network: "public" - floating_ip_args: {} + scenario: + NeutronNetworks.create_and_list_floating_ips: + floating_network: "public" + floating_ip_args: {} runner: - type: "constant" - times: 2 - concurrency: 2 + constant: + times: 2 + concurrency: 2 sla: failure_rate: max: 0 diff --git a/rally-jobs/self-rally.yaml b/rally-jobs/self-rally.yaml index ca7cbb023f..a0c86516bd 100644 --- a/rally-jobs/self-rally.yaml +++ b/rally-jobs/self-rally.yaml @@ -8,14 +8,14 @@ title: Test SLA plugins workloads: - - name: Dummy.dummy description: "Check SLA" - args: - sleep: 0.25 + scenario: + Dummy.dummy: + sleep: 0.25 runner: - type: "constant" - times: 20 - concurrency: 5 + constant: + times: 20 + concurrency: 5 sla: failure_rate: max: 0 @@ -28,31 +28,31 @@ performance_degradation: max_degradation: 50 - - name: Dummy.failure description: Check failure_rate SLA plugin - args: - sleep: 0.2 - from_iteration: 5 - to_iteration: 15 - each: 2 + scenario: + Dummy.failure: + sleep: 0.2 + from_iteration: 5 + to_iteration: 15 + each: 2 runner: - type: "constant" - times: 20 - concurrency: 5 + constant: + times: 20 + concurrency: 5 sla: failure_rate: min: 25 max: 25 - - name: Dummy.dummy_timed_atomic_actions description: Check max_avg_duration_per_atomic SLA plugin - args: - number_of_actions: 5 - sleep_factor: 1 + scenario: + Dummy.dummy_timed_atomic_actions: + number_of_actions: 5 + sleep_factor: 1 runner: - type: "constant" - times: 3 - concurrency: 3 + constant: + times: 3 + concurrency: 3 sla: max_avg_duration_per_atomic: action_0: 1.0 @@ -65,40 +65,40 @@ title: Test constant runner workloads: - - name: Dummy.dummy description: "Check 'constant' runner." - args: - sleep: 0.25 + scenario: + Dummy.dummy: + sleep: 0.25 runner: - type: "constant" - times: 8 - concurrency: 4 - max_cpu_count: 2 + constant: + times: 8 + concurrency: 4 + max_cpu_count: 2 sla: failure_rate: max: 0 - - name: Dummy.dummy - args: - sleep: 0 + scenario: + Dummy.dummy: + sleep: 0 runner: - type: "constant" - times: 4500 - concurrency: 20 + constant: + times: 4500 + concurrency: 20 sla: failure_rate: max: 0 - - name: Dummy.dummy description: > Check the ability of constant runner to terminate scenario by timeout. - args: - sleep: 30 + scenario: + Dummy.dummy: + sleep: 30 runner: - type: "constant" - times: 2 - concurrency: 2 - timeout: 1 + constant: + times: 2 + concurrency: 2 + timeout: 1 sla: failure_rate: min: 100 @@ -107,14 +107,14 @@ title: Test constant_for_duration runner workloads: - - name: Dummy.dummy description: "Check 'constant_for_duration' runner." - args: - sleep: 0.1 + scenario: + Dummy.dummy: + sleep: 0.1 runner: - type: "constant_for_duration" - duration: 5 - concurrency: 5 + constant_for_duration: + duration: 5 + concurrency: 5 sla: failure_rate: max: 0 @@ -123,104 +123,104 @@ title: Test rps runner workloads: - - name: Dummy.dummy description: "Check 'rps' runner." - args: - sleep: 0.001 + scenario: + Dummy.dummy: + sleep: 0.001 runner: - type: "rps" - times: 2000 - rps: 200 + rps: + times: 2000 + rps: 200 sla: failure_rate: max: 0 - - name: Dummy.dummy description: > Check 'rps' runner with float value of requests per second. - args: - sleep: 0.1 + scenario: + Dummy.dummy: + sleep: 0.1 runner: - type: "rps" - times: 5 - rps: 0.5 + rps: + times: 5 + rps: 0.5 sla: failure_rate: max: 0 - - name: Dummy.dummy description: > Check 'rps' runner with float value of requests per second. - args: - sleep: 0.1 + scenario: + Dummy.dummy: + sleep: 0.1 runner: - type: "rps" - times: 5 - rps: 0.2 + rps: + times: 5 + rps: 0.2 sla: failure_rate: max: 0 - - name: Dummy.dummy description: > Check 'max_concurrency' and 'max_cpu_count' properties of 'rps' runner. - args: - sleep: 0.001 + scenario: + Dummy.dummy: + sleep: 0.001 runner: - type: "rps" - times: 200 - rps: 20 - max_concurrency: 10 - max_cpu_count: 3 - sla: - failure_rate: - max: 0 - - - name: Dummy.dummy - description: "Check 'rps' with start, end, step arguments" - args: - sleep: 0.25 - runner: - type: "rps" - times: 55 rps: - start: 1 - end: 10 - step: 1 - max_concurrency: 10 - max_cpu_count: 3 + times: 200 + rps: 20 + max_concurrency: 10 + max_cpu_count: 3 sla: failure_rate: max: 0 - - name: Dummy.dummy description: "Check 'rps' with start, end, step arguments" - args: - sleep: 0.5 + scenario: + Dummy.dummy: + sleep: 0.25 runner: - type: "rps" - times: 55 rps: - start: 1 - end: 10 - step: 1 - duration: 2 - max_concurrency: 10 - max_cpu_count: 3 + times: 55 + rps: + start: 1 + end: 10 + step: 1 + max_concurrency: 10 + max_cpu_count: 3 + sla: + failure_rate: + max: 0 + - + description: "Check 'rps' with start, end, step arguments" + scenario: + Dummy.dummy: + sleep: 0.5 + runner: + rps: + times: 55 + rps: + start: 1 + end: 10 + step: 1 + duration: 2 + max_concurrency: 10 + max_cpu_count: 3 sla: failure_rate: max: 0 - - name: Dummy.dummy description: > Check the ability of rps runner to terminate scenario by timeout. - args: - sleep: 30 + scenario: + Dummy.dummy: + sleep: 30 runner: - type: "rps" - times: 1 - rps: 1 - timeout: 1 + rps: + times: 1 + rps: 1 + timeout: 1 sla: failure_rate: min: 100 @@ -229,13 +229,13 @@ title: Test serial runner workloads: - - name: Dummy.dummy description: "Check 'serial' runner." - args: - sleep: 0.1 + scenario: + Dummy.dummy: + sleep: 0.1 runner: - type: "serial" - times: 20 + serial: + times: 20 sla: failure_rate: max: 0 @@ -244,14 +244,14 @@ title: Test Hook and Trigger plugins workloads: - - name: Dummy.dummy description: "Check sys_call hook." - args: - sleep: 0.75 + scenario: + Dummy.dummy: + sleep: 0.75 runner: - type: "constant" - times: 20 - concurrency: 2 + constant: + times: 20 + concurrency: 2 hooks: - name: sys_call description: Run script @@ -281,14 +281,14 @@ failure_rate: max: 0 - - name: Dummy.dummy description: "Check periodic trigger with iteration unit." - args: - sleep: 0.25 + scenario: + Dummy.dummy: + sleep: 0.25 runner: - type: "constant" - times: 10 - concurrency: 2 + constant: + times: 10 + concurrency: 2 hooks: - name: sys_call description: test hook @@ -304,14 +304,13 @@ failure_rate: max: 0 - - name: Dummy.dummy description: "Check event trigger args." - args: - sleep: 1 + scenario: + Dummy.dummy: + sleep: 1 runner: - type: "constant" - times: 10 - concurrency: 1 + serial: + times: 10 hooks: - name: sys_call description: Get system name @@ -325,14 +324,13 @@ failure_rate: max: 0 - - name: Dummy.dummy description: "Check periodic trigger with time unit." - args: - sleep: 1 + scenario: + Dummy.dummy: + sleep: 1 runner: - type: "constant" - times: 10 - concurrency: 1 + serial: + times: 10 hooks: - name: sys_call description: test hook @@ -349,66 +347,67 @@ title: Test Dummy scenarios workloads: - - name: Dummy.dummy_exception - args: - size_of_message: 5 + scenario: + Dummy.dummy_exception: + size_of_message: 5 runner: - type: "constant" - times: 20 - concurrency: 5 + constant: + times: 20 + concurrency: 5 - - name: Dummy.dummy_exception_probability - args: - exception_probability: 0.05 + scenario: + Dummy.dummy_exception_probability: + exception_probability: 0.05 runner: - type: "constant" - times: 2042 - concurrency: 1 + serial: + times: 2042 - - name: Dummy.dummy_exception_probability - args: - exception_probability: 0.5 + scenario: + Dummy.dummy_exception_probability: + exception_probability: 0.5 runner: - type: "constant" - times: 100 - concurrency: 1 + serial: + times: 100 sla: failure_rate: min: 20 max: 80 - - name: Dummy.dummy_output + scenario: + Dummy.dummy_output: {} runner: - type: "constant" - times: 20 - concurrency: 10 + constant: + times: 20 + concurrency: 10 sla: failure_rate: max: 0 - - name: Dummy.dummy_random_fail_in_atomic - args: - exception_probability: 0.5 + scenario: + Dummy.dummy_random_fail_in_atomic: + exception_probability: 0.5 runner: - type: "constant" - times: 50 - concurrency: 10 + constant: + times: 50 + concurrency: 10 - - name: Dummy.dummy_random_action + scenario: + Dummy.dummy_random_action: {} runner: - type: "constant" - times: 10 - concurrency: 5 + constant: + times: 10 + concurrency: 5 - title: Test function based scenario workloads: - - name: FakePlugin.testplugin + scenario: + FakePlugin.testplugin: {} runner: - type: "constant" - times: 4 - concurrency: 4 + constant: + times: 4 + concurrency: 4 sla: failure_rate: max: 0 @@ -417,39 +416,39 @@ title: Profile generate_random_name method workloads: - - name: RallyProfile.generate_names_in_atomic - args: - number_of_names: 100 + scenario: + RallyProfile.generate_names_in_atomic: + number_of_names: 100 runner: - type: "constant" - times: 1000 - concurrency: 10 + constant: + times: 1000 + concurrency: 10 sla: max_avg_duration_per_atomic: generate_100_names: 0.015 failure_rate: max: 0 - - name: RallyProfile.generate_names_in_atomic - args: - number_of_names: 1000 + scenario: + RallyProfile.generate_names_in_atomic: + number_of_names: 1000 runner: - type: "constant" - times: 500 - concurrency: 10 + constant: + times: 500 + concurrency: 10 sla: max_avg_duration_per_atomic: generate_1000_names: 0.1 failure_rate: max: 0 - - name: RallyProfile.generate_names_in_atomic - args: - number_of_names: 10000 + scenario: + RallyProfile.generate_names_in_atomic: + number_of_names: 10000 runner: - type: "constant" - times: 200 - concurrency: 10 + constant: + times: 200 + concurrency: 10 sla: max_avg_duration_per_atomic: generate_10000_names: 1 @@ -460,26 +459,26 @@ title: Profile atomic actions workloads: - - name: RallyProfile.calculate_atomic - args: - number_of_atomics: 100 + scenario: + RallyProfile.calculate_atomic: + number_of_atomics: 100 runner: - type: "constant" - times: 300 - concurrency: 10 + constant: + times: 300 + concurrency: 10 sla: max_avg_duration_per_atomic: calculate_100_atomics: 0.04 failure_rate: max: 0 - - name: RallyProfile.calculate_atomic - args: - number_of_atomics: 500 + scenario: + RallyProfile.calculate_atomic: + number_of_atomics: 500 runner: - type: "constant" - times: 100 - concurrency: 10 + constant: + times: 100 + concurrency: 10 sla: max_avg_duration_per_atomic: calculate_500_atomics: 0.5 diff --git a/rally/task/engine.py b/rally/task/engine.py index 7b55db39ab..e9f843b861 100644 --- a/rally/task/engine.py +++ b/rally/task/engine.py @@ -638,27 +638,20 @@ class TaskConfig(object): "items": { "type": "object", "properties": { - "name": {"type": "string"}, + "scenario": { + "$ref": "#/definitions/singleEntity"}, "description": {"type": "string"}, - "args": {"type": "object"}, - "runner": { - "type": "object", - "properties": { - "type": {"type": "string"} - }, - "required": ["type"] - }, - + "$ref": "#/definitions/singleEntity"}, "sla": {"type": "object"}, "hooks": { "type": "array", "items": HOOK_CONFIG, }, - "context": {"type": "object"} + "contexts": {"type": "object"} }, "additionalProperties": False, - "required": ["name", "runner"] + "required": ["scenario", "runner"] } } }, @@ -668,7 +661,17 @@ class TaskConfig(object): } }, "additionalProperties": False, - "required": ["title", "subtasks"] + "required": ["title", "subtasks"], + "definitions": { + "singleEntity": { + "type": "object", + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + ".*": {"type": "object"} + } + } + } } CONFIG_SCHEMAS = {1: CONFIG_SCHEMA_V1, 2: CONFIG_SCHEMA_V2} @@ -676,13 +679,15 @@ class TaskConfig(object): def __init__(self, config): """TaskConfig constructor. + Validates and represents different versions of task configuration in + unified form. + :param config: Dict with configuration of specified task + :raises Exception: in case of validation error. (This gets reraised as + InvalidTaskException. if we raise it here as InvalidTaskException, + then "Task config is invalid: " gets prepended to the message twice """ if config is None: - # NOTE(stpierre): This gets reraised as - # InvalidTaskException. if we raise it here as - # InvalidTaskException, then "Task config is invalid: " - # gets prepended to the message twice. raise Exception(_("Input task is empty")) self.version = self._get_version(config) @@ -708,7 +713,13 @@ class TaskConfig(object): workloads = [] for position, wconf in enumerate(sconf["workloads"]): # fill all missed properties of a Workload + + wconf["name"], wconf["args"] = list( + wconf["scenario"].items())[0] + del wconf["scenario"] + wconf["position"] = position + if not wconf.get("description", ""): try: wconf["description"] = scenario.Scenario.get( @@ -718,12 +729,16 @@ class TaskConfig(object): # let's fail an issue with loading plugin at a # validation step pass - wconf.setdefault("args", {}) - wconf.setdefault("context", {}) - wconf.setdefault("runner", {}) + + wconf["context"] = wconf.pop("contexts", {}) + + runner_type, runner_cfg = list( + wconf["runner"].items())[0] + runner_cfg["type"] = runner_type + wconf["runner"] = runner_cfg + wconf.setdefault("sla", {}) wconf.setdefault("hooks", []) - # store hooks in the format which we have in db wconf["hooks"] = [{"config": h} for h in wconf["hooks"]] workloads.append(wconf) sconf["workloads"] = workloads @@ -755,7 +770,13 @@ class TaskConfig(object): for name, v1_workloads in config.items(): for v1_workload in v1_workloads: v2_workload = copy.deepcopy(v1_workload) - v2_workload["name"] = name + v2_workload["scenario"] = {name: v2_workload.pop("args", {})} + v2_workload["sla"] = v2_workload.pop("sla", {}) + v2_workload["contexts"] = v2_workload.pop("context", {}) + if "runner" in v2_workload: + runner_type = v2_workload["runner"].pop("type") + v2_workload["runner"] = { + runner_type: v2_workload["runner"]} subtasks.append({"title": name, "workloads": [v2_workload]}) return {"title": "Task (adopted from task format v1)", "subtasks": subtasks} diff --git a/tests/unit/task/test_engine.py b/tests/unit/task/test_engine.py index e48b4ee898..e9b6321e47 100644 --- a/tests/unit/task/test_engine.py +++ b/tests/unit/task/test_engine.py @@ -1020,53 +1020,31 @@ class TaskTestCase(test.TestCase): self.assertRaises(exceptions.InvalidTaskException, engine.TaskConfig, mock.MagicMock) - @mock.patch("rally.task.engine.TaskConfig._get_version") - @mock.patch("rally.task.engine.TaskConfig._validate_json") - def test__adopt_task_format_v1( - self, mock_task_config__validate_json, - mock_task_config__get_version): - mock_task_config__get_version.return_value = 1 + def test__adopt_task_format_v1(self): + + # mock all redundant checks :) + class TaskConfig(engine.TaskConfig): + def __init__(self): + pass + config = collections.OrderedDict() - config["a.task"] = [{"s": 1}, {"s": 2}] - config["b.task"] = [{"s": 3}] - self.assertEqual([ - {"title": "a.task", - "context": {}, - "description": None, - "group": None, - "tags": [], - "workloads": [{"s": 1, - "name": "a.task", - "args": {}, - "context": {}, - "runner": {}, - "sla": {}, - "hooks": [], - "position": 0}]}, - {"title": "a.task", - "context": {}, - "description": None, - "group": None, - "tags": [], - "workloads": [{"s": 2, - "name": "a.task", - "args": {}, - "context": {}, - "runner": {}, - "sla": {}, - "hooks": [], - "position": 0}]}, - {"title": "b.task", - "context": {}, - "description": None, - "group": None, - "tags": [], - "workloads": [{"s": 3, - "name": "b.task", - "args": {}, - "context": {}, - "runner": {}, - "sla": {}, - "hooks": [], - "position": 0}]} - ], engine.TaskConfig(config).subtasks) + config["a.task"] = [{"s": 1, "context": {"foo": "bar"}}, {"s": 2}] + config["b.task"] = [{"s": 3, "sla": {"key": "value"}}] + self.assertEqual( + {"title": "Task (adopted from task format v1)", + "subtasks": [{"title": "a.task", + "workloads": [{"s": 1, + "scenario": {"a.task": {}}, + "sla": {}, + "contexts": {"foo": "bar"}}]}, + {"title": "a.task", + "workloads": [{"s": 2, + "scenario": {"a.task": {}}, + "sla": {}, + "contexts": {}}]}, + {"title": "b.task", + "workloads": [{"s": 3, + "scenario": {"b.task": {}}, + "sla": {"key": "value"}, + "contexts": {}}]}]}, + TaskConfig._adopt_task_format_v1(config))