From 4eab35d6ca910d204ac4a7def7c664c85982ea22 Mon Sep 17 00:00:00 2001 From: Julia Varigina Date: Mon, 19 Oct 2015 17:44:47 -0700 Subject: [PATCH] Added workload formulas Workload formulas specify the concurrency and the number of iterations for any particular test scenario based on the configuration of open stack being tested. The first commit contains the initial versions of this formulas. All of them scale concurrency and number of iterations proportionally to number of controllers. These formulas need to be revised and fine-tuned iteratively. Also added support for basic math functions (min, max, round, ceil) inside jinja templates. Change-Id: I956bc3d9ff831b098cc8be3ee55283a567360906 --- certification/openstack/macro/macro.yaml | 8 ++-- .../openstack/scenario/authentication.yaml | 2 +- certification/openstack/scenario/cinder.yaml | 26 ++++++------ certification/openstack/scenario/glance.yaml | 6 +-- .../openstack/scenario/keystone.yaml | 14 +++---- certification/openstack/scenario/neutron.yaml | 26 ++++++------ certification/openstack/scenario/nova.yaml | 22 +++++----- certification/openstack/task_arguments.yaml | 6 ++- rally/api.py | 41 +++++++++++++++++-- tests/unit/test_api.py | 16 ++++++++ 10 files changed, 111 insertions(+), 56 deletions(-) diff --git a/certification/openstack/macro/macro.yaml b/certification/openstack/macro/macro.yaml index 910d648ca4..7ed26b3756 100644 --- a/certification/openstack/macro/macro.yaml +++ b/certification/openstack/macro/macro.yaml @@ -33,9 +33,9 @@ volumes: -1 {%- endmacro %} -{%- macro constant_runner(concurrency=1, times=1) %} +{%- macro constant_runner(concurrency=1, times=1, is_smoke=True) %} type: constant - {%- if smoke %} + {%- if is_smoke %} concurrency: 1 times: 1 {%- else %} @@ -44,9 +44,9 @@ {%- endif %} {%- endmacro %} -{%- macro rps_runner(rps=1, times=1) %} +{%- macro rps_runner(rps=1, times=1, is_smoke=True) %} type: rps - {%- if smoke %} + {%- if is_smoke %} rps: 1 times: 1 {%- else %} diff --git a/certification/openstack/scenario/authentication.yaml b/certification/openstack/scenario/authentication.yaml index e7c416c0dd..d31f71e6af 100644 --- a/certification/openstack/scenario/authentication.yaml +++ b/certification/openstack/scenario/authentication.yaml @@ -3,6 +3,6 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ rps_runner() }} + {{ rps_runner(rps=15*controllers_amount, times=20000*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} diff --git a/certification/openstack/scenario/cinder.yaml b/certification/openstack/scenario/cinder.yaml index 89171f5ad9..c2fe11d1cc 100644 --- a/certification/openstack/scenario/cinder.yaml +++ b/certification/openstack/scenario/cinder.yaml @@ -8,7 +8,7 @@ {{ unlimited_volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(30, 10*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -23,7 +23,7 @@ {{ volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -39,7 +39,7 @@ {{ unlimited_volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} - @@ -51,7 +51,7 @@ {{ unlimited_volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} - @@ -63,7 +63,7 @@ {{ unlimited_volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -78,7 +78,7 @@ {{ unlimited_volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -94,7 +94,7 @@ {{ volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -109,7 +109,7 @@ {{ unlimited_volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} - @@ -122,7 +122,7 @@ {{ unlimited_volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -140,7 +140,7 @@ {{ unlimited_volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(40, 13*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -155,7 +155,7 @@ {{ volumes() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -177,7 +177,7 @@ servers_per_tenant: 1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(10, 3*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -188,6 +188,6 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} \ No newline at end of file diff --git a/certification/openstack/scenario/glance.yaml b/certification/openstack/scenario/glance.yaml index a184f6f3d8..39f46ac897 100644 --- a/certification/openstack/scenario/glance.yaml +++ b/certification/openstack/scenario/glance.yaml @@ -5,7 +5,7 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -16,7 +16,7 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -25,6 +25,6 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} diff --git a/certification/openstack/scenario/keystone.yaml b/certification/openstack/scenario/keystone.yaml index a79a8c582a..3db23b386a 100644 --- a/certification/openstack/scenario/keystone.yaml +++ b/certification/openstack/scenario/keystone.yaml @@ -3,7 +3,7 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner(15*controllers_amount,60*controllers_amount) }} + {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -12,7 +12,7 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner(15*controllers_amount,60*controllers_amount) }} + {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -21,7 +21,7 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner(15*controllers_amount,60*controllers_amount) }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 10*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -30,7 +30,7 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner(15*controllers_amount,60*controllers_amount) }} + {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -39,7 +39,7 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner(15*controllers_amount,60*controllers_amount) }} + {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -48,7 +48,7 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner(15*controllers_amount,60*controllers_amount) }} + {{ constant_runner(concurrency=min(50, 3*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -57,6 +57,6 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner(15*controllers_amount,60*controllers_amount) }} + {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }} sla: {{ no_failures_sla() }} diff --git a/certification/openstack/scenario/neutron.yaml b/certification/openstack/scenario/neutron.yaml index 9b18209220..0e6d5f45b1 100644 --- a/certification/openstack/scenario/neutron.yaml +++ b/certification/openstack/scenario/neutron.yaml @@ -9,7 +9,7 @@ network: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -27,7 +27,7 @@ port: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -49,7 +49,7 @@ router: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -68,7 +68,7 @@ subnet: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -83,7 +83,7 @@ network: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -101,7 +101,7 @@ port: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -122,7 +122,7 @@ router: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -141,7 +141,7 @@ subnet: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -159,7 +159,7 @@ network: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -182,7 +182,7 @@ port: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -207,7 +207,7 @@ router: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -229,7 +229,7 @@ subnet: -1 {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -240,6 +240,6 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} \ No newline at end of file diff --git a/certification/openstack/scenario/nova.yaml b/certification/openstack/scenario/nova.yaml index d66044d50e..8f8a8aaa4a 100644 --- a/certification/openstack/scenario/nova.yaml +++ b/certification/openstack/scenario/nova.yaml @@ -12,7 +12,7 @@ {{ unlimited_nova(keypairs=true) }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -24,7 +24,7 @@ {{ unlimited_nova(keypairs=true) }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=67*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -36,7 +36,7 @@ {{ unlimited_nova(keypairs=true) }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=67*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -63,7 +63,7 @@ {{ unlimited_nova() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -81,7 +81,7 @@ {{ unlimited_nova() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -100,7 +100,7 @@ {{ unlimited_nova() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 10*controllers_amount), times=333*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -122,7 +122,7 @@ {{ unlimited_nova() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -142,7 +142,7 @@ {{ unlimited_nova() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 3*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -161,7 +161,7 @@ {{ unlimited_nova() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -179,7 +179,7 @@ {{ unlimited_nova() }} {% endcall %} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} @@ -190,6 +190,6 @@ context: {{ user_context(tenants_amount, users_amount, use_existing_users) }} runner: - {{ constant_runner() }} + {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }} sla: {{ no_failures_sla() }} diff --git a/certification/openstack/task_arguments.yaml b/certification/openstack/task_arguments.yaml index 4b5da1dfd1..0cb7f59c7a 100644 --- a/certification/openstack/task_arguments.yaml +++ b/certification/openstack/task_arguments.yaml @@ -12,4 +12,8 @@ smoke: true users_amount: 1 tenants_amount: 1 - controllers_amount: 1 + controllers_amount: 3 + compute_amount: 77 + storage_amount: 20 + network_amount: 1 + diff --git a/rally/api.py b/rally/api.py index 52b4614cf9..182a1ff1f5 100644 --- a/rally/api.py +++ b/rally/api.py @@ -156,13 +156,29 @@ class Task(object): env = jinja2.Environment( loader=jinja2.FileSystemLoader(template_dir, encoding="utf8")) + env.globals.update(cls.create_template_functions()) ast = env.parse(task_template) + # NOTE(Julia Varigina): + # Bug in jinja2.meta.find_undeclared_variables + # + # The method shows inconsistent behavior: + # it does not return undeclared variables that appear + # in included templates only (via {%- include "some_template.yaml"-%}) + # and in the same time is declared in jinja2.Environment.globals. + # + # This is different for undeclared variables that appear directly + # in task_template. The method jinja2.meta.find_undeclared_variables + # returns an undeclared variable that is used in task_template + # and is set in jinja2.Environment.globals. + # + # Despite this bug, jinja resolves values + # declared in jinja2.Environment.globals for both types of undeclared + # variables and successfully renders templates in both cases. required_kwargs = jinja2.meta.find_undeclared_variables(ast) - - missing = set(required_kwargs) - set(kwargs) - set(dir(builtins)) + missing = (set(required_kwargs) - set(kwargs) - set(dir(builtins)) - + set(env.globals)) real_missing = [mis for mis in missing if is_really_missing(mis, task_template)] - if real_missing: multi_msg = _("Please specify next template task arguments: %s") single_msg = _("Please specify template task argument: %s") @@ -172,6 +188,25 @@ class Task(object): return env.from_string(task_template).render(**kwargs) + @classmethod + def create_template_functions(cls): + + def template_min(int1, int2): + return min(int1, int2) + + def template_max(int1, int2): + return max(int1, int2) + + def template_round(float1): + return int(round(float1)) + + def template_ceil(float1): + import math + return int(math.ceil(float1)) + + return {"min": template_min, "max": template_max, + "ceil": template_ceil, "round": template_round} + @classmethod def create(cls, deployment, tag): """Create a task without starting it. diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 64cc071991..fc6eda9a19 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -132,6 +132,22 @@ class TaskAPITestCase(test.TestCase): os.path.dirname(other_template_path)) self.assertEqual(expect, actual) + def test_render_template_min(self): + template = "{{ min(1, 2)}}" + self.assertEqual("1", api.Task.render_template(template)) + + def test_render_template_max(self): + template = "{{ max(1, 2)}}" + self.assertEqual("2", api.Task.render_template(template)) + + def test_render_template_ceil(self): + template = "{{ ceil(2.2)}}" + self.assertEqual("3", api.Task.render_template(template)) + + def test_render_template_round(self): + template = "{{ round(2.2)}}" + self.assertEqual("2", api.Task.render_template(template)) + @mock.patch("rally.common.objects.Deployment.get", return_value={"uuid": "b0d9cd6c-2c94-4417-a238-35c7019d0257"}) @mock.patch("rally.common.objects.Task")