Added workload formulas

Workload formulas specify the concurrency and the number of iterations
for any particular test scenario based on the configuration of open stack
being tested. The first commit contains the initial versions of this
formulas. All of them scale concurrency and number of iterations
proportionally to number of controllers. These formulas need to be revised
and fine-tuned iteratively.

Also added support for basic math functions (min, max, round, ceil)
inside jinja templates.

Change-Id: I956bc3d9ff831b098cc8be3ee55283a567360906
This commit is contained in:
Julia Varigina 2015-10-19 17:44:47 -07:00
parent d39d629f04
commit 4eab35d6ca
10 changed files with 111 additions and 56 deletions

View File

@ -33,9 +33,9 @@
volumes: -1 volumes: -1
{%- endmacro %} {%- endmacro %}
{%- macro constant_runner(concurrency=1, times=1) %} {%- macro constant_runner(concurrency=1, times=1, is_smoke=True) %}
type: constant type: constant
{%- if smoke %} {%- if is_smoke %}
concurrency: 1 concurrency: 1
times: 1 times: 1
{%- else %} {%- else %}
@ -44,9 +44,9 @@
{%- endif %} {%- endif %}
{%- endmacro %} {%- endmacro %}
{%- macro rps_runner(rps=1, times=1) %} {%- macro rps_runner(rps=1, times=1, is_smoke=True) %}
type: rps type: rps
{%- if smoke %} {%- if is_smoke %}
rps: 1 rps: 1
times: 1 times: 1
{%- else %} {%- else %}

View File

@ -3,6 +3,6 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ rps_runner() }} {{ rps_runner(rps=15*controllers_amount, times=20000*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}

View File

@ -8,7 +8,7 @@
{{ unlimited_volumes() }} {{ unlimited_volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(30, 10*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -23,7 +23,7 @@
{{ volumes() }} {{ volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -39,7 +39,7 @@
{{ unlimited_volumes() }} {{ unlimited_volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
- -
@ -51,7 +51,7 @@
{{ unlimited_volumes() }} {{ unlimited_volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
- -
@ -63,7 +63,7 @@
{{ unlimited_volumes() }} {{ unlimited_volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -78,7 +78,7 @@
{{ unlimited_volumes() }} {{ unlimited_volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -94,7 +94,7 @@
{{ volumes() }} {{ volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -109,7 +109,7 @@
{{ unlimited_volumes() }} {{ unlimited_volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
- -
@ -122,7 +122,7 @@
{{ unlimited_volumes() }} {{ unlimited_volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -140,7 +140,7 @@
{{ unlimited_volumes() }} {{ unlimited_volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(40, 13*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -155,7 +155,7 @@
{{ volumes() }} {{ volumes() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -177,7 +177,7 @@
servers_per_tenant: 1 servers_per_tenant: 1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(10, 3*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -188,6 +188,6 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}

View File

@ -5,7 +5,7 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -16,7 +16,7 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -25,6 +25,6 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}

View File

@ -3,7 +3,7 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }} {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -12,7 +12,7 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }} {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -21,7 +21,7 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }} {{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 10*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -30,7 +30,7 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }} {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -39,7 +39,7 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }} {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -48,7 +48,7 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }} {{ constant_runner(concurrency=min(50, 3*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -57,6 +57,6 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }} {{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}

View File

@ -9,7 +9,7 @@
network: -1 network: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -27,7 +27,7 @@
port: -1 port: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -49,7 +49,7 @@
router: -1 router: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -68,7 +68,7 @@
subnet: -1 subnet: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -83,7 +83,7 @@
network: -1 network: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -101,7 +101,7 @@
port: -1 port: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -122,7 +122,7 @@
router: -1 router: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -141,7 +141,7 @@
subnet: -1 subnet: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -159,7 +159,7 @@
network: -1 network: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -182,7 +182,7 @@
port: -1 port: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -207,7 +207,7 @@
router: -1 router: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -229,7 +229,7 @@
subnet: -1 subnet: -1
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -240,6 +240,6 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}

View File

@ -12,7 +12,7 @@
{{ unlimited_nova(keypairs=true) }} {{ unlimited_nova(keypairs=true) }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -24,7 +24,7 @@
{{ unlimited_nova(keypairs=true) }} {{ unlimited_nova(keypairs=true) }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=67*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -36,7 +36,7 @@
{{ unlimited_nova(keypairs=true) }} {{ unlimited_nova(keypairs=true) }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=67*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -63,7 +63,7 @@
{{ unlimited_nova() }} {{ unlimited_nova() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -81,7 +81,7 @@
{{ unlimited_nova() }} {{ unlimited_nova() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -100,7 +100,7 @@
{{ unlimited_nova() }} {{ unlimited_nova() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 10*controllers_amount), times=333*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -122,7 +122,7 @@
{{ unlimited_nova() }} {{ unlimited_nova() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -142,7 +142,7 @@
{{ unlimited_nova() }} {{ unlimited_nova() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 3*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -161,7 +161,7 @@
{{ unlimited_nova() }} {{ unlimited_nova() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -179,7 +179,7 @@
{{ unlimited_nova() }} {{ unlimited_nova() }}
{% endcall %} {% endcall %}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}
@ -190,6 +190,6 @@
context: context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }} {{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner: runner:
{{ constant_runner() }} {{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla: sla:
{{ no_failures_sla() }} {{ no_failures_sla() }}

View File

@ -12,4 +12,8 @@
smoke: true smoke: true
users_amount: 1 users_amount: 1
tenants_amount: 1 tenants_amount: 1
controllers_amount: 1 controllers_amount: 3
compute_amount: 77
storage_amount: 20
network_amount: 1

View File

@ -156,13 +156,29 @@ class Task(object):
env = jinja2.Environment( env = jinja2.Environment(
loader=jinja2.FileSystemLoader(template_dir, encoding="utf8")) loader=jinja2.FileSystemLoader(template_dir, encoding="utf8"))
env.globals.update(cls.create_template_functions())
ast = env.parse(task_template) ast = env.parse(task_template)
# NOTE(Julia Varigina):
# Bug in jinja2.meta.find_undeclared_variables
#
# The method shows inconsistent behavior:
# it does not return undeclared variables that appear
# in included templates only (via {%- include "some_template.yaml"-%})
# and in the same time is declared in jinja2.Environment.globals.
#
# This is different for undeclared variables that appear directly
# in task_template. The method jinja2.meta.find_undeclared_variables
# returns an undeclared variable that is used in task_template
# and is set in jinja2.Environment.globals.
#
# Despite this bug, jinja resolves values
# declared in jinja2.Environment.globals for both types of undeclared
# variables and successfully renders templates in both cases.
required_kwargs = jinja2.meta.find_undeclared_variables(ast) required_kwargs = jinja2.meta.find_undeclared_variables(ast)
missing = (set(required_kwargs) - set(kwargs) - set(dir(builtins)) -
missing = set(required_kwargs) - set(kwargs) - set(dir(builtins)) set(env.globals))
real_missing = [mis for mis in missing real_missing = [mis for mis in missing
if is_really_missing(mis, task_template)] if is_really_missing(mis, task_template)]
if real_missing: if real_missing:
multi_msg = _("Please specify next template task arguments: %s") multi_msg = _("Please specify next template task arguments: %s")
single_msg = _("Please specify template task argument: %s") single_msg = _("Please specify template task argument: %s")
@ -172,6 +188,25 @@ class Task(object):
return env.from_string(task_template).render(**kwargs) return env.from_string(task_template).render(**kwargs)
@classmethod
def create_template_functions(cls):
def template_min(int1, int2):
return min(int1, int2)
def template_max(int1, int2):
return max(int1, int2)
def template_round(float1):
return int(round(float1))
def template_ceil(float1):
import math
return int(math.ceil(float1))
return {"min": template_min, "max": template_max,
"ceil": template_ceil, "round": template_round}
@classmethod @classmethod
def create(cls, deployment, tag): def create(cls, deployment, tag):
"""Create a task without starting it. """Create a task without starting it.

View File

@ -132,6 +132,22 @@ class TaskAPITestCase(test.TestCase):
os.path.dirname(other_template_path)) os.path.dirname(other_template_path))
self.assertEqual(expect, actual) self.assertEqual(expect, actual)
def test_render_template_min(self):
template = "{{ min(1, 2)}}"
self.assertEqual("1", api.Task.render_template(template))
def test_render_template_max(self):
template = "{{ max(1, 2)}}"
self.assertEqual("2", api.Task.render_template(template))
def test_render_template_ceil(self):
template = "{{ ceil(2.2)}}"
self.assertEqual("3", api.Task.render_template(template))
def test_render_template_round(self):
template = "{{ round(2.2)}}"
self.assertEqual("2", api.Task.render_template(template))
@mock.patch("rally.common.objects.Deployment.get", @mock.patch("rally.common.objects.Deployment.get",
return_value={"uuid": "b0d9cd6c-2c94-4417-a238-35c7019d0257"}) return_value={"uuid": "b0d9cd6c-2c94-4417-a238-35c7019d0257"})
@mock.patch("rally.common.objects.Task") @mock.patch("rally.common.objects.Task")