openstack/rally
Code Issues Proposed changes

Added workload formulas

Browse Source 
Workload formulas specify the concurrency and the number of iterations
for any particular test scenario based on the configuration of open stack
being tested. The first commit contains the initial versions of this
formulas. All of them scale concurrency and number of iterations
proportionally to number of controllers. These formulas need to be revised
and fine-tuned iteratively.

Also added support for basic math functions (min, max, round, ceil)
inside jinja templates.

Change-Id: I956bc3d9ff831b098cc8be3ee55283a567360906
This commit is contained in:
Julia Varigina 2015-10-19 17:44:47 -07:00
parent d39d629f04
commit 4eab35d6ca
10 changed files with 111 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
certification/openstack/macro/macro.yaml
View File

@ -33,9 +33,9 @@
volumes: -1
{%- endmacro %}
{%- macro constant_runner(concurrency=1, times=1) %}
{%- macro constant_runner(concurrency=1, times=1, is_smoke=True) %}
type: constant
{%- if smoke %}
{%- if is_smoke %}
concurrency: 1
times: 1
{%- else %}
@ -44,9 +44,9 @@
{%- endif %}
{%- endmacro %}
{%- macro rps_runner(rps=1, times=1) %}
{%- macro rps_runner(rps=1, times=1, is_smoke=True) %}
type: rps
{%- if smoke %}
{%- if is_smoke %}
rps: 1
times: 1
{%- else %}

2
certification/openstack/scenario/authentication.yaml
View File

@ -3,6 +3,6 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ rps_runner() }}
{{ rps_runner(rps=15*controllers_amount, times=20000*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}

26
certification/openstack/scenario/cinder.yaml
View File

@ -8,7 +8,7 @@
{{ unlimited_volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(30, 10*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -23,7 +23,7 @@
{{ volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -39,7 +39,7 @@
{{ unlimited_volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
-
@ -51,7 +51,7 @@
{{ unlimited_volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
-
@ -63,7 +63,7 @@
{{ unlimited_volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -78,7 +78,7 @@
{{ unlimited_volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -94,7 +94,7 @@
{{ volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -109,7 +109,7 @@
{{ unlimited_volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
-
@ -122,7 +122,7 @@
{{ unlimited_volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(100, 33*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -140,7 +140,7 @@
{{ unlimited_volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(40, 13*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -155,7 +155,7 @@
{{ volumes() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -177,7 +177,7 @@
servers_per_tenant: 1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(10, 3*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -188,6 +188,6 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}

6
certification/openstack/scenario/glance.yaml
View File

@ -5,7 +5,7 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -16,7 +16,7 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -25,6 +25,6 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}

14
certification/openstack/scenario/keystone.yaml
View File

@ -3,7 +3,7 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }}
{{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -12,7 +12,7 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }}
{{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -21,7 +21,7 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount),times=min(200, 10*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -30,7 +30,7 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }}
{{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -39,7 +39,7 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }}
{{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -48,7 +48,7 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }}
{{ constant_runner(concurrency=min(50, 3*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -57,6 +57,6 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner(15*controllers_amount,60*controllers_amount) }}
{{ constant_runner(concurrency=min(50, 7*controllers_amount),times=min(200, 67*controllers_amount), is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}

26
certification/openstack/scenario/neutron.yaml
View File

@ -9,7 +9,7 @@
network: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -27,7 +27,7 @@
port: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -49,7 +49,7 @@
router: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -68,7 +68,7 @@
subnet: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -83,7 +83,7 @@
network: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -101,7 +101,7 @@
port: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -122,7 +122,7 @@
router: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -141,7 +141,7 @@
subnet: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -159,7 +159,7 @@
network: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -182,7 +182,7 @@
port: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -207,7 +207,7 @@
router: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -229,7 +229,7 @@
subnet: -1
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -240,6 +240,6 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=2*controllers_amount, times=8*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}

22
certification/openstack/scenario/nova.yaml
View File

@ -12,7 +12,7 @@
{{ unlimited_nova(keypairs=true) }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -24,7 +24,7 @@
{{ unlimited_nova(keypairs=true) }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=67*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -36,7 +36,7 @@
{{ unlimited_nova(keypairs=true) }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=67*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -63,7 +63,7 @@
{{ unlimited_nova() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -81,7 +81,7 @@
{{ unlimited_nova() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -100,7 +100,7 @@
{{ unlimited_nova() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 10*controllers_amount), times=333*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -122,7 +122,7 @@
{{ unlimited_nova() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -142,7 +142,7 @@
{{ unlimited_nova() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 3*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -161,7 +161,7 @@
{{ unlimited_nova() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -179,7 +179,7 @@
{{ unlimited_nova() }}
{% endcall %}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}
@ -190,6 +190,6 @@
context:
{{ user_context(tenants_amount, users_amount, use_existing_users) }}
runner:
{{ constant_runner() }}
{{ constant_runner(concurrency=min(50, 2*controllers_amount), times=17*controllers_amount, is_smoke=smoke) }}
sla:
{{ no_failures_sla() }}

6
certification/openstack/task_arguments.yaml
View File

@ -12,4 +12,8 @@
smoke: true
users_amount: 1
tenants_amount: 1
controllers_amount: 1
controllers_amount: 3
compute_amount: 77
storage_amount: 20
network_amount: 1

41
rally/api.py
View File

@ -156,13 +156,29 @@ class Task(object):
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(template_dir, encoding="utf8"))
env.globals.update(cls.create_template_functions())
ast = env.parse(task_template)
# NOTE(Julia Varigina):
# Bug in jinja2.meta.find_undeclared_variables
#
# The method shows inconsistent behavior:
# it does not return undeclared variables that appear
# in included templates only (via {%- include "some_template.yaml"-%})
# and in the same time is declared in jinja2.Environment.globals.
#
# This is different for undeclared variables that appear directly
# in task_template. The method jinja2.meta.find_undeclared_variables
# returns an undeclared variable that is used in task_template
# and is set in jinja2.Environment.globals.
#
# Despite this bug, jinja resolves values
# declared in jinja2.Environment.globals for both types of undeclared
# variables and successfully renders templates in both cases.
required_kwargs = jinja2.meta.find_undeclared_variables(ast)
missing = set(required_kwargs) - set(kwargs) - set(dir(builtins))
missing = (set(required_kwargs) - set(kwargs) - set(dir(builtins)) -
set(env.globals))
real_missing = [mis for mis in missing
if is_really_missing(mis, task_template)]
if real_missing:
multi_msg = _("Please specify next template task arguments: %s")
single_msg = _("Please specify template task argument: %s")
@ -172,6 +188,25 @@ class Task(object):
return env.from_string(task_template).render(**kwargs)
@classmethod
def create_template_functions(cls):
def template_min(int1, int2):
return min(int1, int2)
def template_max(int1, int2):
return max(int1, int2)
def template_round(float1):
return int(round(float1))
def template_ceil(float1):
import math
return int(math.ceil(float1))
return {"min": template_min, "max": template_max,
"ceil": template_ceil, "round": template_round}
@classmethod
def create(cls, deployment, tag):
"""Create a task without starting it.

16
tests/unit/test_api.py
View File

@ -132,6 +132,22 @@ class TaskAPITestCase(test.TestCase):
os.path.dirname(other_template_path))
self.assertEqual(expect, actual)
def test_render_template_min(self):
template = "{{ min(1, 2)}}"
self.assertEqual("1", api.Task.render_template(template))
def test_render_template_max(self):
template = "{{ max(1, 2)}}"
self.assertEqual("2", api.Task.render_template(template))
def test_render_template_ceil(self):
template = "{{ ceil(2.2)}}"
self.assertEqual("3", api.Task.render_template(template))
def test_render_template_round(self):
template = "{{ round(2.2)}}"
self.assertEqual("2", api.Task.render_template(template))
@mock.patch("rally.common.objects.Deployment.get",
return_value={"uuid": "b0d9cd6c-2c94-4417-a238-35c7019d0257"})
@mock.patch("rally.common.objects.Task")