performa/shaker
Code Issues Proposed changes

Introduce record id and reference from sla-record

Browse Source 
Rename field 'agent_id' into 'agent' to stay similar to other
references

Change-Id: Iad9c6d189200fefcbff119fba0f778d18079114c
This commit is contained in:
Ilya Shakhat 2015-04-20 19:21:30 +03:00
parent 523fbdb645
commit 1a65fba420
9 changed files with 73 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scenarios/misc/static_agent.yaml
View File

@ -1,3 +1,5 @@
title: Static Agents
description:
This scenario runs tests on pre-deployed static agents
@ -13,10 +15,13 @@ execution:
title: List all files
class: shell
program: ls -al
sla:
- "[type == 'agent'] >> (stderr == '')"
-
title: Run sample script
class: shell
script: |
#!/bin/bash
echo "hello"
echo "world"
echo "hello world"
sla:
- "[type == 'agent'] >> (stdout & '.*hello world.*')"

2
scenarios/networking/l2.yaml
View File

@ -1,3 +1,5 @@
title: L2
description:
This scenario launches pairs of VMs in the same private network. Every VM is
hosted on a separate compute node.

27
shaker/engine/report.py
View File

@ -33,13 +33,12 @@ LOG = logging.getLogger(__name__)
def calculate_stats(records, tests):
aggregates = []
# scenario -> test -> concurrency -> [record]
rec_map = collections.defaultdict(
functools.partial(collections.defaultdict,
functools.partial(collections.defaultdict, list)))
for record in records:
for record in records.values():
aggregator = aggregators.get_aggregator(tests[record['test']])
aggregator.record_summary(record)
@ -54,24 +53,26 @@ def calculate_stats(records, tests):
for concurrency, per_concurrency in per_test.items():
summary = aggregator.concurrency_summary(per_concurrency)
if summary:
summary.update(dict(scenario=scenario, test=test,
record_id = utils.make_record_id()
summary.update(dict(id=record_id,
scenario=scenario, test=test,
concurrency=concurrency,
type='concurrency'))
aggregates.append(summary)
records[record_id] = summary
concurrency_aggregates.append(summary)
per_test_summary = aggregator.test_summary(concurrency_aggregates)
if per_test_summary:
per_test_summary.update(dict(scenario=scenario, test=test,
record_id = utils.make_record_id()
per_test_summary.update(dict(id=record_id,
scenario=scenario, test=test,
type='test'))
aggregates.append(per_test_summary)
return aggregates
records[record_id] = per_test_summary
def verify_sla(records, tests):
record_map = collections.defaultdict(list) # test -> [record]
for r in records:
for r in records.values():
if 'sla' in tests[r['test']]:
record_map[r['test']].append(r)
@ -83,6 +84,11 @@ def verify_sla(records, tests):
return sla_records
def output_sla(sla_records):
return [dict(record=item.record['id'], state=item.state,
expression=item.expression) for item in sla_records]
def _get_location(record):
return '.'.join([str(record.get(s))
for s in ['scenario', 'test', 'concurrency',
@ -122,9 +128,10 @@ def generate_report(data, report_template, report_filename, subunit_filename):
LOG.debug('Generating report, template: %s, output: %s',
report_template, report_filename or '<dummy>')
data['records'] += calculate_stats(data['records'], data['tests'])
calculate_stats(data['records'], data['tests'])
sla_records = verify_sla(data['records'], data['tests'])
data['sla'] = output_sla(sla_records)
if subunit_filename:
save_to_subunit(sla_records, subunit_filename)

19
shaker/engine/server.py
View File

@ -73,7 +73,7 @@ def _pick_agents(agents, size):
def execute(quorum, execution, agents):
records = []
records = {}
for test in execution['tests']:
LOG.debug('Running test %s on all agents', test)
@ -85,16 +85,18 @@ def execute(quorum, execution, agents):
execution_result = quorum.execute(executors)
for agent_id, data in execution_result.items():
data.update(dict(
agent_id=agent_id,
for agent_id, record in execution_result.items():
record_id = utils.make_record_id()
record.update(dict(
id=record_id,
agent=agent_id,
node=agents[agent_id].get('node'),
concurrency=len(selected_agents),
test=test_title,
executor=test.get('class'),
type='agent',
))
records.append(data)
records[record_id] = record
LOG.info('Execution is done')
return records
@ -142,7 +144,8 @@ def play_scenario(scenario):
quorum.join(set(agents.keys()))
execution_result = execute(quorum, scenario['execution'], agents)
for record in execution_result:
# extend every record with reference to scenario
for record in execution_result.values():
record['scenario'] = (scenario.get('title') or
scenario.get('file_name'))
output['records'] = execution_result
@ -167,7 +170,7 @@ def main():
config.REPORT_OPTS
)
output = dict(records=[], agents={}, scenarios={}, tests={})
output = dict(records={}, agents={}, scenarios={}, tests={})
for scenario_file_name in [cfg.CONF.scenario]:
scenario = utils.read_yaml_file(scenario_file_name)
@ -177,7 +180,7 @@ def main():
play_output = play_scenario(scenario)
output['scenarios'][scenario['title']] = play_output['scenario']
output['records'] += play_output['records']
output['records'].update(play_output['records'])
output['agents'].update(play_output['agents'])
output['tests'].update(play_output['tests'])

2
shaker/engine/sla.py
View File

@ -112,7 +112,7 @@ def dump_ast_node(node):
elif isinstance(node, ast.Name):
return node.id
elif isinstance(node, ast.Str):
return '\'node.s\''
return '"%s"' % node.s
elif isinstance(node, ast.BinOp):
return '%s %s %s' % (_format(node.left), _operators[type(node.op)],
_format(node.right))

5
shaker/engine/utils.py
View File

@ -16,6 +16,7 @@
import logging as std_logging
import os
import random
import uuid
from oslo_config import cfg
from oslo_log import log as logging
@ -138,6 +139,10 @@ def random_string(length=6):
return ''.join(random.sample('adefikmoprstuz', length))
def make_record_id():
return str(uuid.uuid4())
def copy_dict_kv(source):
return dict((k, v) for k, v in source.items())

36
shaker/resources/report_template.html
View File

@ -45,13 +45,13 @@
$(function () {
const fields = ["scenario", "test", "concurrency", "node", "agent_id"];
const fields = ["scenario", "test", "concurrency", "node", "agent"];
const state = {
scenario: "",
test: "",
concurrency: "",
node: "",
agent_id: ""
agent: ""
};
const tablePageSize = 10;
@ -104,7 +104,7 @@
var values_set = {};
$.each(records, function (j, record) {
if (record["type"] == "raw") {
if (record["type"] == "agent" && record[field]) {
values_set["" + record[field]] = true;
}
});
@ -123,9 +123,8 @@
function filterRecords(records, type) {
var res = [];
for (var i = 0; i < records.length; i++) {
var record = records[i];
if (type != "" && record.type == type) {
$.each(records, function(_i, record){
if (type == "" || record.type == type) {
var flag = true;
$.each(state, function (key, value) {
if (value != "") {
@ -136,7 +135,7 @@
res.push(record);
}
}
}
});
return res;
}
@ -258,7 +257,7 @@
function showAgentsTable(contentArea, filteredRecords) {
var agent_ids = {};
$.each(filteredRecords, function(_i, record){
agent_ids[record.agent_id] = true;
agent_ids[record.agent] = true;
});
var agents = getAgentsByIds(agent_ids);
@ -324,7 +323,8 @@
/* Update cycle */
function update() {
var filteredRecords = filterRecords(report.records, "raw");
var filteredRecords = filterRecords(report.records, "agent");
console.log(filteredRecords);
var slaRecords = [];
$.each(filterRecords(report.records), function(_i, record) {
if (record["sla_info"]) {
@ -342,21 +342,21 @@
if (state["scenario"] != "" && state["test"] != "" &&
state["concurrency"] != "" && state["node"] != "" &&
state["agent_id"] != "") {
state["agent"] != "") {
showExecutorRecord(contentArea, filteredRecords[0]);
}
if (state["scenario"] != "" && state["test"] != "" &&
state["concurrency"] != "" && state["node"] == "" &&
state["agent_id"] == "") {
var agg_recs = filterRecords(report.records, "agg_concurrency");
state["agent"] == "") {
var agg_recs = filterRecords(report.records, "concurrency");
if (agg_recs.length > 0) {
showConcurrencySummary(contentArea, agg_recs[0]);
}
}
if (state["scenario"] != "" && state["test"] != "" &&
state["concurrency"] == "" && state["node"] == "" &&
state["agent_id"] == "") {
var test_recs = filterRecords(report.records, "agg_test");
state["agent"] == "") {
var test_recs = filterRecords(report.records, "test");
if (test_recs.length > 0) {
showTestSummary(contentArea, test_recs[0]);
}
@ -367,7 +367,7 @@
if (state["scenario"] != "") {
showScenarioSource(contentArea, report.scenarios[state["scenario"]]);
}
if (state["agent_id"] == "") {
if (state["agent"] == "") {
showAgentsTable(contentArea, filteredRecords);
}
}
@ -443,8 +443,8 @@
<select id="node_selector" class="form-control" data-placeholder="- Any node -"></select>
</div>
<div class="col-md-2">
<label for="agent_id_selector" title="Agent id">Agent</label>
<select id="agent_id_selector" class="form-control" data-placeholder="- Any agent -"></select>
<label for="agent_selector" title="Agent">Agent</label>
<select id="agent_selector" class="form-control" data-placeholder="- Any agent -"></select>
</div>
</div>
@ -538,7 +538,7 @@
<td><a href="{{ test_url }}">{{ test }}</a></td>
<td><a href="{{ concurrency_url }}">{{ concurrency }}</a></td>
<td><a href="{{ node_url }}">{{ node }}</a></td>
<td><a href="{{ agent_id_url }}">{{ agent_id }}</a></td>
<td><a href="{{ agent_url }}">{{ agent }}</a></td>
<td>
{{#if is_status_ok }}
<span class="text-success">

8
tests/test_report.py
View File

@ -22,12 +22,12 @@ from shaker.engine import sla
class TestReport(testtools.TestCase):
def test_verify_sla(self):
records = [{'type': 'agent', 'test': 'iperf_tcp',
records = {0: {'id': 0, 'type': 'agent', 'test': 'iperf_tcp',
'stats': {'bandwidth': {'mean': 700, 'min': 400}}},
{'type': 'agent', 'test': 'iperf_udp',
1: {'id': 1, 'type': 'agent', 'test': 'iperf_udp',
'stats': {'bandwidth': {'mean': 1000, 'min': 800}}},
{'type': 'agent', 'test': 'iperf_tcp',
'stats': {'bandwidth': {'mean': 850, 'min': 600}}}]
2: {'id': 2, 'type': 'agent', 'test': 'iperf_tcp',
'stats': {'bandwidth': {'mean': 850, 'min': 600}}}}
tests = {
'iperf_tcp': {

6
tests/test_sla.py
View File

@ -56,6 +56,12 @@ class TestSla(testtools.TestCase):
sla_records)
def test_dump_ast_node(self):
self.assertEqual('(stderr == "")', sla.dump_ast_node(
ast.parse('stderr == ""', mode='eval')))
self.assertEqual('(stderr & ".*")', sla.dump_ast_node(
ast.parse('stderr & ".*"', mode='eval')))
self.assertEqual('(stats.bandwidth.mean > 900)', sla.dump_ast_node(
ast.parse('stats.bandwidth.mean > 900', mode='eval')))