diff --git a/scenarios/misc/static_agent.yaml b/scenarios/misc/static_agent.yaml index d9182f2..3c8c4c2 100644 --- a/scenarios/misc/static_agent.yaml +++ b/scenarios/misc/static_agent.yaml @@ -1,3 +1,5 @@ +title: Static Agents + description: This scenario runs tests on pre-deployed static agents @@ -13,10 +15,13 @@ execution: title: List all files class: shell program: ls -al + sla: + - "[type == 'agent'] >> (stderr == '')" - title: Run sample script class: shell script: | #!/bin/bash - echo "hello" - echo "world" + echo "hello world" + sla: + - "[type == 'agent'] >> (stdout & '.*hello world.*')" diff --git a/scenarios/networking/l2.yaml b/scenarios/networking/l2.yaml index 3a5e48b..d7dabda 100644 --- a/scenarios/networking/l2.yaml +++ b/scenarios/networking/l2.yaml @@ -1,3 +1,5 @@ +title: L2 + description: This scenario launches pairs of VMs in the same private network. Every VM is hosted on a separate compute node. diff --git a/shaker/engine/report.py b/shaker/engine/report.py index 7a8ee53..cb4c4b6 100644 --- a/shaker/engine/report.py +++ b/shaker/engine/report.py @@ -33,13 +33,12 @@ LOG = logging.getLogger(__name__) def calculate_stats(records, tests): - aggregates = [] # scenario -> test -> concurrency -> [record] rec_map = collections.defaultdict( functools.partial(collections.defaultdict, functools.partial(collections.defaultdict, list))) - for record in records: + for record in records.values(): aggregator = aggregators.get_aggregator(tests[record['test']]) aggregator.record_summary(record) @@ -54,24 +53,26 @@ def calculate_stats(records, tests): for concurrency, per_concurrency in per_test.items(): summary = aggregator.concurrency_summary(per_concurrency) if summary: - summary.update(dict(scenario=scenario, test=test, + record_id = utils.make_record_id() + summary.update(dict(id=record_id, + scenario=scenario, test=test, concurrency=concurrency, type='concurrency')) - aggregates.append(summary) + records[record_id] = summary concurrency_aggregates.append(summary) per_test_summary = aggregator.test_summary(concurrency_aggregates) if per_test_summary: - per_test_summary.update(dict(scenario=scenario, test=test, + record_id = utils.make_record_id() + per_test_summary.update(dict(id=record_id, + scenario=scenario, test=test, type='test')) - aggregates.append(per_test_summary) - - return aggregates + records[record_id] = per_test_summary def verify_sla(records, tests): record_map = collections.defaultdict(list) # test -> [record] - for r in records: + for r in records.values(): if 'sla' in tests[r['test']]: record_map[r['test']].append(r) @@ -83,6 +84,11 @@ def verify_sla(records, tests): return sla_records +def output_sla(sla_records): + return [dict(record=item.record['id'], state=item.state, + expression=item.expression) for item in sla_records] + + def _get_location(record): return '.'.join([str(record.get(s)) for s in ['scenario', 'test', 'concurrency', @@ -122,9 +128,10 @@ def generate_report(data, report_template, report_filename, subunit_filename): LOG.debug('Generating report, template: %s, output: %s', report_template, report_filename or '') - data['records'] += calculate_stats(data['records'], data['tests']) + calculate_stats(data['records'], data['tests']) sla_records = verify_sla(data['records'], data['tests']) + data['sla'] = output_sla(sla_records) if subunit_filename: save_to_subunit(sla_records, subunit_filename) diff --git a/shaker/engine/server.py b/shaker/engine/server.py index c91c6ee..ddd1d10 100644 --- a/shaker/engine/server.py +++ b/shaker/engine/server.py @@ -73,7 +73,7 @@ def _pick_agents(agents, size): def execute(quorum, execution, agents): - records = [] + records = {} for test in execution['tests']: LOG.debug('Running test %s on all agents', test) @@ -85,16 +85,18 @@ def execute(quorum, execution, agents): execution_result = quorum.execute(executors) - for agent_id, data in execution_result.items(): - data.update(dict( - agent_id=agent_id, + for agent_id, record in execution_result.items(): + record_id = utils.make_record_id() + record.update(dict( + id=record_id, + agent=agent_id, node=agents[agent_id].get('node'), concurrency=len(selected_agents), test=test_title, executor=test.get('class'), type='agent', )) - records.append(data) + records[record_id] = record LOG.info('Execution is done') return records @@ -142,7 +144,8 @@ def play_scenario(scenario): quorum.join(set(agents.keys())) execution_result = execute(quorum, scenario['execution'], agents) - for record in execution_result: + # extend every record with reference to scenario + for record in execution_result.values(): record['scenario'] = (scenario.get('title') or scenario.get('file_name')) output['records'] = execution_result @@ -167,7 +170,7 @@ def main(): config.REPORT_OPTS ) - output = dict(records=[], agents={}, scenarios={}, tests={}) + output = dict(records={}, agents={}, scenarios={}, tests={}) for scenario_file_name in [cfg.CONF.scenario]: scenario = utils.read_yaml_file(scenario_file_name) @@ -177,7 +180,7 @@ def main(): play_output = play_scenario(scenario) output['scenarios'][scenario['title']] = play_output['scenario'] - output['records'] += play_output['records'] + output['records'].update(play_output['records']) output['agents'].update(play_output['agents']) output['tests'].update(play_output['tests']) diff --git a/shaker/engine/sla.py b/shaker/engine/sla.py index 7d8bfc8..fc65810 100644 --- a/shaker/engine/sla.py +++ b/shaker/engine/sla.py @@ -112,7 +112,7 @@ def dump_ast_node(node): elif isinstance(node, ast.Name): return node.id elif isinstance(node, ast.Str): - return '\'node.s\'' + return '"%s"' % node.s elif isinstance(node, ast.BinOp): return '%s %s %s' % (_format(node.left), _operators[type(node.op)], _format(node.right)) diff --git a/shaker/engine/utils.py b/shaker/engine/utils.py index 598f4fb..58a20a4 100644 --- a/shaker/engine/utils.py +++ b/shaker/engine/utils.py @@ -16,6 +16,7 @@ import logging as std_logging import os import random +import uuid from oslo_config import cfg from oslo_log import log as logging @@ -138,6 +139,10 @@ def random_string(length=6): return ''.join(random.sample('adefikmoprstuz', length)) +def make_record_id(): + return str(uuid.uuid4()) + + def copy_dict_kv(source): return dict((k, v) for k, v in source.items()) diff --git a/shaker/resources/report_template.html b/shaker/resources/report_template.html index 7e4e823..30860ff 100644 --- a/shaker/resources/report_template.html +++ b/shaker/resources/report_template.html @@ -45,13 +45,13 @@ $(function () { - const fields = ["scenario", "test", "concurrency", "node", "agent_id"]; + const fields = ["scenario", "test", "concurrency", "node", "agent"]; const state = { scenario: "", test: "", concurrency: "", node: "", - agent_id: "" + agent: "" }; const tablePageSize = 10; @@ -104,7 +104,7 @@ var values_set = {}; $.each(records, function (j, record) { - if (record["type"] == "raw") { + if (record["type"] == "agent" && record[field]) { values_set["" + record[field]] = true; } }); @@ -123,9 +123,8 @@ function filterRecords(records, type) { var res = []; - for (var i = 0; i < records.length; i++) { - var record = records[i]; - if (type != "" && record.type == type) { + $.each(records, function(_i, record){ + if (type == "" || record.type == type) { var flag = true; $.each(state, function (key, value) { if (value != "") { @@ -136,7 +135,7 @@ res.push(record); } } - } + }); return res; } @@ -258,7 +257,7 @@ function showAgentsTable(contentArea, filteredRecords) { var agent_ids = {}; $.each(filteredRecords, function(_i, record){ - agent_ids[record.agent_id] = true; + agent_ids[record.agent] = true; }); var agents = getAgentsByIds(agent_ids); @@ -324,7 +323,8 @@ /* Update cycle */ function update() { - var filteredRecords = filterRecords(report.records, "raw"); + var filteredRecords = filterRecords(report.records, "agent"); + console.log(filteredRecords); var slaRecords = []; $.each(filterRecords(report.records), function(_i, record) { if (record["sla_info"]) { @@ -342,21 +342,21 @@ if (state["scenario"] != "" && state["test"] != "" && state["concurrency"] != "" && state["node"] != "" && - state["agent_id"] != "") { + state["agent"] != "") { showExecutorRecord(contentArea, filteredRecords[0]); } if (state["scenario"] != "" && state["test"] != "" && state["concurrency"] != "" && state["node"] == "" && - state["agent_id"] == "") { - var agg_recs = filterRecords(report.records, "agg_concurrency"); + state["agent"] == "") { + var agg_recs = filterRecords(report.records, "concurrency"); if (agg_recs.length > 0) { showConcurrencySummary(contentArea, agg_recs[0]); } } if (state["scenario"] != "" && state["test"] != "" && state["concurrency"] == "" && state["node"] == "" && - state["agent_id"] == "") { - var test_recs = filterRecords(report.records, "agg_test"); + state["agent"] == "") { + var test_recs = filterRecords(report.records, "test"); if (test_recs.length > 0) { showTestSummary(contentArea, test_recs[0]); } @@ -367,7 +367,7 @@ if (state["scenario"] != "") { showScenarioSource(contentArea, report.scenarios[state["scenario"]]); } - if (state["agent_id"] == "") { + if (state["agent"] == "") { showAgentsTable(contentArea, filteredRecords); } } @@ -443,8 +443,8 @@
- - + +
@@ -538,7 +538,7 @@ {{ test }} {{ concurrency }} {{ node }} - {{ agent_id }} + {{ agent }} {{#if is_status_ok }} diff --git a/tests/test_report.py b/tests/test_report.py index 86f70b4..4a2c306 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -22,12 +22,12 @@ from shaker.engine import sla class TestReport(testtools.TestCase): def test_verify_sla(self): - records = [{'type': 'agent', 'test': 'iperf_tcp', - 'stats': {'bandwidth': {'mean': 700, 'min': 400}}}, - {'type': 'agent', 'test': 'iperf_udp', - 'stats': {'bandwidth': {'mean': 1000, 'min': 800}}}, - {'type': 'agent', 'test': 'iperf_tcp', - 'stats': {'bandwidth': {'mean': 850, 'min': 600}}}] + records = {0: {'id': 0, 'type': 'agent', 'test': 'iperf_tcp', + 'stats': {'bandwidth': {'mean': 700, 'min': 400}}}, + 1: {'id': 1, 'type': 'agent', 'test': 'iperf_udp', + 'stats': {'bandwidth': {'mean': 1000, 'min': 800}}}, + 2: {'id': 2, 'type': 'agent', 'test': 'iperf_tcp', + 'stats': {'bandwidth': {'mean': 850, 'min': 600}}}} tests = { 'iperf_tcp': { diff --git a/tests/test_sla.py b/tests/test_sla.py index ff6cb7a..1d02e61 100644 --- a/tests/test_sla.py +++ b/tests/test_sla.py @@ -56,6 +56,12 @@ class TestSla(testtools.TestCase): sla_records) def test_dump_ast_node(self): + self.assertEqual('(stderr == "")', sla.dump_ast_node( + ast.parse('stderr == ""', mode='eval'))) + + self.assertEqual('(stderr & ".*")', sla.dump_ast_node( + ast.parse('stderr & ".*"', mode='eval'))) + self.assertEqual('(stats.bandwidth.mean > 900)', sla.dump_ast_node( ast.parse('stats.bandwidth.mean > 900', mode='eval')))