performa/shaker
Code Issues Proposed changes

Make SLA evaluation tolerant to errors

Browse Source 
Change-Id: I3638ea4df1d6d68f2bbf5f9f813ba24c81e39b68
This commit is contained in:
Ilya Shakhat 2015-12-23 13:28:31 +03:00
parent 0eea99ad93
commit 44903d56e0
4 changed files with 68 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
shaker/engine/report.py
View File

@ -92,8 +92,7 @@ def log_sla(sla_records):
LOG.info('*' * 80)
for item in sla_records:
test_id = _get_location(item.record) + ':' + item.expression
LOG.info('%-73s %6s' % (test_id,
'[%s]' % ('OK' if item.state else 'FAIL')))
LOG.info('%-73s %7s' % (test_id, '[%s]' % item.state))
LOG.info('*' * 80)
@ -112,6 +111,8 @@ def _get_location(record):
def save_to_subunit(sla_records, subunit_filename):
LOG.debug('Writing subunit stream to: %s', subunit_filename)
fd = None
state2subunit = {sla.STATE_TRUE: 'success',
sla.STATE_FALSE: 'fail'}
try:
fd = open(subunit_filename, 'w')
output = subunit_v2.StreamResultToBytes(fd)
@ -120,14 +121,14 @@ def save_to_subunit(sla_records, subunit_filename):
output.startTestRun()
test_id = _get_location(item.record) + ':' + item.expression
if not item.state:
if item.state != sla.STATE_TRUE:
output.status(test_id=test_id, file_name='results',
mime_type='text/plain; charset="utf8"', eof=True,
file_bytes=yaml.safe_dump(
item.record, default_flow_style=False))
output.status(test_id=test_id,
test_status='success' if item.state else 'fail')
test_status=state2subunit.get(item.state, 'skip'))
output.stopTestRun()
LOG.info('Subunit stream saved to: %s', subunit_filename)

40
shaker/engine/sla.py
View File

@ -19,7 +19,13 @@ import operator as op
import re
class SLAException(Exception):
pass
SLAItem = collections.namedtuple('SLAItem', ['record', 'state', 'expression'])
STATE_TRUE = 'OK'
STATE_FALSE = 'FAIL'
# supported operators
operators = {ast.Add: op.add, ast.Sub: op.sub, ast.Mult: op.mul,
@ -51,19 +57,15 @@ def _eval(node, ctx):
if isinstance(node, ast.Num):
return node.n
elif isinstance(node, ast.Name):
return ctx.get(node.id)
r = ctx.get(node.id)
if r is None:
raise SLAException('Value "%s" is not found' % dump_ast_node(node))
return r
elif isinstance(node, ast.Str):
return node.s
elif isinstance(node, ast.BinOp):
if isinstance(node.op, ast.RShift):
# left -- array, right -- condition
filtered = _eval(node.left, ctx)
result = []
for record in filtered:
state = _eval(node.right, record)
result.append(SLAItem(record=record, state=state,
expression=dump_ast_node(node.right)))
return result
return _eval_top(ctx, node) # the top expression
elif isinstance(node.op, ast.BitAnd):
s = _eval(node.left, ctx)
return ((s is not None) and
@ -87,7 +89,10 @@ def _eval(node, ctx):
r = operators[type(node.op)](r, _eval(node.values[i], ctx))
return r
elif isinstance(node, ast.Attribute):
return _eval(node.value, ctx).get(node.attr)
r = _eval(node.value, ctx).get(node.attr)
if r is None:
raise SLAException('Value "%s" is not found' % dump_ast_node(node))
return r
elif isinstance(node, ast.List):
records = ctx
filtered = []
@ -100,6 +105,21 @@ def _eval(node, ctx):
raise TypeError(node)
def _eval_top(ctx, node):
result = []
# left -- array, right -- condition
filtered = _eval(node.left, ctx)
for record in filtered:
try:
right = _eval(node.right, record)
state = (STATE_TRUE if right else STATE_FALSE)
except (SLAException, TypeError) as e:
state = str(e)
result.append(SLAItem(record=record, state=state,
expression=dump_ast_node(node.right)))
return result
def dump_ast_node(node):
_operators = {ast.Add: '+', ast.Sub: '-', ast.Mult: '*',
ast.Div: '/', ast.Pow: '**', ast.BitXor: '^',

10
shaker/tests/test_report.py
View File

@ -45,15 +45,15 @@ class TestReport(testtools.TestCase):
sla_records = report.verify_sla(records, tests)
self.assertIn(sla.SLAItem(records[0], False,
self.assertIn(sla.SLAItem(records[0], sla.STATE_FALSE,
'stats.bandwidth.mean > 800'), sla_records)
self.assertIn(sla.SLAItem(records[0], False,
self.assertIn(sla.SLAItem(records[0], sla.STATE_FALSE,
'stats.bandwidth.min > 500'), sla_records)
self.assertIn(sla.SLAItem(records[1], True,
self.assertIn(sla.SLAItem(records[1], sla.STATE_TRUE,
'stats.bandwidth.mean > 900'), sla_records)
self.assertIn(sla.SLAItem(records[2], True,
self.assertIn(sla.SLAItem(records[2], sla.STATE_TRUE,
'stats.bandwidth.mean > 800'), sla_records)
self.assertIn(sla.SLAItem(records[2], True,
self.assertIn(sla.SLAItem(records[2], sla.STATE_TRUE,
'stats.bandwidth.min > 500'), sla_records)

33
shaker/tests/test_sla.py
View File

@ -31,7 +31,13 @@ class TestSla(testtools.TestCase):
self.assertEqual(True, sla.eval_expr('"some text" & "\w+\s+\w+"'))
self.assertEqual(False, sla.eval_expr('"some text" & "\d+"'))
self.assertEqual(False, sla.eval_expr('a & "\d+"', {})) # a == None
self.assertEqual(False, sla.eval_expr('a & "\d+"', {'a': ''}))
def test_eval_non_existent_ref(self):
self.assertRaises(sla.SLAException, sla.eval_expr,
'2 + a.c', {'a': {'b': 40}})
self.assertRaises(sla.SLAException, sla.eval_expr,
'2 + e.b', {'a': {'b': 40}})
def test_eval_sla(self):
records = [{'type': 'agent', 'test': 'iperf_tcp',
@ -45,16 +51,20 @@ class TestSla(testtools.TestCase):
sla_records = sla.eval_expr('[type == "agent"] >> (%s)' % expr,
records)
self.assertEqual([
sla.SLAItem(record=records[0], state=False, expression=expr),
sla.SLAItem(record=records[1], state=True, expression=expr)],
sla.SLAItem(record=records[0], state=sla.STATE_FALSE,
expression=expr),
sla.SLAItem(record=records[1], state=sla.STATE_TRUE,
expression=expr)],
sla_records)
expr = 'stats.bandwidth.mean > 900'
sla_records = sla.eval_expr('[test == "iperf_udp", type == "node"] >> '
'(%s)' % expr, records)
self.assertEqual([
sla.SLAItem(record=records[1], state=True, expression=expr),
sla.SLAItem(record=records[2], state=False, expression=expr)],
sla.SLAItem(record=records[1], state=sla.STATE_TRUE,
expression=expr),
sla.SLAItem(record=records[2], state=sla.STATE_FALSE,
expression=expr)],
sla_records)
def test_dump_ast_node(self):
@ -71,3 +81,16 @@ class TestSla(testtools.TestCase):
'stats.ping.mean < 0.35)')
self.assertEqual(expr,
sla.dump_ast_node(ast.parse(expr, mode='eval')))
def test_eval_sla_undefined_ref(self):
records = [{'type': 'agent', 'test': 'iperf_tcp',
'stats': {'bandwidth': {'mean': 850}}}]
expr = 'stats.nonexistent.mean > 800'
sla_records = sla.eval_expr('[type == "agent"] >> (%s)' % expr,
records)
self.assertEqual([
sla.SLAItem(record=records[0],
state='Value "stats.nonexistent" is not found',
expression=expr)],
sla_records)