Installed builtin reordering everywhere
Installed the builtin reordering code into NonrecursiveRuleTheory and MaterializedViewTheory. Added tests for MaterializedViewTheory. Also tweaked the builtin interface and simplified how other modules utilize that interface. Closes-bug: 1350074 Closes-bug: 1350077 Change-Id: I30ec45917686225c43204eafa9364a64ecae3dd7
This commit is contained in:
Tim Hinrichs
parent
12d3dd2376
commit
878e5a74f4
|
|
@ -119,7 +119,8 @@ class DatetimeBuiltins(object):
|
|||
return cls.to_datetime(x) == cls.to_datetime(y)
|
||||
|
||||
|
||||
start_builtin_map = {
|
||||
# the registry for builtins
|
||||
_builtin_map = {
|
||||
'comparison': [
|
||||
{'func': 'lt(x,y)', 'num_inputs': 2, 'code': lambda x, y: x < y},
|
||||
{'func': 'lteq(x,y)', 'num_inputs': 2, 'code': lambda x, y: x <= y},
|
||||
|
|
@ -182,6 +183,7 @@ class CongressBuiltinPred(object):
|
|||
self.predargs = arglist
|
||||
self.num_inputs = num_inputs
|
||||
self.code = code
|
||||
self.num_outputs = len(arglist) - num_inputs
|
||||
|
||||
def __str__(self):
|
||||
predall = str(self.predname) + " " + str(self.predargs)\
|
||||
|
|
@ -195,8 +197,8 @@ class CongressBuiltinPred(object):
|
|||
except Exception:
|
||||
print "Unexpected error in parsing predicate string"
|
||||
|
||||
def pred_to_string(self):
|
||||
return self.predname + '(' + str(self.predargs) + ')'
|
||||
def __str__(self):
|
||||
return self.predname + '(' + ",".join(self.predargs) + ')'
|
||||
|
||||
|
||||
class CongressBuiltinCategoryMap(object):
|
||||
|
|
@ -235,7 +237,7 @@ class CongressBuiltinCategoryMap(object):
|
|||
# print 'category exists'
|
||||
for predtriple in value:
|
||||
pred = self.dict_predtriple_to_pred(predtriple)
|
||||
if not self.check_if_builtin(pred):
|
||||
if not self.builtin_is_registered(pred):
|
||||
self.categorydict[key].append(pred)
|
||||
self.sync_with_predlist(pred.predname, pred, key, 'add')
|
||||
|
||||
|
|
@ -267,7 +269,7 @@ class CongressBuiltinCategoryMap(object):
|
|||
self.categorydict[category].remove(pred)
|
||||
self.sync_with_predlist(name, pred, category, 'del')
|
||||
|
||||
def get_builtin_category_name(self, predname, predinputs):
|
||||
def get_category_name(self, predname, predinputs):
|
||||
if predname in self.preddict:
|
||||
if self.preddict[predname][0].num_inputs == predinputs:
|
||||
return self.preddict[predname][1]
|
||||
|
|
@ -309,54 +311,52 @@ class CongressBuiltinCategoryMap(object):
|
|||
else:
|
||||
assert("Category does not exist")
|
||||
|
||||
def check_if_builtin(self, predtotest):
|
||||
def builtin_is_registered(self, predtotest):
|
||||
"""Given a CongressBuiltinPred, check if it has been registered."""
|
||||
pname = predtotest.predname
|
||||
if pname in self.preddict:
|
||||
if self.preddict[pname][0].num_inputs == predtotest.num_inputs:
|
||||
return True
|
||||
return False
|
||||
|
||||
def check_if_builtin_by_name(self, predname, arity):
|
||||
def is_builtin(self, predname, arity):
|
||||
"""Given a name and arity, check if it is a builtin."""
|
||||
# print "check_if_builtin_by_name {} {}".format(predname, arity)
|
||||
if predname in self.preddict:
|
||||
if len(self.preddict[predname][0].predargs) == arity:
|
||||
return True
|
||||
return False
|
||||
|
||||
def return_builtin_pred(self, predname):
|
||||
def builtin(self, predname):
|
||||
"""Return a CongressBuiltinPred with name PREDNAME or None."""
|
||||
if predname in self.preddict:
|
||||
return self.preddict[predname][0]
|
||||
return None
|
||||
|
||||
def builtin_num_outputs(self, predname):
|
||||
if predname in self.preddict:
|
||||
pred = self.preddict[predname][0]
|
||||
return len(pred.predargs) - pred.num_inputs
|
||||
return 0
|
||||
|
||||
def list_available_builtins(self):
|
||||
"""Print out the list of builtins, by category."""
|
||||
for key, value in self.categorydict.items():
|
||||
predlist = self.categorydict[key]
|
||||
for pred in predlist:
|
||||
print pred
|
||||
print pred.pred_to_string()
|
||||
print str(pred)
|
||||
|
||||
def eval_builtin(self, code, arglist):
|
||||
return code(*arglist)
|
||||
|
||||
# a Singleton that serves as the entry point for builtin functionality
|
||||
builtin_registry = CongressBuiltinCategoryMap(_builtin_map)
|
||||
|
||||
|
||||
def main():
|
||||
cbcmap = CongressBuiltinCategoryMap(start_builtin_map)
|
||||
cbcmap = CongressBuiltinCategoryMap(_builtin_map)
|
||||
cbcmap.list_available_builtins()
|
||||
predl = cbcmap.return_builtin_pred('lt')
|
||||
predl = cbcmap.builtin('lt')
|
||||
print predl
|
||||
print 'printing pred'
|
||||
predl.string_to_pred('ltc(x,y)')
|
||||
cbcmap.list_available_builtins()
|
||||
cbcmap.delete_builtin('arithmetic', 'max', 2)
|
||||
cbcmap.list_available_builtins()
|
||||
predl = cbcmap.return_builtin_pred('plus')
|
||||
result = cbcmap.eval_builtin(predl.code, [1, 2])
|
||||
predl = cbcmap.builtin('plus')
|
||||
result = predl.code(1, 2)
|
||||
print result
|
||||
print cbcmap
|
||||
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@
|
|||
import unittest
|
||||
|
||||
from congress.openstack.common import log as logging
|
||||
from congress.policy.builtin.congressbuiltin \
|
||||
import CongressBuiltinCategoryMap as builtins
|
||||
from congress.policy.builtin.congressbuiltin import _builtin_map
|
||||
from congress.policy.builtin.congressbuiltin import CongressBuiltinCategoryMap
|
||||
from congress.policy.builtin.congressbuiltin import CongressBuiltinPred
|
||||
from congress.policy.builtin.congressbuiltin import start_builtin_map
|
||||
|
||||
from congress.policy import compile
|
||||
from congress.policy import runtime
|
||||
from congress.tests import helper
|
||||
|
|
@ -40,15 +40,12 @@ append_builtin = {'arithmetic': [{'func': 'div(x,y)',
|
|||
'num_inputs': 2,
|
||||
'code': 'lambda x,y: x / y'}]}
|
||||
|
||||
NREC_THEORY = 'non-recursive theory'
|
||||
DB_THEORY = 'database'
|
||||
|
||||
|
||||
class TestBuiltins(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.cbcmap = builtins(start_builtin_map)
|
||||
self.predl = self.cbcmap.return_builtin_pred('lt')
|
||||
self.cbcmap = CongressBuiltinCategoryMap(_builtin_map)
|
||||
self.predl = self.cbcmap.builtin('lt')
|
||||
|
||||
def test_add_and_delete_map(self):
|
||||
cbcmap_before = self.cbcmap
|
||||
|
|
@ -58,10 +55,10 @@ class TestBuiltins(unittest.TestCase):
|
|||
|
||||
def test_add_map_only(self):
|
||||
self.cbcmap.add_map(append_builtin)
|
||||
predl = self.cbcmap.return_builtin_pred('div')
|
||||
predl = self.cbcmap.builtin('div')
|
||||
self.assertNotEqual(predl, None)
|
||||
self.cbcmap.add_map(addmap)
|
||||
predl = self.cbcmap.return_builtin_pred('max')
|
||||
predl = self.cbcmap.builtin('max')
|
||||
self.assertNotEqual(predl, None)
|
||||
|
||||
def test_add_and_delete_builtin(self):
|
||||
|
|
@ -71,27 +68,27 @@ class TestBuiltins(unittest.TestCase):
|
|||
self.assertTrue(self.cbcmap.mapequal(cbcmap_before))
|
||||
|
||||
def test_string_pred_string(self):
|
||||
predstring = self.predl.pred_to_string()
|
||||
predstring = str(self.predl)
|
||||
self.assertNotEqual(predstring, 'ltc(x,y')
|
||||
|
||||
def test_add_and_delete_to_category(self):
|
||||
cbcmap_before = self.cbcmap
|
||||
arglist = ['x', 'y', 'z']
|
||||
pred = CongressBuiltinPred('testfunc', arglist, 1, 'lambda x: not x')
|
||||
pred = CongressBuiltinPred('testfunc', arglist, 1, lambda x: not x)
|
||||
self.cbcmap.insert_to_category('arithmetic', pred)
|
||||
self.cbcmap.delete_from_category('arithmetic', pred)
|
||||
self.assertTrue(self.cbcmap.mapequal(cbcmap_before))
|
||||
|
||||
def test_all_checks(self):
|
||||
predtotest = self.cbcmap.return_builtin_pred('lt')
|
||||
self.assertTrue(self.cbcmap.check_if_builtin(predtotest))
|
||||
predtotest = self.cbcmap.builtin('lt')
|
||||
self.assertTrue(self.cbcmap.builtin_is_registered(predtotest))
|
||||
|
||||
def test_eval_builtin(self):
|
||||
predl = self.cbcmap.return_builtin_pred('plus')
|
||||
result = self.cbcmap.eval_builtin(predl.code, [1, 2])
|
||||
predl = self.cbcmap.builtin('plus')
|
||||
result = predl.code(1, 2)
|
||||
self.assertEqual(result, 3)
|
||||
predl = self.cbcmap.return_builtin_pred('gt')
|
||||
result = self.cbcmap.eval_builtin(predl.code, [1, 2])
|
||||
predl = self.cbcmap.builtin('gt')
|
||||
result = predl.code(1, 2)
|
||||
self.assertEqual(result, False)
|
||||
|
||||
|
||||
|
|
@ -237,7 +234,12 @@ class TestReorder(unittest.TestCase):
|
|||
'Unsafety propagates')
|
||||
|
||||
|
||||
class TestNonrecursive(unittest.TestCase):
|
||||
NREC_THEORY = 'non-recursive theory test'
|
||||
MAT_THEORY = 'materialized view theory test'
|
||||
|
||||
|
||||
class TestTheories(unittest.TestCase):
|
||||
|
||||
def prep_runtime(self, code=None, msg=None, target=None):
|
||||
# compile source
|
||||
if msg is not None:
|
||||
|
|
@ -247,8 +249,10 @@ class TestNonrecursive(unittest.TestCase):
|
|||
if target is None:
|
||||
target = NREC_THEORY
|
||||
run = runtime.Runtime()
|
||||
run.theory[NREC_THEORY] = runtime.NonrecursiveRuleTheory()
|
||||
run.theory[DB_THEORY] = runtime.Database(name="Database", abbr="DB")
|
||||
run.theory[NREC_THEORY] = runtime.NonrecursiveRuleTheory(
|
||||
name="Nonrecursive", abbr="NRT")
|
||||
run.theory[MAT_THEORY] = runtime.MaterializedViewTheory(
|
||||
name="Materialized", abbr="MAT")
|
||||
run.debug_mode()
|
||||
run.insert(code, target=target)
|
||||
return run
|
||||
|
|
@ -257,9 +261,11 @@ class TestNonrecursive(unittest.TestCase):
|
|||
self.assertTrue(helper.datalog_equal(
|
||||
actual_string, correct_string, msg))
|
||||
|
||||
def test_builtins(self):
|
||||
def test_materialized_builtins(self):
|
||||
self.test_builtins(MAT_THEORY)
|
||||
|
||||
def test_builtins(self, th=NREC_THEORY):
|
||||
"""Test the mechanism that implements builtins."""
|
||||
th = NREC_THEORY
|
||||
run = self.prep_runtime()
|
||||
run.insert('p(x) :- q(x,y), plus(x,y,z), r(z)'
|
||||
'q(1,2)'
|
||||
|
|
@ -316,10 +322,28 @@ class TestNonrecursive(unittest.TestCase):
|
|||
self.check_equal(run.select('p(x)', target=th),
|
||||
'p(4)', "Bound output")
|
||||
|
||||
def test_builtins_content(self):
|
||||
run = self.prep_runtime()
|
||||
run.insert('p(x, z) :- plus(x,y,z), q(x), r(y)'
|
||||
'q(4)'
|
||||
'r(5)', target=th)
|
||||
self.check_equal(run.select('p(x, y)', target=th),
|
||||
'p(4, 9)',
|
||||
"Reordering")
|
||||
|
||||
run = self.prep_runtime()
|
||||
run.insert('p(x, z) :- plus(x,y,z), q(x), q(y)'
|
||||
'q(4)'
|
||||
'q(5)', target=th)
|
||||
self.check_equal(run.select('p(x, y)', target=th),
|
||||
'p(4, 9) p(4, 8) p(5, 9) p(5, 10)',
|
||||
"Reordering with self joins")
|
||||
|
||||
def test_materialized_builtins_content(self):
|
||||
self.test_builtins_content(MAT_THEORY)
|
||||
|
||||
def test_builtins_content(self, th=NREC_THEORY):
|
||||
"""Test the content of the builtins, not the mechanism"""
|
||||
def check_true(code, msg):
|
||||
th = NREC_THEORY
|
||||
run = self.prep_runtime('')
|
||||
run.insert(code, target=th)
|
||||
self.check_equal(
|
||||
|
|
|
|||
|
|
@ -24,8 +24,7 @@ import CongressLexer
|
|||
import CongressParser
|
||||
import utility
|
||||
|
||||
from builtin.congressbuiltin import CongressBuiltinCategoryMap as cbcmap
|
||||
from builtin.congressbuiltin import start_builtin_map as initbuiltin
|
||||
from builtin.congressbuiltin import builtin_registry
|
||||
|
||||
|
||||
class CongressException (Exception):
|
||||
|
|
@ -402,6 +401,8 @@ class Literal (object):
|
|||
return self.table.endswith('+') or self.table.endswith('-')
|
||||
|
||||
def tablename(self):
|
||||
# implemented simply so that we can call tablename() on either
|
||||
# rules or literals
|
||||
return self.table
|
||||
|
||||
|
||||
|
|
@ -715,7 +716,8 @@ def reorder_for_safety(rule):
|
|||
is evaluated. Reordering is stable, meaning that if the rule is
|
||||
properly ordered, no changes are made.
|
||||
"""
|
||||
cbcmapinst = cbcmap(initbuiltin)
|
||||
if not is_rule(rule):
|
||||
return rule
|
||||
safe_vars = set()
|
||||
unsafe_literals = []
|
||||
unsafe_variables = {} # dictionary from literal to its unsafe vars
|
||||
|
|
@ -741,9 +743,8 @@ def reorder_for_safety(rule):
|
|||
target_vars = None
|
||||
if lit.is_negated():
|
||||
target_vars = lit.variable_names()
|
||||
elif cbcmapinst.check_if_builtin_by_name(
|
||||
lit.table, len(lit.arguments)):
|
||||
builtin = cbcmapinst.return_builtin_pred(lit.table)
|
||||
elif builtin_registry.is_builtin(lit.table, len(lit.arguments)):
|
||||
builtin = builtin_registry.builtin(lit.table)
|
||||
target_vars = lit.arguments[0:builtin.num_inputs]
|
||||
target_vars = set([x.name for x in target_vars if x.is_variable()])
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ import cStringIO
|
|||
import os
|
||||
from unify import bi_unify_lists
|
||||
|
||||
from builtin.congressbuiltin import CongressBuiltinCategoryMap
|
||||
from builtin.congressbuiltin import start_builtin_map
|
||||
from builtin.congressbuiltin import builtin_registry
|
||||
|
||||
# FIXME there is a circular import here because compile.py imports runtime.py
|
||||
import compile
|
||||
|
|
@ -261,7 +260,6 @@ class Theory(object):
|
|||
self.trace_prefix = self.abbr[0:maxlength]
|
||||
else:
|
||||
self.trace_prefix = self.abbr + " " * (maxlength - len(self.abbr))
|
||||
self.cbcmap = CongressBuiltinCategoryMap(start_builtin_map)
|
||||
|
||||
def set_tracer(self, tracer):
|
||||
self.tracer = tracer
|
||||
|
|
@ -599,11 +597,9 @@ class TopDownTheory(Theory):
|
|||
elif lit.tablename() == 'false':
|
||||
self.print_fail(lit, context.binding, context.depth)
|
||||
return False
|
||||
elif self.cbcmap.check_if_builtin_by_name(lit.tablename(),
|
||||
len(lit.arguments)):
|
||||
elif builtin_registry.is_builtin(lit.table, len(lit.arguments)):
|
||||
self.print_call(lit, context.binding, context.depth)
|
||||
cbc = self.cbcmap.return_builtin_pred(lit.tablename())
|
||||
builtin_code = cbc.code
|
||||
builtin = builtin_registry.builtin(lit.table)
|
||||
# copy arguments into variables
|
||||
# PLUGGED is an instance of compile.Literal
|
||||
plugged = lit.plug(context.binding)
|
||||
|
|
@ -611,17 +607,16 @@ class TopDownTheory(Theory):
|
|||
# PLUGGED.arguments is a list of compile.Term
|
||||
# create args for function
|
||||
args = []
|
||||
for i in xrange(0, cbc.num_inputs):
|
||||
for i in xrange(0, builtin.num_inputs):
|
||||
assert plugged.arguments[i].is_object(), \
|
||||
("Builtins must be evaluated only after their "
|
||||
"inputs are ground: {} with num-inputs {}".format(
|
||||
str(plugged), cbc.num_inputs))
|
||||
str(plugged), builtin.num_inputs))
|
||||
args.append(plugged.arguments[i].name)
|
||||
# evaluate builtin: must return number, string, or iterable
|
||||
# of numbers/strings
|
||||
# print "args: " + str(args)
|
||||
try:
|
||||
result = self.cbcmap.eval_builtin(builtin_code, args)
|
||||
result = builtin.code(*args)
|
||||
except Exception as e:
|
||||
errmsg = "Error in builtin: " + str(e)
|
||||
self.print_note(lit, context.binding, context.depth, errmsg)
|
||||
|
|
@ -632,7 +627,7 @@ class TopDownTheory(Theory):
|
|||
# "Result: " + str(result))
|
||||
success = None
|
||||
undo = []
|
||||
if self.cbcmap.builtin_num_outputs(lit.table) > 0:
|
||||
if builtin.num_outputs > 0:
|
||||
# with return values, local success means we can bind
|
||||
# the results to the return value arguments
|
||||
if isinstance(result, (int, long, float, basestring)):
|
||||
|
|
@ -643,7 +638,7 @@ class TopDownTheory(Theory):
|
|||
unifier = self.new_bi_unifier()
|
||||
undo = bi_unify_lists(result,
|
||||
unifier,
|
||||
lit.arguments[cbc.num_inputs:],
|
||||
lit.arguments[builtin.num_inputs:],
|
||||
context.binding)
|
||||
# print "unifier: " + str(undo)
|
||||
success = undo is not None
|
||||
|
|
@ -1239,11 +1234,12 @@ class NonrecursiveRuleTheory(TopDownTheory):
|
|||
changes = []
|
||||
self.log(None, "Update " + iterstr(events))
|
||||
for event in events:
|
||||
formula = compile.reorder_for_safety(event.formula)
|
||||
if event.insert:
|
||||
if self.insert_actual(event.formula):
|
||||
if self.insert_actual(formula):
|
||||
changes.append(event)
|
||||
else:
|
||||
if self.delete_actual(event.formula):
|
||||
if self.delete_actual(formula):
|
||||
changes.append(event)
|
||||
return changes
|
||||
|
||||
|
|
@ -1423,7 +1419,6 @@ class DeltaRuleTheory (Theory):
|
|||
return False
|
||||
self.log(rule.tablename(), "Insert 2: {}".format(str(rule)))
|
||||
for delta in self.compute_delta_rules([rule]):
|
||||
self.reorder(delta)
|
||||
self.insert_delta(delta)
|
||||
self.originals.add(rule)
|
||||
return True
|
||||
|
|
@ -1588,23 +1583,16 @@ class DeltaRuleTheory (Theory):
|
|||
for rule in formulas:
|
||||
if rule.is_atom():
|
||||
continue
|
||||
rule = compile.reorder_for_safety(rule)
|
||||
for literal in rule.body:
|
||||
if builtin_registry.is_builtin(
|
||||
literal.table, len(literal.arguments)):
|
||||
continue
|
||||
newbody = [lit for lit in rule.body if lit is not literal]
|
||||
delta_rules.append(
|
||||
DeltaRule(literal, rule.head, newbody, rule))
|
||||
return delta_rules
|
||||
|
||||
@classmethod
|
||||
def reorder(cls, delta):
|
||||
"""Given a delta rule DELTA, re-order its body for efficient
|
||||
and correct computation.
|
||||
"""
|
||||
# ensure negatives come after positives
|
||||
positives = [lit for lit in delta.body if not lit.is_negated()]
|
||||
negatives = [lit for lit in delta.body if lit.is_negated()]
|
||||
positives.extend(negatives)
|
||||
delta.body = positives
|
||||
|
||||
|
||||
class MaterializedViewTheory(TopDownTheory):
|
||||
"""A theory that stores the table contents of views explicitly.
|
||||
|
|
@ -1778,7 +1766,6 @@ class MaterializedViewTheory(TopDownTheory):
|
|||
# need to eliminate self-joins here so that we fill all
|
||||
# the tables introduced by self-join elimination.
|
||||
for rule in DeltaRuleTheory.eliminate_self_joins([formula]):
|
||||
DeltaRuleTheory.reorder(rule)
|
||||
new_event = Event(formula=rule, insert=event.insert,
|
||||
target=event.target)
|
||||
self.enqueue(new_event)
|
||||
|
|
|
|||
|
|
@ -444,16 +444,6 @@ class TestRuntime(base.TestCase):
|
|||
run, 'q(1,2) r(2,3) r(2,4) u(3,5) u(4,6) s(1,3) s(1,4)',
|
||||
'Insert into non-unary with different propagation')
|
||||
|
||||
# Negation ordering
|
||||
code = ("p(x) :- not q(x), r(x)")
|
||||
run = self.prep_runtime(code, "Negation ordering")
|
||||
run.insert('r(1)', MAT_THEORY)
|
||||
run.insert('r(2)', MAT_THEORY)
|
||||
run.insert('q(1)', MAT_THEORY)
|
||||
self.check_class(
|
||||
run, 'r(1) r(2) q(1) p(2)',
|
||||
'Reordering negation')
|
||||
|
||||
def test_select(self):
|
||||
"""Materialized Theory: test the SELECT event handler."""
|
||||
code = ("p(x, y) :- q(x), r(y)")
|
||||
|
|
|
|||
Loading…
Reference in New Issue