Merge "Installed builtin reordering everywhere"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
93b47129f3
|
|
@ -119,7 +119,8 @@ class DatetimeBuiltins(object):
|
|||
return cls.to_datetime(x) == cls.to_datetime(y)
|
||||
|
||||
|
||||
start_builtin_map = {
|
||||
# the registry for builtins
|
||||
_builtin_map = {
|
||||
'comparison': [
|
||||
{'func': 'lt(x,y)', 'num_inputs': 2, 'code': lambda x, y: x < y},
|
||||
{'func': 'lteq(x,y)', 'num_inputs': 2, 'code': lambda x, y: x <= y},
|
||||
|
|
@ -182,6 +183,7 @@ class CongressBuiltinPred(object):
|
|||
self.predargs = arglist
|
||||
self.num_inputs = num_inputs
|
||||
self.code = code
|
||||
self.num_outputs = len(arglist) - num_inputs
|
||||
|
||||
def __str__(self):
|
||||
predall = str(self.predname) + " " + str(self.predargs)\
|
||||
|
|
@ -195,8 +197,8 @@ class CongressBuiltinPred(object):
|
|||
except Exception:
|
||||
print "Unexpected error in parsing predicate string"
|
||||
|
||||
def pred_to_string(self):
|
||||
return self.predname + '(' + str(self.predargs) + ')'
|
||||
def __str__(self):
|
||||
return self.predname + '(' + ",".join(self.predargs) + ')'
|
||||
|
||||
|
||||
class CongressBuiltinCategoryMap(object):
|
||||
|
|
@ -235,7 +237,7 @@ class CongressBuiltinCategoryMap(object):
|
|||
# print 'category exists'
|
||||
for predtriple in value:
|
||||
pred = self.dict_predtriple_to_pred(predtriple)
|
||||
if not self.check_if_builtin(pred):
|
||||
if not self.builtin_is_registered(pred):
|
||||
self.categorydict[key].append(pred)
|
||||
self.sync_with_predlist(pred.predname, pred, key, 'add')
|
||||
|
||||
|
|
@ -267,7 +269,7 @@ class CongressBuiltinCategoryMap(object):
|
|||
self.categorydict[category].remove(pred)
|
||||
self.sync_with_predlist(name, pred, category, 'del')
|
||||
|
||||
def get_builtin_category_name(self, predname, predinputs):
|
||||
def get_category_name(self, predname, predinputs):
|
||||
if predname in self.preddict:
|
||||
if self.preddict[predname][0].num_inputs == predinputs:
|
||||
return self.preddict[predname][1]
|
||||
|
|
@ -309,54 +311,52 @@ class CongressBuiltinCategoryMap(object):
|
|||
else:
|
||||
assert("Category does not exist")
|
||||
|
||||
def check_if_builtin(self, predtotest):
|
||||
def builtin_is_registered(self, predtotest):
|
||||
"""Given a CongressBuiltinPred, check if it has been registered."""
|
||||
pname = predtotest.predname
|
||||
if pname in self.preddict:
|
||||
if self.preddict[pname][0].num_inputs == predtotest.num_inputs:
|
||||
return True
|
||||
return False
|
||||
|
||||
def check_if_builtin_by_name(self, predname, arity):
|
||||
def is_builtin(self, predname, arity):
|
||||
"""Given a name and arity, check if it is a builtin."""
|
||||
# print "check_if_builtin_by_name {} {}".format(predname, arity)
|
||||
if predname in self.preddict:
|
||||
if len(self.preddict[predname][0].predargs) == arity:
|
||||
return True
|
||||
return False
|
||||
|
||||
def return_builtin_pred(self, predname):
|
||||
def builtin(self, predname):
|
||||
"""Return a CongressBuiltinPred with name PREDNAME or None."""
|
||||
if predname in self.preddict:
|
||||
return self.preddict[predname][0]
|
||||
return None
|
||||
|
||||
def builtin_num_outputs(self, predname):
|
||||
if predname in self.preddict:
|
||||
pred = self.preddict[predname][0]
|
||||
return len(pred.predargs) - pred.num_inputs
|
||||
return 0
|
||||
|
||||
def list_available_builtins(self):
|
||||
"""Print out the list of builtins, by category."""
|
||||
for key, value in self.categorydict.items():
|
||||
predlist = self.categorydict[key]
|
||||
for pred in predlist:
|
||||
print pred
|
||||
print pred.pred_to_string()
|
||||
print str(pred)
|
||||
|
||||
def eval_builtin(self, code, arglist):
|
||||
return code(*arglist)
|
||||
|
||||
# a Singleton that serves as the entry point for builtin functionality
|
||||
builtin_registry = CongressBuiltinCategoryMap(_builtin_map)
|
||||
|
||||
|
||||
def main():
|
||||
cbcmap = CongressBuiltinCategoryMap(start_builtin_map)
|
||||
cbcmap = CongressBuiltinCategoryMap(_builtin_map)
|
||||
cbcmap.list_available_builtins()
|
||||
predl = cbcmap.return_builtin_pred('lt')
|
||||
predl = cbcmap.builtin('lt')
|
||||
print predl
|
||||
print 'printing pred'
|
||||
predl.string_to_pred('ltc(x,y)')
|
||||
cbcmap.list_available_builtins()
|
||||
cbcmap.delete_builtin('arithmetic', 'max', 2)
|
||||
cbcmap.list_available_builtins()
|
||||
predl = cbcmap.return_builtin_pred('plus')
|
||||
result = cbcmap.eval_builtin(predl.code, [1, 2])
|
||||
predl = cbcmap.builtin('plus')
|
||||
result = predl.code(1, 2)
|
||||
print result
|
||||
print cbcmap
|
||||
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@
|
|||
import unittest
|
||||
|
||||
from congress.openstack.common import log as logging
|
||||
from congress.policy.builtin.congressbuiltin \
|
||||
import CongressBuiltinCategoryMap as builtins
|
||||
from congress.policy.builtin.congressbuiltin import _builtin_map
|
||||
from congress.policy.builtin.congressbuiltin import CongressBuiltinCategoryMap
|
||||
from congress.policy.builtin.congressbuiltin import CongressBuiltinPred
|
||||
from congress.policy.builtin.congressbuiltin import start_builtin_map
|
||||
|
||||
from congress.policy import compile
|
||||
from congress.policy import runtime
|
||||
from congress.tests import helper
|
||||
|
|
@ -40,15 +40,12 @@ append_builtin = {'arithmetic': [{'func': 'div(x,y)',
|
|||
'num_inputs': 2,
|
||||
'code': 'lambda x,y: x / y'}]}
|
||||
|
||||
NREC_THEORY = 'non-recursive theory'
|
||||
DB_THEORY = 'database'
|
||||
|
||||
|
||||
class TestBuiltins(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.cbcmap = builtins(start_builtin_map)
|
||||
self.predl = self.cbcmap.return_builtin_pred('lt')
|
||||
self.cbcmap = CongressBuiltinCategoryMap(_builtin_map)
|
||||
self.predl = self.cbcmap.builtin('lt')
|
||||
|
||||
def test_add_and_delete_map(self):
|
||||
cbcmap_before = self.cbcmap
|
||||
|
|
@ -58,10 +55,10 @@ class TestBuiltins(unittest.TestCase):
|
|||
|
||||
def test_add_map_only(self):
|
||||
self.cbcmap.add_map(append_builtin)
|
||||
predl = self.cbcmap.return_builtin_pred('div')
|
||||
predl = self.cbcmap.builtin('div')
|
||||
self.assertNotEqual(predl, None)
|
||||
self.cbcmap.add_map(addmap)
|
||||
predl = self.cbcmap.return_builtin_pred('max')
|
||||
predl = self.cbcmap.builtin('max')
|
||||
self.assertNotEqual(predl, None)
|
||||
|
||||
def test_add_and_delete_builtin(self):
|
||||
|
|
@ -71,27 +68,27 @@ class TestBuiltins(unittest.TestCase):
|
|||
self.assertTrue(self.cbcmap.mapequal(cbcmap_before))
|
||||
|
||||
def test_string_pred_string(self):
|
||||
predstring = self.predl.pred_to_string()
|
||||
predstring = str(self.predl)
|
||||
self.assertNotEqual(predstring, 'ltc(x,y')
|
||||
|
||||
def test_add_and_delete_to_category(self):
|
||||
cbcmap_before = self.cbcmap
|
||||
arglist = ['x', 'y', 'z']
|
||||
pred = CongressBuiltinPred('testfunc', arglist, 1, 'lambda x: not x')
|
||||
pred = CongressBuiltinPred('testfunc', arglist, 1, lambda x: not x)
|
||||
self.cbcmap.insert_to_category('arithmetic', pred)
|
||||
self.cbcmap.delete_from_category('arithmetic', pred)
|
||||
self.assertTrue(self.cbcmap.mapequal(cbcmap_before))
|
||||
|
||||
def test_all_checks(self):
|
||||
predtotest = self.cbcmap.return_builtin_pred('lt')
|
||||
self.assertTrue(self.cbcmap.check_if_builtin(predtotest))
|
||||
predtotest = self.cbcmap.builtin('lt')
|
||||
self.assertTrue(self.cbcmap.builtin_is_registered(predtotest))
|
||||
|
||||
def test_eval_builtin(self):
|
||||
predl = self.cbcmap.return_builtin_pred('plus')
|
||||
result = self.cbcmap.eval_builtin(predl.code, [1, 2])
|
||||
predl = self.cbcmap.builtin('plus')
|
||||
result = predl.code(1, 2)
|
||||
self.assertEqual(result, 3)
|
||||
predl = self.cbcmap.return_builtin_pred('gt')
|
||||
result = self.cbcmap.eval_builtin(predl.code, [1, 2])
|
||||
predl = self.cbcmap.builtin('gt')
|
||||
result = predl.code(1, 2)
|
||||
self.assertEqual(result, False)
|
||||
|
||||
|
||||
|
|
@ -237,7 +234,12 @@ class TestReorder(unittest.TestCase):
|
|||
'Unsafety propagates')
|
||||
|
||||
|
||||
class TestNonrecursive(unittest.TestCase):
|
||||
NREC_THEORY = 'non-recursive theory test'
|
||||
MAT_THEORY = 'materialized view theory test'
|
||||
|
||||
|
||||
class TestTheories(unittest.TestCase):
|
||||
|
||||
def prep_runtime(self, code=None, msg=None, target=None):
|
||||
# compile source
|
||||
if msg is not None:
|
||||
|
|
@ -247,8 +249,10 @@ class TestNonrecursive(unittest.TestCase):
|
|||
if target is None:
|
||||
target = NREC_THEORY
|
||||
run = runtime.Runtime()
|
||||
run.theory[NREC_THEORY] = runtime.NonrecursiveRuleTheory()
|
||||
run.theory[DB_THEORY] = runtime.Database(name="Database", abbr="DB")
|
||||
run.theory[NREC_THEORY] = runtime.NonrecursiveRuleTheory(
|
||||
name="Nonrecursive", abbr="NRT")
|
||||
run.theory[MAT_THEORY] = runtime.MaterializedViewTheory(
|
||||
name="Materialized", abbr="MAT")
|
||||
run.debug_mode()
|
||||
run.insert(code, target=target)
|
||||
return run
|
||||
|
|
@ -257,9 +261,11 @@ class TestNonrecursive(unittest.TestCase):
|
|||
self.assertTrue(helper.datalog_equal(
|
||||
actual_string, correct_string, msg))
|
||||
|
||||
def test_builtins(self):
|
||||
def test_materialized_builtins(self):
|
||||
self.test_builtins(MAT_THEORY)
|
||||
|
||||
def test_builtins(self, th=NREC_THEORY):
|
||||
"""Test the mechanism that implements builtins."""
|
||||
th = NREC_THEORY
|
||||
run = self.prep_runtime()
|
||||
run.insert('p(x) :- q(x,y), plus(x,y,z), r(z)'
|
||||
'q(1,2)'
|
||||
|
|
@ -316,10 +322,28 @@ class TestNonrecursive(unittest.TestCase):
|
|||
self.check_equal(run.select('p(x)', target=th),
|
||||
'p(4)', "Bound output")
|
||||
|
||||
def test_builtins_content(self):
|
||||
run = self.prep_runtime()
|
||||
run.insert('p(x, z) :- plus(x,y,z), q(x), r(y)'
|
||||
'q(4)'
|
||||
'r(5)', target=th)
|
||||
self.check_equal(run.select('p(x, y)', target=th),
|
||||
'p(4, 9)',
|
||||
"Reordering")
|
||||
|
||||
run = self.prep_runtime()
|
||||
run.insert('p(x, z) :- plus(x,y,z), q(x), q(y)'
|
||||
'q(4)'
|
||||
'q(5)', target=th)
|
||||
self.check_equal(run.select('p(x, y)', target=th),
|
||||
'p(4, 9) p(4, 8) p(5, 9) p(5, 10)',
|
||||
"Reordering with self joins")
|
||||
|
||||
def test_materialized_builtins_content(self):
|
||||
self.test_builtins_content(MAT_THEORY)
|
||||
|
||||
def test_builtins_content(self, th=NREC_THEORY):
|
||||
"""Test the content of the builtins, not the mechanism"""
|
||||
def check_true(code, msg):
|
||||
th = NREC_THEORY
|
||||
run = self.prep_runtime('')
|
||||
run.insert(code, target=th)
|
||||
self.check_equal(
|
||||
|
|
|
|||
|
|
@ -24,8 +24,7 @@ import CongressLexer
|
|||
import CongressParser
|
||||
import utility
|
||||
|
||||
from builtin.congressbuiltin import CongressBuiltinCategoryMap as cbcmap
|
||||
from builtin.congressbuiltin import start_builtin_map as initbuiltin
|
||||
from builtin.congressbuiltin import builtin_registry
|
||||
|
||||
|
||||
class CongressException (Exception):
|
||||
|
|
@ -402,6 +401,8 @@ class Literal (object):
|
|||
return self.table.endswith('+') or self.table.endswith('-')
|
||||
|
||||
def tablename(self):
|
||||
# implemented simply so that we can call tablename() on either
|
||||
# rules or literals
|
||||
return self.table
|
||||
|
||||
|
||||
|
|
@ -715,7 +716,8 @@ def reorder_for_safety(rule):
|
|||
is evaluated. Reordering is stable, meaning that if the rule is
|
||||
properly ordered, no changes are made.
|
||||
"""
|
||||
cbcmapinst = cbcmap(initbuiltin)
|
||||
if not is_rule(rule):
|
||||
return rule
|
||||
safe_vars = set()
|
||||
unsafe_literals = []
|
||||
unsafe_variables = {} # dictionary from literal to its unsafe vars
|
||||
|
|
@ -741,9 +743,8 @@ def reorder_for_safety(rule):
|
|||
target_vars = None
|
||||
if lit.is_negated():
|
||||
target_vars = lit.variable_names()
|
||||
elif cbcmapinst.check_if_builtin_by_name(
|
||||
lit.table, len(lit.arguments)):
|
||||
builtin = cbcmapinst.return_builtin_pred(lit.table)
|
||||
elif builtin_registry.is_builtin(lit.table, len(lit.arguments)):
|
||||
builtin = builtin_registry.builtin(lit.table)
|
||||
target_vars = lit.arguments[0:builtin.num_inputs]
|
||||
target_vars = set([x.name for x in target_vars if x.is_variable()])
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ import cStringIO
|
|||
import os
|
||||
from unify import bi_unify_lists
|
||||
|
||||
from builtin.congressbuiltin import CongressBuiltinCategoryMap
|
||||
from builtin.congressbuiltin import start_builtin_map
|
||||
from builtin.congressbuiltin import builtin_registry
|
||||
|
||||
# FIXME there is a circular import here because compile.py imports runtime.py
|
||||
import compile
|
||||
|
|
@ -261,7 +260,6 @@ class Theory(object):
|
|||
self.trace_prefix = self.abbr[0:maxlength]
|
||||
else:
|
||||
self.trace_prefix = self.abbr + " " * (maxlength - len(self.abbr))
|
||||
self.cbcmap = CongressBuiltinCategoryMap(start_builtin_map)
|
||||
|
||||
def set_tracer(self, tracer):
|
||||
self.tracer = tracer
|
||||
|
|
@ -599,11 +597,9 @@ class TopDownTheory(Theory):
|
|||
elif lit.tablename() == 'false':
|
||||
self.print_fail(lit, context.binding, context.depth)
|
||||
return False
|
||||
elif self.cbcmap.check_if_builtin_by_name(lit.tablename(),
|
||||
len(lit.arguments)):
|
||||
elif builtin_registry.is_builtin(lit.table, len(lit.arguments)):
|
||||
self.print_call(lit, context.binding, context.depth)
|
||||
cbc = self.cbcmap.return_builtin_pred(lit.tablename())
|
||||
builtin_code = cbc.code
|
||||
builtin = builtin_registry.builtin(lit.table)
|
||||
# copy arguments into variables
|
||||
# PLUGGED is an instance of compile.Literal
|
||||
plugged = lit.plug(context.binding)
|
||||
|
|
@ -611,17 +607,16 @@ class TopDownTheory(Theory):
|
|||
# PLUGGED.arguments is a list of compile.Term
|
||||
# create args for function
|
||||
args = []
|
||||
for i in xrange(0, cbc.num_inputs):
|
||||
for i in xrange(0, builtin.num_inputs):
|
||||
assert plugged.arguments[i].is_object(), \
|
||||
("Builtins must be evaluated only after their "
|
||||
"inputs are ground: {} with num-inputs {}".format(
|
||||
str(plugged), cbc.num_inputs))
|
||||
str(plugged), builtin.num_inputs))
|
||||
args.append(plugged.arguments[i].name)
|
||||
# evaluate builtin: must return number, string, or iterable
|
||||
# of numbers/strings
|
||||
# print "args: " + str(args)
|
||||
try:
|
||||
result = self.cbcmap.eval_builtin(builtin_code, args)
|
||||
result = builtin.code(*args)
|
||||
except Exception as e:
|
||||
errmsg = "Error in builtin: " + str(e)
|
||||
self.print_note(lit, context.binding, context.depth, errmsg)
|
||||
|
|
@ -632,7 +627,7 @@ class TopDownTheory(Theory):
|
|||
# "Result: " + str(result))
|
||||
success = None
|
||||
undo = []
|
||||
if self.cbcmap.builtin_num_outputs(lit.table) > 0:
|
||||
if builtin.num_outputs > 0:
|
||||
# with return values, local success means we can bind
|
||||
# the results to the return value arguments
|
||||
if isinstance(result, (int, long, float, basestring)):
|
||||
|
|
@ -643,7 +638,7 @@ class TopDownTheory(Theory):
|
|||
unifier = self.new_bi_unifier()
|
||||
undo = bi_unify_lists(result,
|
||||
unifier,
|
||||
lit.arguments[cbc.num_inputs:],
|
||||
lit.arguments[builtin.num_inputs:],
|
||||
context.binding)
|
||||
# print "unifier: " + str(undo)
|
||||
success = undo is not None
|
||||
|
|
@ -1239,11 +1234,12 @@ class NonrecursiveRuleTheory(TopDownTheory):
|
|||
changes = []
|
||||
self.log(None, "Update " + iterstr(events))
|
||||
for event in events:
|
||||
formula = compile.reorder_for_safety(event.formula)
|
||||
if event.insert:
|
||||
if self.insert_actual(event.formula):
|
||||
if self.insert_actual(formula):
|
||||
changes.append(event)
|
||||
else:
|
||||
if self.delete_actual(event.formula):
|
||||
if self.delete_actual(formula):
|
||||
changes.append(event)
|
||||
return changes
|
||||
|
||||
|
|
@ -1423,7 +1419,6 @@ class DeltaRuleTheory (Theory):
|
|||
return False
|
||||
self.log(rule.tablename(), "Insert 2: {}".format(str(rule)))
|
||||
for delta in self.compute_delta_rules([rule]):
|
||||
self.reorder(delta)
|
||||
self.insert_delta(delta)
|
||||
self.originals.add(rule)
|
||||
return True
|
||||
|
|
@ -1588,23 +1583,16 @@ class DeltaRuleTheory (Theory):
|
|||
for rule in formulas:
|
||||
if rule.is_atom():
|
||||
continue
|
||||
rule = compile.reorder_for_safety(rule)
|
||||
for literal in rule.body:
|
||||
if builtin_registry.is_builtin(
|
||||
literal.table, len(literal.arguments)):
|
||||
continue
|
||||
newbody = [lit for lit in rule.body if lit is not literal]
|
||||
delta_rules.append(
|
||||
DeltaRule(literal, rule.head, newbody, rule))
|
||||
return delta_rules
|
||||
|
||||
@classmethod
|
||||
def reorder(cls, delta):
|
||||
"""Given a delta rule DELTA, re-order its body for efficient
|
||||
and correct computation.
|
||||
"""
|
||||
# ensure negatives come after positives
|
||||
positives = [lit for lit in delta.body if not lit.is_negated()]
|
||||
negatives = [lit for lit in delta.body if lit.is_negated()]
|
||||
positives.extend(negatives)
|
||||
delta.body = positives
|
||||
|
||||
|
||||
class MaterializedViewTheory(TopDownTheory):
|
||||
"""A theory that stores the table contents of views explicitly.
|
||||
|
|
@ -1778,7 +1766,6 @@ class MaterializedViewTheory(TopDownTheory):
|
|||
# need to eliminate self-joins here so that we fill all
|
||||
# the tables introduced by self-join elimination.
|
||||
for rule in DeltaRuleTheory.eliminate_self_joins([formula]):
|
||||
DeltaRuleTheory.reorder(rule)
|
||||
new_event = Event(formula=rule, insert=event.insert,
|
||||
target=event.target)
|
||||
self.enqueue(new_event)
|
||||
|
|
|
|||
|
|
@ -444,16 +444,6 @@ class TestRuntime(base.TestCase):
|
|||
run, 'q(1,2) r(2,3) r(2,4) u(3,5) u(4,6) s(1,3) s(1,4)',
|
||||
'Insert into non-unary with different propagation')
|
||||
|
||||
# Negation ordering
|
||||
code = ("p(x) :- not q(x), r(x)")
|
||||
run = self.prep_runtime(code, "Negation ordering")
|
||||
run.insert('r(1)', MAT_THEORY)
|
||||
run.insert('r(2)', MAT_THEORY)
|
||||
run.insert('q(1)', MAT_THEORY)
|
||||
self.check_class(
|
||||
run, 'r(1) r(2) q(1) p(2)',
|
||||
'Reordering negation')
|
||||
|
||||
def test_select(self):
|
||||
"""Materialized Theory: test the SELECT event handler."""
|
||||
code = ("p(x, y) :- q(x), r(y)")
|
||||
|
|
|
|||
Loading…
Reference in New Issue