Adds a parser that compiles gerrit search queries into sqlalchemy filters to support the gerrit search syntax both in arbitrary searches (ctrl-o) as well as dashboards. Change-Id: Ibc8e5c180b0ca16727938725cfe6380ba7315205changes/31/109631/2
James E. Blair
9 years ago
parent
42419904a8
commit
6181e4f9bf
@ -0,0 +1,26 @@
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from gertty.search import tokenizer, parser
|
||||
|
||||
class SearchSyntaxError(Exception):
|
||||
pass
|
||||
|
||||
class SearchCompiler(object):
|
||||
def __init__(self, app):
|
||||
self.lexer = tokenizer.SearchTokenizer()
|
||||
self.parser = parser.SearchParser()
|
||||
|
||||
def parse(self, data):
|
||||
return self.parser.parse(data, lexer=self.lexer)
|
||||
@ -0,0 +1,236 @@
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime
|
||||
import re
|
||||
|
||||
import ply.yacc as yacc
|
||||
from sqlalchemy.sql.expression import and_, or_
|
||||
|
||||
import gertty.db
|
||||
import gertty.search
|
||||
|
||||
from tokenizer import tokens
|
||||
|
||||
def SearchParser():
|
||||
def p_terms(p):
|
||||
'''expression : list_expr
|
||||
| paren_expr
|
||||
| boolean_expr
|
||||
| negative_expr
|
||||
| term'''
|
||||
p[0] = p[1]
|
||||
|
||||
def p_list_expr(p):
|
||||
'''list_expr : expression expression'''
|
||||
p[0] = and_(p[1], p[2])
|
||||
|
||||
def p_paren_expr(p):
|
||||
'''paren_expr : LPAREN expression RPAREN'''
|
||||
p[0] = p[2]
|
||||
|
||||
def p_boolean_expr(p):
|
||||
'''boolean_expr : expression AND expression
|
||||
| expression OR expression'''
|
||||
if p[2] == 'and':
|
||||
p[0] = and_(p[1], p[3])
|
||||
elif p[2] == 'or':
|
||||
p[0] = or_(p[1], p[3])
|
||||
else:
|
||||
raise SyntaxErro()
|
||||
|
||||
def p_negative_expr(p):
|
||||
'''negative_expr : NOT expression
|
||||
| NEG expression'''
|
||||
p[0] = not p[1]
|
||||
|
||||
def p_term(p):
|
||||
'''term : age_term
|
||||
| change_term
|
||||
| owner_term
|
||||
| reviewer_term
|
||||
| commit_term
|
||||
| project_term
|
||||
| project_key_term
|
||||
| branch_term
|
||||
| topic_term
|
||||
| ref_term
|
||||
| label_term
|
||||
| message_term
|
||||
| comment_term
|
||||
| has_term
|
||||
| is_term
|
||||
| status_term
|
||||
| op_term'''
|
||||
p[0] = p[1]
|
||||
|
||||
def p_string(p):
|
||||
'''string : SSTRING
|
||||
| DSTRING
|
||||
| USTRING'''
|
||||
p[0] = p[1]
|
||||
|
||||
def p_age_unit(p):
|
||||
'''age_unit : SECONDS
|
||||
| MINUTES
|
||||
| HOURS
|
||||
| DAYS
|
||||
| WEEKS
|
||||
| MONTHS
|
||||
| YEARS'''
|
||||
p[0] = p[1]
|
||||
|
||||
def p_age_term(p):
|
||||
'''age_term : OP_AGE NUMBER age_unit'''
|
||||
now = datetime.datetime.utcnow()
|
||||
delta = p[1]
|
||||
unit = p[2]
|
||||
if unit == 'minutes':
|
||||
delta = delta * 60
|
||||
elif unit == 'hours':
|
||||
delta = delta * 60 * 60
|
||||
elif unit == 'days':
|
||||
delta = delta * 60 * 60 * 60
|
||||
elif unit == 'weeks':
|
||||
delta = delta * 60 * 60 * 60 * 7
|
||||
elif unit == 'months':
|
||||
delta = delta * 60 * 60 * 60 * 30
|
||||
elif unit == 'years':
|
||||
delta = delta * 60 * 60 * 60 * 365
|
||||
p[0] = gertty.db.change_table.c.updated < (now-delta)
|
||||
|
||||
def p_change_term(p):
|
||||
'''change_term : OP_CHANGE CHANGE_ID
|
||||
| OP_CHANGE NUMBER'''
|
||||
if type(p[2]) == int:
|
||||
p[0] = gertty.db.change_table.c.number == p[2]
|
||||
else:
|
||||
p[0] = gertty.db.change_table.c.change_id == p[2]
|
||||
|
||||
def p_owner_term(p):
|
||||
'''owner_term : OP_OWNER string'''
|
||||
p[0] = gertty.db.change_table.c.owner == p[2]
|
||||
|
||||
def p_reviewer_term(p):
|
||||
'''reviewer_term : OP_REVIEWER string'''
|
||||
p[0] = gertty.db.approval_table.c.name == p[2]
|
||||
|
||||
def p_commit_term(p):
|
||||
'''commit_term : OP_COMMIT string'''
|
||||
p[0] = gertty.db.revision_table.c.commit == p[2]
|
||||
|
||||
def p_project_term(p):
|
||||
'''project_term : OP_PROJECT string'''
|
||||
#TODO: support regex
|
||||
p[0] = gertty.db.project_table.c.name == p[2]
|
||||
|
||||
def p_project_key_term(p):
|
||||
'''project_key_term : OP_PROJECT_KEY NUMBER'''
|
||||
p[0] = gertty.db.change_table.c.project_key == p[2]
|
||||
|
||||
def p_branch_term(p):
|
||||
'''branch_term : OP_BRANCH string'''
|
||||
#TODO: support regex
|
||||
p[0] = gertty.db.change_table.c.branch == p[2]
|
||||
|
||||
def p_topic_term(p):
|
||||
'''topic_term : OP_TOPIC string'''
|
||||
#TODO: support regex
|
||||
p[0] = gertty.db.change_table.c.topic == p[2]
|
||||
|
||||
def p_ref_term(p):
|
||||
'''ref_term : OP_REF string'''
|
||||
#TODO: support regex
|
||||
p[0] = gertty.db.change_table.c.branch == p[2][len('refs/heads/'):]
|
||||
|
||||
label_re = re.compile(r'(?P<label>[a-zA-Z0-9_-]+([a-zA-Z]|((?<![-+])[0-9])))'
|
||||
r'(?P<operator>[<>]?=?)(?P<value>[-+]?[0-9]+)'
|
||||
r'($|,user=(?P<user>\S+))')
|
||||
|
||||
def p_label_term(p):
|
||||
'''label_term : OP_LABEL string'''
|
||||
args = label_re.match(p[2])
|
||||
label = args.group('label')
|
||||
op = args.group('operator') or '='
|
||||
value = int(args.group('value'))
|
||||
user = args.group('user')
|
||||
|
||||
filters = []
|
||||
filters.append(gertty.db.approval_table.c.category == label)
|
||||
if op == '=':
|
||||
filters.append(gertty.db.approval_table.c.value == value)
|
||||
elif op == '>=':
|
||||
filters.append(gertty.db.approval_table.c.value >= value)
|
||||
elif op == '<=':
|
||||
filters.append(gertty.db.approval_table.c.value <= value)
|
||||
if user is not None:
|
||||
filters.append(gertty.db.approval_table.c.name == user)
|
||||
p[0] = and_(*filters)
|
||||
|
||||
def p_message_term(p):
|
||||
'''message_term : OP_MESSAGE string'''
|
||||
p[0] = gertty.db.revision_table.c.message.like(p[1])
|
||||
|
||||
def p_comment_term(p):
|
||||
'''comment_term : OP_COMMENT string'''
|
||||
p[0] = and_(gertty.db.message_table.c.message.like(p[1]),
|
||||
gertty.db.comment_table.c.message.like(p[1]))
|
||||
|
||||
def p_has_term(p):
|
||||
'''has_term : OP_HAS string'''
|
||||
#TODO: implement star
|
||||
if p[2] == 'draft':
|
||||
p[0] = gertty.db.message_table.c.pending == True
|
||||
else:
|
||||
raise gertty.search.SearchSyntaxError('Syntax error: has:%s is not supported' % p[2])
|
||||
|
||||
def p_is_term(p):
|
||||
'''is_term : OP_IS string'''
|
||||
#TODO: implement starred, watched, owner, reviewer, draft
|
||||
if p[2] == 'reviewed':
|
||||
p[0] = gertty.db.approval_table.c.value != 0
|
||||
elif p[2] == 'open':
|
||||
p[0] = gertty.db.change_table.c.status.notin_(['MERGED', 'ABANDONED'])
|
||||
elif p[2] == 'closed':
|
||||
p[0] = gertty.db.change_table.c.status.in_(['MERGED', 'ABANDONED'])
|
||||
elif p[2] == 'submitted':
|
||||
p[0] = gertty.db.change_table.c.status == 'SUBMITTED'
|
||||
elif p[2] == 'merged':
|
||||
p[0] = gertty.db.change_table.c.status == 'MERGED'
|
||||
elif p[2] == 'abandoned':
|
||||
p[0] = gertty.db.change_table.c.status == 'ABANDONED'
|
||||
else:
|
||||
raise gertty.search.SearchSyntaxError('Syntax error: has:%s is not supported' % p[2])
|
||||
|
||||
def p_status_term(p):
|
||||
'''status_term : OP_STATUS string'''
|
||||
if p[2] == 'open':
|
||||
p[0] = gertty.db.change_table.c.status.notin_(['MERGED', 'ABANDONED'])
|
||||
elif p[2] == 'closed':
|
||||
p[0] = gertty.db.change_table.c.status.in_(['MERGED', 'ABANDONED'])
|
||||
else:
|
||||
p[0] = gertty.db.change_table.c.status == p[2].upper()
|
||||
|
||||
def p_op_term(p):
|
||||
'op_term : OP'
|
||||
raise SyntaxError()
|
||||
|
||||
def p_error(p):
|
||||
if p:
|
||||
raise gertty.search.SearchSyntaxError('Syntax error at "%s" in search string "%s" (col %s)' % (
|
||||
p.lexer.lexdata[p.lexpos:], p.lexer.lexdata, p.lexpos))
|
||||
else:
|
||||
raise gertty.search.SearchSyntaxError('Syntax error: EOF in search string')
|
||||
|
||||
return yacc.yacc(debug=0, write_tables=0)
|
||||
@ -0,0 +1,73 @@
|
||||
import gertty.search
|
||||
import re
|
||||
import sys
|
||||
|
||||
label_re = re.compile(r'(?P<label>[a-zA-Z0-9_-]+([a-zA-Z]|((?<![-+])[0-9])))'
|
||||
r'(?P<operator>[<>]?=?)(?P<value>[-+]?[0-9]+)'
|
||||
r'($|,user=(?P<user>\S+))')
|
||||
|
||||
for a in [
|
||||
'Code-Review=1',
|
||||
'Code-Review=+1',
|
||||
'Code-Review=-1',
|
||||
'Code-Review>=+1',
|
||||
'Code-Review<=-1',
|
||||
'Code-Review+1',
|
||||
'Code-Review-1',
|
||||
]:
|
||||
for b in [
|
||||
'',
|
||||
',user=corvus',
|
||||
]:
|
||||
data = a+b
|
||||
print
|
||||
print data
|
||||
m = label_re.match(data)
|
||||
print 'res', m and m.groups()
|
||||
|
||||
#sys.exit(0)
|
||||
parser = gertty.search.SearchCompiler(None)
|
||||
|
||||
import tokenizer
|
||||
lexer = tokenizer.SearchTokenizer()
|
||||
lexer.input("project:foo/bar")
|
||||
|
||||
# Tokenize
|
||||
while True:
|
||||
tok = lexer.token()
|
||||
if not tok: break # No more input
|
||||
print tok
|
||||
|
||||
#TODO: unit test
|
||||
for a in [
|
||||
'label:Code-Review=1',
|
||||
'label:Code-Review=+1',
|
||||
'label:Code-Review=-1',
|
||||
'label:Code-Review>=+1',
|
||||
'label:Code-Review<=-1',
|
||||
'label:Code-Review+1',
|
||||
'label:Code-Review-1',
|
||||
]:
|
||||
for b in [
|
||||
'',
|
||||
',user=corvus',
|
||||
]:
|
||||
data = a+b
|
||||
print
|
||||
print data
|
||||
result = parser.parse(data)
|
||||
print 'res', str(result)
|
||||
|
||||
for data in [
|
||||
'_project_key:18 status:open',
|
||||
'project:foo/bar status:open',
|
||||
'project:foo and status:open',
|
||||
'project:foo or status:open',
|
||||
'project:foo and (status:merged or status:new)',
|
||||
'project:foo or project:bar or project:baz',
|
||||
'project:foo project:bar project:baz',
|
||||
]:
|
||||
print
|
||||
print data
|
||||
result = parser.parse(data)
|
||||
print 'res', str(result)
|
||||
@ -0,0 +1,156 @@
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import ply.lex as lex
|
||||
|
||||
operators = {
|
||||
'age': 'OP_AGE',
|
||||
'change': 'OP_CHANGE',
|
||||
'owner': 'OP_OWNER',
|
||||
#'OP_OWNERIN', # needs local group membership
|
||||
'reviewer': 'OP_REVIEWER',
|
||||
#'OP_REVIEWERIN', # needs local group membership
|
||||
'commit': 'OP_COMMIT',
|
||||
'project': 'OP_PROJECT',
|
||||
'_project_key': 'OP_PROJECT_KEY', # internal gertty use only
|
||||
'branch': 'OP_BRANCH',
|
||||
'topic': 'OP_TOPIC',
|
||||
'ref': 'OP_REF',
|
||||
#'tr': 'OP_TR', # needs trackingids
|
||||
#'bug': 'OP_BUG', # needs trackingids
|
||||
'label': 'OP_LABEL',
|
||||
'message': 'OP_MESSAGE',
|
||||
'comment': 'OP_COMMENT',
|
||||
#'file': 'OP_FILE', # needs local file list
|
||||
'has': 'OP_HAS',
|
||||
'is': 'OP_IS',
|
||||
'status': 'OP_STATUS',
|
||||
}
|
||||
|
||||
reserved = {
|
||||
'or|OR': 'OR',
|
||||
'not|NOT': 'NOT',
|
||||
}
|
||||
|
||||
tokens = [
|
||||
'OP',
|
||||
'AND',
|
||||
'OR',
|
||||
'NOT',
|
||||
'NEG',
|
||||
'LPAREN',
|
||||
'RPAREN',
|
||||
'SECONDS',
|
||||
'MINUTES',
|
||||
'HOURS',
|
||||
'DAYS',
|
||||
'WEEKS',
|
||||
'MONTHS',
|
||||
'YEARS',
|
||||
'NUMBER',
|
||||
'CHANGE_ID',
|
||||
'SSTRING',
|
||||
'DSTRING',
|
||||
'USTRING',
|
||||
#'REGEX',
|
||||
#'SHA',
|
||||
] + operators.values()
|
||||
|
||||
def SearchTokenizer():
|
||||
t_CHANGE_ID = r'I[a-fA-F0-9]{7,40}'
|
||||
t_LPAREN = r'\('
|
||||
t_RPAREN = r'\)'
|
||||
t_NEG = r'!'
|
||||
|
||||
def t_OP(t):
|
||||
r'[a-zA-Z_][a-zA-Z_]*:'
|
||||
t.type = operators.get(t.value[:-1], 'OP')
|
||||
return t
|
||||
|
||||
def t_SSTRING(t):
|
||||
r"'([^\\']+|\\'|\\\\)*'"
|
||||
t.value=t.value[1:-1].decode("string-escape")
|
||||
return t
|
||||
|
||||
def t_DSTRING(t):
|
||||
r'"([^\\"]+|\\"|\\\\)*"'
|
||||
t.value=t.value[1:-1].decode("string-escape")
|
||||
return t
|
||||
|
||||
def t_AND(t):
|
||||
r'and|AND'
|
||||
return t
|
||||
|
||||
def t_OR(t):
|
||||
r'or|OR'
|
||||
return t
|
||||
|
||||
def t_NOT(t):
|
||||
r'not|NOT'
|
||||
return t
|
||||
|
||||
def t_INTEGER(t):
|
||||
r'[+-]\d+'
|
||||
t.value = int(t.value)
|
||||
return t
|
||||
|
||||
def t_NUMBER(t):
|
||||
r'\d+'
|
||||
t.value = int(t.value)
|
||||
return t
|
||||
|
||||
def t_USTRING(t):
|
||||
r'([^\s\(\)!]+)'
|
||||
t.value=t.value.decode("string-escape")
|
||||
return t
|
||||
|
||||
def t_SECONDS(t):
|
||||
r's|sec|second|seconds'
|
||||
t.value = 'seconds'
|
||||
|
||||
def t_MINUTES(t):
|
||||
r'm|min|minute|minutes'
|
||||
t.value = 'minutes'
|
||||
|
||||
def t_HOURS(t):
|
||||
r'h|hr|hour|hours'
|
||||
t.value = 'hours'
|
||||
|
||||
def t_DAYS(t):
|
||||
r'd|day|days'
|
||||
t.value = 'days'
|
||||
|
||||
def t_WEEKS(t):
|
||||
r'w|week|weeks'
|
||||
t.value = 'weeks'
|
||||
|
||||
def t_MONTHS(t):
|
||||
r'mon|month|months'
|
||||
t.value = 'months'
|
||||
|
||||
def t_YEARS(t):
|
||||
r'y|year|years'
|
||||
t.value = 'years'
|
||||
|
||||
def t_newline(t):
|
||||
r'\n+'
|
||||
t.lexer.lineno += len(t.value)
|
||||
|
||||
t_ignore = ' \t'
|
||||
|
||||
def t_error(t):
|
||||
print "Illegal character '%s'" % t.value[0]
|
||||
t.lexer.skip(1)
|
||||
|
||||
return lex.lex()
|
||||
Loading…
Reference in new issue