openstack
/
monasca-analytics
Code Issues Proposed changes

Merge "Add Banana specific APIs to typecheck and get list of components."

This commit is contained in:
Jenkins 2016-11-15 21:31:09 +00:00 committed by Gerrit Code Review
parent c8cfdd54e7 716a72d9b9
commit 281fee265d
9 changed files with 187 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
devstack/settings
View File

@ -47,7 +47,7 @@ SPARK_DIR="/opt/spark"
SPARK_DOWNLOAD="$SPARK_DIR/download"
SPARK_VERSION=${SPARK_VERSION:-1.6.1}
SPARK_TARBALL_NAME="spark-${SPARK_VERSION}.tgz"
SPARK_URL="http://apache.claz.org/spark/spark-$SPARK_VERSION/$SPARK_TARBALL_NAME"
SPARK_URL="http://archive.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK_TARBALL_NAME"
BASE_KAFKA_VERSION=${BASE_KAFKA_VERSION:-0.9.0.0}
KAFKA_DIR="/opt/kafka"

4
monasca_analytics/banana/grammar/base_ast.py
View File

@ -153,11 +153,11 @@ class Span(object):
DUMMY_SPAN = Span(None, 0, 0)
def from_parse_fatal(parse_fatal_exception):
def from_pyparsing_exception(parse_fatal_exception):
"""
Convert the provided ParseFatalException into a Span.
:type parse_fatal_exception: pyparsing.ParseFatalException
:type parse_fatal_exception: pyparsing.ParseBaseException
:param parse_fatal_exception: Exception to convert.
:rtype: Span
:return: Returns the span mapping to that fatal exception.

54
monasca_analytics/banana/pass_manager.py
View File

@ -26,13 +26,13 @@ import monasca_analytics.banana.typeck.config as typeck
import monasca_analytics.exception.banana as exception
def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()):
def execute_banana_string(banana, driver=None, emitter=emit.PrintEmitter()):
"""
Execute the provided banana string.
It will run the parse phase, and the typechecker.
:type banana_str: str
:param banana_str: The string to parse and type check.
:type driver: monasca_analytics.spark.driver.DriverExecutor
:type banana: str
:param banana: The string to parse and type check.
:type driver: monasca_analytics.spark.driver.DriverExecutor | None
:param driver: Driver that will manage the created
components and connect them together.
:type emitter: emit.Emitter
@ -41,7 +41,7 @@ def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()):
try:
# Convert the grammar into an AST
parser = grammar.banana_grammar(emitter)
ast = parser.parse(banana_str)
ast = parser.parse(banana)
# Compute the type table for the given AST
type_table = typeck.typeck(ast)
# Remove from the tree path that are "dead"
@ -49,30 +49,56 @@ def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()):
# Check that there's at least one path to be executed
deadpathck.contains_at_least_one_path_to_a_sink(ast, type_table)
# Evaluate the script
ev.eval_ast(ast, type_table, driver)
if driver is not None:
ev.eval_ast(ast, type_table, driver)
except exception.BananaException as err:
emitter.emit_error(err.get_span(), str(err))
except p.ParseSyntaxException as err:
emitter.emit_error(span_util.from_parse_fatal(err), err.msg)
emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg)
except p.ParseFatalException as err:
emitter.emit_error(span_util.from_parse_fatal(err), err.msg)
emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg)
except p.ParseException as err:
emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg)
def compute_type_table(banana_str):
def try_compute_type_table(banana):
"""
Compute the type table for the provided banana string
if possible. Does not throw any exception if it fails.
:type banana: str
:param banana: The string to parse and type check.
"""
try:
# Convert the grammar into an AST
parser = grammar.banana_grammar()
ast = parser.parse(banana)
# Compute the type table for the given AST
return typeck.typeck(ast)
except exception.BananaException:
return None
except p.ParseSyntaxException:
return None
except p.ParseFatalException:
return None
except p.ParseException:
return None
def compute_type_table(banana):
"""
Compute the type table for the provided banana string
if possible.
:type banana_str: str
:param banana_str: The string to parse and type check.
:type banana: str
:param banana: The string to parse and type check.
"""
# Convert the grammar into an AST
parser = grammar.banana_grammar()
ast = parser.parse(banana_str)
ast = parser.parse(banana)
# Compute the type table for the given AST
return typeck.typeck(ast)
def compute_evaluation_context(banana_str, cb=lambda *a, **k: None):
def compute_evaluation_context(banana, cb=lambda *a, **k: None):
"""
Compute the evaluation context for the provided
banana string.
@ -81,7 +107,7 @@ def compute_evaluation_context(banana_str, cb=lambda *a, **k: None):
:param cb: Callback called after each statement
"""
parser = grammar.banana_grammar()
ast = parser.parse(banana_str)
ast = parser.parse(banana)
type_table = typeck.typeck(ast)
context = ctx.EvaluationContext()

13
monasca_analytics/banana/typeck/type_table.py
View File

@ -164,6 +164,19 @@ class TypeTable(object):
))
self._variables = new_snapshot
def to_json(self):
"""
Convert this type table into a dictionary.
Useful to serialize the type table.
:rtype: dict
:return: Returns this type table as a dict.
"""
res = {}
for key, val in self._variables.iteritems():
res[key.inner_val()] = val.to_json()
return res
def __contains__(self, key):
"""
Test if the type table contains or not the provided

33
monasca_analytics/banana/typeck/type_util.py
View File

@ -59,6 +59,10 @@ class IsType(object):
def default_value(self):
pass
@abc.abstractmethod
def to_json(self):
pass
class Any(IsType):
"""
@ -84,6 +88,9 @@ class Any(IsType):
def default_value(self):
return {}
def to_json(self):
return {"id": "any"}
class String(IsType):
"""
@ -102,6 +109,9 @@ class String(IsType):
def default_value(self):
return ""
def to_json(self):
return {"id": "string"}
class Number(String):
"""
@ -120,6 +130,9 @@ class Number(String):
def default_value(self):
return 0
def to_json(self):
return {"id": "number"}
class Enum(String):
"""
@ -142,6 +155,12 @@ class Enum(String):
def default_value(self):
return ""
def to_json(self):
return {
"id": "enum",
"variants": self.variants
}
def attach_to_root(root_obj, obj1, span, erase_existing=False):
"""
@ -281,6 +300,12 @@ class Object(String):
default_value[key] = val.default_value()
return default_value
def to_json(self):
res = {"id": "object", "props": {}}
for key, val in self.props.iteritems():
res["props"][key] = val.to_json()
return res
class Component(IsType):
"""
@ -397,7 +422,13 @@ class Component(IsType):
return hash(str(self))
def default_value(self):
return {}
return None
def to_json(self):
res = {"id": "component", "name": self.class_name, "args": []}
for arg in self.ctor_properties:
res["args"].append(arg.to_json())
return res
class Source(Component):

7
monasca_analytics/component/params.py
View File

@ -45,3 +45,10 @@ class ParamDescriptor(object):
self.default_value = default
self.param_type = _type
self.validator = validator
def to_json(self):
return {
"name": self.param_name,
"default_value": self.default_value,
"type": self.param_type.to_json(),
}

23
monasca_analytics/monanas.py
View File

@ -76,6 +76,29 @@ class Monanas(object):
# Try to change the configuration.
executor.execute_banana_string(banana_str, self._driver, emitter)
def typeck_configuration(self, banana_str, emitter):
"""Only type check the provided configuration.
:type banana_str: str
:param banana_str: New configuration.
:type emitter: emit.JsonEmitter
:param emitter: a Json emitter instance
"""
executor.execute_banana_string(banana_str, None, emitter)
def compute_type_table(self, banana_str):
"""Compute the type table for the provided configuration.
:type banana_str: str
:param banana_str: Configuration to test.
:rtype: dict
:return: Returns the type table
"""
type_table = executor.try_compute_type_table(banana_str)
if type_table is not None:
return type_table.to_json()
return {}
def start_streaming(self):
"""Starts streaming data.

69
monasca_analytics/web_service/request_handler.py
View File

@ -24,6 +24,7 @@ import voluptuous
import monasca_analytics.banana.emitter as emit
import monasca_analytics.exception.monanas as err
import monasca_analytics.util.common_util as introspect
from monasca_analytics.web_service import web_service_model
@ -77,12 +78,13 @@ class BananaHandler(web.RequestHandler):
the banana configuration language.
"""
def initialize(self, monanas):
def initialize(self, monanas, typeck_only):
"""Initialize the handler.
:param monanas: A Monana's instance.
"""
self._monanas = monanas
self._typeck_only = typeck_only
@web.asynchronous
def post(self):
@ -93,18 +95,15 @@ class BananaHandler(web.RequestHandler):
body = json.loads(self.request.body)
web_service_model.banana_model(body)
emitter = emit.JsonEmitter()
# TODO(Joan): Change that
self._monanas.try_change_configuration(body["content"], emitter)
if self._typeck_only:
self._monanas.typeck_configuration(body["content"],
emitter)
else:
self._monanas.try_change_configuration(body["content"],
emitter)
self.write(emitter.result)
except (AttributeError, voluptuous.Invalid, ValueError):
except (AttributeError, voluptuous.Invalid, ValueError) as e:
self.set_status(400, "The request body was malformed.")
except (err.MonanasBindSourcesError,
err.MonanasAlreadyStartedStreaming,
err.MonanasAlreadyStoppedStreaming) as e:
self.set_status(400, e.__str__())
except err.MonanasStreamingError as e:
self.set_status(500, e.__str__())
terminate = (True, e.__str__())
except Exception as e:
tb = traceback.format_exc()
print(tb)
@ -118,3 +117,51 @@ class BananaHandler(web.RequestHandler):
if terminate[0]:
logger.error(terminate[1])
self._monanas.stop_streaming_and_terminate()
class BananaMetaDataHandler(web.RequestHandler):
def initialize(self, monanas):
"""Initializes the handler.
:param monanas: Monanas -- A Monanas's instance.
"""
self._monanas = monanas
@web.asynchronous
def get(self):
all_components = introspect.get_available_classes()
result = {"components": []}
for kind, components in all_components.iteritems():
for component in components:
result["components"].append({
"name": component.__name__,
"description": component.__doc__,
"params": map(lambda x: x.to_json(),
component.get_params()),
})
self.write(result)
self.flush()
self.finish()
@web.asynchronous
def post(self):
try:
body = json.loads(self.request.body)
web_service_model.banana_model(body)
type_table = self._monanas.compute_type_table(body["content"])
self.write(type_table)
except (AttributeError, voluptuous.Invalid, ValueError) as e:
logger.warn("Wrong request: {}.".
format(e))
self.set_status(400, "The request body was malformed.")
except Exception as e:
tb = traceback.format_exc()
print(tb)
logger.error("Unexpected error: {}. {}".
format(sys.exc_info()[0], e))
self.set_status(500, "Internal server error.")
self.flush()
self.finish()

12
monasca_analytics/web_service/web_service.py
View File

@ -29,7 +29,17 @@ class WebService(web.Application):
params = {"monanas": self._monanas}
handlers = [
(r"/", request_handler.MonanasHandler, params),
(r"/banana", request_handler.BananaHandler, params),
(r"/banana", request_handler.BananaHandler, {
"monanas": self._monanas,
"typeck_only": False
}),
(r"/banana/typeck", request_handler.BananaHandler, {
"monanas": self._monanas,
"typeck_only": True
}),
(r"/banana/metadata", request_handler.BananaMetaDataHandler, {
"monanas": self._monanas,
})
]
settings = {}