From 716a72d9b988a81ab1f1f4519fa44b8b235c5e77 Mon Sep 17 00:00:00 2001 From: Joan Varvenne Date: Wed, 21 Sep 2016 15:30:55 +0100 Subject: [PATCH] Add Banana specific APIs to typecheck and get list of components. This commit add two APIs: * POST /banana/typeck { "content": "" } This API type check the provided banana content but does not evaluate it. * POST /banana/metadata { "content": "" } This API returns the associated TypeTable of the provided banana content. This is useful to provide auto-completion in an editor. * GET /banana/metadata This API returns the list of components availables. This is also useful to provide auto-completion on existing components in an editor for banana. Change-Id: I31c1de86fe420458ac98aad9d9c8ae6ca73fcc81 --- devstack/settings | 2 +- monasca_analytics/banana/grammar/base_ast.py | 4 +- monasca_analytics/banana/pass_manager.py | 54 +++++++++++---- monasca_analytics/banana/typeck/type_table.py | 13 ++++ monasca_analytics/banana/typeck/type_util.py | 33 ++++++++- monasca_analytics/component/params.py | 7 ++ monasca_analytics/monanas.py | 23 +++++++ .../web_service/request_handler.py | 69 ++++++++++++++++--- monasca_analytics/web_service/web_service.py | 12 +++- 9 files changed, 187 insertions(+), 30 deletions(-) diff --git a/devstack/settings b/devstack/settings index 6069284..f921c1c 100644 --- a/devstack/settings +++ b/devstack/settings @@ -47,7 +47,7 @@ SPARK_DIR="/opt/spark" SPARK_DOWNLOAD="$SPARK_DIR/download" SPARK_VERSION=${SPARK_VERSION:-1.6.1} SPARK_TARBALL_NAME="spark-${SPARK_VERSION}.tgz" -SPARK_URL="http://apache.claz.org/spark/spark-$SPARK_VERSION/$SPARK_TARBALL_NAME" +SPARK_URL="http://archive.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK_TARBALL_NAME" BASE_KAFKA_VERSION=${BASE_KAFKA_VERSION:-0.9.0.0} KAFKA_DIR="/opt/kafka" diff --git a/monasca_analytics/banana/grammar/base_ast.py b/monasca_analytics/banana/grammar/base_ast.py index f11044f..dabe917 100644 --- a/monasca_analytics/banana/grammar/base_ast.py +++ b/monasca_analytics/banana/grammar/base_ast.py @@ -129,11 +129,11 @@ class Span(object): DUMMY_SPAN = Span(None, 0, 0) -def from_parse_fatal(parse_fatal_exception): +def from_pyparsing_exception(parse_fatal_exception): """ Convert the provided ParseFatalException into a Span. - :type parse_fatal_exception: pyparsing.ParseFatalException + :type parse_fatal_exception: pyparsing.ParseBaseException :param parse_fatal_exception: Exception to convert. :rtype: Span :return: Returns the span mapping to that fatal exception. diff --git a/monasca_analytics/banana/pass_manager.py b/monasca_analytics/banana/pass_manager.py index 7eb7ba3..36a4ca5 100644 --- a/monasca_analytics/banana/pass_manager.py +++ b/monasca_analytics/banana/pass_manager.py @@ -26,13 +26,13 @@ import monasca_analytics.banana.typeck.config as typeck import monasca_analytics.exception.banana as exception -def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()): +def execute_banana_string(banana, driver=None, emitter=emit.PrintEmitter()): """ Execute the provided banana string. It will run the parse phase, and the typechecker. - :type banana_str: str - :param banana_str: The string to parse and type check. - :type driver: monasca_analytics.spark.driver.DriverExecutor + :type banana: str + :param banana: The string to parse and type check. + :type driver: monasca_analytics.spark.driver.DriverExecutor | None :param driver: Driver that will manage the created components and connect them together. :type emitter: emit.Emitter @@ -41,7 +41,7 @@ def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()): try: # Convert the grammar into an AST parser = grammar.banana_grammar(emitter) - ast = parser.parse(banana_str) + ast = parser.parse(banana) # Compute the type table for the given AST type_table = typeck.typeck(ast) # Remove from the tree path that are "dead" @@ -49,30 +49,56 @@ def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()): # Check that there's at least one path to be executed deadpathck.contains_at_least_one_path_to_a_sink(ast, type_table) # Evaluate the script - ev.eval_ast(ast, type_table, driver) + if driver is not None: + ev.eval_ast(ast, type_table, driver) except exception.BananaException as err: emitter.emit_error(err.get_span(), str(err)) except p.ParseSyntaxException as err: - emitter.emit_error(span_util.from_parse_fatal(err), err.msg) + emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg) except p.ParseFatalException as err: - emitter.emit_error(span_util.from_parse_fatal(err), err.msg) + emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg) + except p.ParseException as err: + emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg) -def compute_type_table(banana_str): +def try_compute_type_table(banana): + """ + Compute the type table for the provided banana string + if possible. Does not throw any exception if it fails. + :type banana: str + :param banana: The string to parse and type check. + """ + try: + # Convert the grammar into an AST + parser = grammar.banana_grammar() + ast = parser.parse(banana) + # Compute the type table for the given AST + return typeck.typeck(ast) + except exception.BananaException: + return None + except p.ParseSyntaxException: + return None + except p.ParseFatalException: + return None + except p.ParseException: + return None + + +def compute_type_table(banana): """ Compute the type table for the provided banana string if possible. - :type banana_str: str - :param banana_str: The string to parse and type check. + :type banana: str + :param banana: The string to parse and type check. """ # Convert the grammar into an AST parser = grammar.banana_grammar() - ast = parser.parse(banana_str) + ast = parser.parse(banana) # Compute the type table for the given AST return typeck.typeck(ast) -def compute_evaluation_context(banana_str, cb=lambda *a, **k: None): +def compute_evaluation_context(banana, cb=lambda *a, **k: None): """ Compute the evaluation context for the provided banana string. @@ -81,7 +107,7 @@ def compute_evaluation_context(banana_str, cb=lambda *a, **k: None): :param cb: Callback called after each statement """ parser = grammar.banana_grammar() - ast = parser.parse(banana_str) + ast = parser.parse(banana) type_table = typeck.typeck(ast) context = ctx.EvaluationContext() diff --git a/monasca_analytics/banana/typeck/type_table.py b/monasca_analytics/banana/typeck/type_table.py index 55a756a..37bcd02 100644 --- a/monasca_analytics/banana/typeck/type_table.py +++ b/monasca_analytics/banana/typeck/type_table.py @@ -164,6 +164,19 @@ class TypeTable(object): )) self._variables = new_snapshot + def to_json(self): + """ + Convert this type table into a dictionary. + Useful to serialize the type table. + + :rtype: dict + :return: Returns this type table as a dict. + """ + res = {} + for key, val in self._variables.iteritems(): + res[key.inner_val()] = val.to_json() + return res + def __contains__(self, key): """ Test if the type table contains or not the provided diff --git a/monasca_analytics/banana/typeck/type_util.py b/monasca_analytics/banana/typeck/type_util.py index b5e6444..0121726 100644 --- a/monasca_analytics/banana/typeck/type_util.py +++ b/monasca_analytics/banana/typeck/type_util.py @@ -59,6 +59,10 @@ class IsType(object): def default_value(self): pass + @abc.abstractmethod + def to_json(self): + pass + class Any(IsType): """ @@ -84,6 +88,9 @@ class Any(IsType): def default_value(self): return {} + def to_json(self): + return {"id": "any"} + class String(IsType): """ @@ -102,6 +109,9 @@ class String(IsType): def default_value(self): return "" + def to_json(self): + return {"id": "string"} + class Number(String): """ @@ -120,6 +130,9 @@ class Number(String): def default_value(self): return 0 + def to_json(self): + return {"id": "number"} + class Enum(String): """ @@ -142,6 +155,12 @@ class Enum(String): def default_value(self): return "" + def to_json(self): + return { + "id": "enum", + "variants": self.variants + } + def attach_to_root(root_obj, obj1, span, erase_existing=False): """ @@ -281,6 +300,12 @@ class Object(String): default_value[key] = val.default_value() return default_value + def to_json(self): + res = {"id": "object", "props": {}} + for key, val in self.props.iteritems(): + res["props"][key] = val.to_json() + return res + class Component(IsType): """ @@ -397,7 +422,13 @@ class Component(IsType): return hash(str(self)) def default_value(self): - return {} + return None + + def to_json(self): + res = {"id": "component", "name": self.class_name, "args": []} + for arg in self.ctor_properties: + res["args"].append(arg.to_json()) + return res class Source(Component): diff --git a/monasca_analytics/component/params.py b/monasca_analytics/component/params.py index b445f7d..3c7a820 100644 --- a/monasca_analytics/component/params.py +++ b/monasca_analytics/component/params.py @@ -45,3 +45,10 @@ class ParamDescriptor(object): self.default_value = default self.param_type = _type self.validator = validator + + def to_json(self): + return { + "name": self.param_name, + "default_value": self.default_value, + "type": self.param_type.to_json(), + } diff --git a/monasca_analytics/monanas.py b/monasca_analytics/monanas.py index 2ddbf1f..2e6b1e4 100644 --- a/monasca_analytics/monanas.py +++ b/monasca_analytics/monanas.py @@ -76,6 +76,29 @@ class Monanas(object): # Try to change the configuration. executor.execute_banana_string(banana_str, self._driver, emitter) + def typeck_configuration(self, banana_str, emitter): + """Only type check the provided configuration. + + :type banana_str: str + :param banana_str: New configuration. + :type emitter: emit.JsonEmitter + :param emitter: a Json emitter instance + """ + executor.execute_banana_string(banana_str, None, emitter) + + def compute_type_table(self, banana_str): + """Compute the type table for the provided configuration. + + :type banana_str: str + :param banana_str: Configuration to test. + :rtype: dict + :return: Returns the type table + """ + type_table = executor.try_compute_type_table(banana_str) + if type_table is not None: + return type_table.to_json() + return {} + def start_streaming(self): """Starts streaming data. diff --git a/monasca_analytics/web_service/request_handler.py b/monasca_analytics/web_service/request_handler.py index 175ae60..18d6e4f 100644 --- a/monasca_analytics/web_service/request_handler.py +++ b/monasca_analytics/web_service/request_handler.py @@ -24,6 +24,7 @@ import voluptuous import monasca_analytics.banana.emitter as emit import monasca_analytics.exception.monanas as err +import monasca_analytics.util.common_util as introspect from monasca_analytics.web_service import web_service_model @@ -77,12 +78,13 @@ class BananaHandler(web.RequestHandler): the banana configuration language. """ - def initialize(self, monanas): + def initialize(self, monanas, typeck_only): """Initialize the handler. :param monanas: A Monana's instance. """ self._monanas = monanas + self._typeck_only = typeck_only @web.asynchronous def post(self): @@ -93,18 +95,15 @@ class BananaHandler(web.RequestHandler): body = json.loads(self.request.body) web_service_model.banana_model(body) emitter = emit.JsonEmitter() - # TODO(Joan): Change that - self._monanas.try_change_configuration(body["content"], emitter) + if self._typeck_only: + self._monanas.typeck_configuration(body["content"], + emitter) + else: + self._monanas.try_change_configuration(body["content"], + emitter) self.write(emitter.result) - except (AttributeError, voluptuous.Invalid, ValueError): + except (AttributeError, voluptuous.Invalid, ValueError) as e: self.set_status(400, "The request body was malformed.") - except (err.MonanasBindSourcesError, - err.MonanasAlreadyStartedStreaming, - err.MonanasAlreadyStoppedStreaming) as e: - self.set_status(400, e.__str__()) - except err.MonanasStreamingError as e: - self.set_status(500, e.__str__()) - terminate = (True, e.__str__()) except Exception as e: tb = traceback.format_exc() print(tb) @@ -118,3 +117,51 @@ class BananaHandler(web.RequestHandler): if terminate[0]: logger.error(terminate[1]) self._monanas.stop_streaming_and_terminate() + + +class BananaMetaDataHandler(web.RequestHandler): + + def initialize(self, monanas): + """Initializes the handler. + + :param monanas: Monanas -- A Monanas's instance. + """ + self._monanas = monanas + + @web.asynchronous + def get(self): + all_components = introspect.get_available_classes() + result = {"components": []} + for kind, components in all_components.iteritems(): + for component in components: + result["components"].append({ + "name": component.__name__, + "description": component.__doc__, + "params": map(lambda x: x.to_json(), + component.get_params()), + }) + self.write(result) + self.flush() + self.finish() + + @web.asynchronous + def post(self): + + try: + body = json.loads(self.request.body) + web_service_model.banana_model(body) + type_table = self._monanas.compute_type_table(body["content"]) + self.write(type_table) + except (AttributeError, voluptuous.Invalid, ValueError) as e: + logger.warn("Wrong request: {}.". + format(e)) + self.set_status(400, "The request body was malformed.") + except Exception as e: + tb = traceback.format_exc() + print(tb) + logger.error("Unexpected error: {}. {}". + format(sys.exc_info()[0], e)) + self.set_status(500, "Internal server error.") + + self.flush() + self.finish() diff --git a/monasca_analytics/web_service/web_service.py b/monasca_analytics/web_service/web_service.py index e9947df..9de9c6b 100644 --- a/monasca_analytics/web_service/web_service.py +++ b/monasca_analytics/web_service/web_service.py @@ -29,7 +29,17 @@ class WebService(web.Application): params = {"monanas": self._monanas} handlers = [ (r"/", request_handler.MonanasHandler, params), - (r"/banana", request_handler.BananaHandler, params), + (r"/banana", request_handler.BananaHandler, { + "monanas": self._monanas, + "typeck_only": False + }), + (r"/banana/typeck", request_handler.BananaHandler, { + "monanas": self._monanas, + "typeck_only": True + }), + (r"/banana/metadata", request_handler.BananaMetaDataHandler, { + "monanas": self._monanas, + }) ] settings = {}