diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 7cb168c..0000000 --- a/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -*~ -*.egg-info -*.swp -*.log -*.pyc -*.pyo -.eggs -.project -.pydevproject -.vagrant -.idea -.tox -.coverage -cover diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 67db858..0000000 --- a/LICENSE +++ /dev/null @@ -1,175 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. diff --git a/Makefile b/Makefile deleted file mode 100644 index abeca03..0000000 --- a/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -ifeq (testspec,$(firstword $(MAKECMDGOALS))) - # use the rest as arguments for "run" - TEST_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) - # ...and turn them into do-nothing targets - $(eval $(TEST_ARGS):;@:) -endif - -MONANAS_SRC=monasca_analytics -PYTHON=python - -all: test style - -test: - $(PYTHON) -m unittest discover -v - -testspec: - $(PYTHON) -m unittest -v $(TEST_ARGS) - -clean: - find $(MONANAS_SRC) -type f -name '*.pyc' -exec rm {} + - -style: - find $(MONANAS_SRC) -type f -name '*.py' -exec pep8 --max-line-length 79 {} + - -start: - bash -c "sleep 7; curl -H \"Content-Type: application/json\" -d '{\"action\": \"start_streaming\"}' http://localhost:3000/" & - $(PYTHON) run.py -p ~/spark/spark-1.6.1 -c ./config/metric_experiments.json -l ./config/logging.json - - -.PHONY: all test clean style testspec diff --git a/README.md b/README.md deleted file mode 100644 index 802da43..0000000 --- a/README.md +++ /dev/null @@ -1,110 +0,0 @@ -Team and repository tags -======================== - -[![Team and repository tags](https://governance.openstack.org/tc/badges/monasca-analytics.svg)](https://governance.openstack.org/tc/reference/tags/index.html) - - - -# MoNanas - Monasca Analytics Framework - -![MoNanas Logo](doc/images/monanas-logo.png) - -## Overview - -Monasca Analytics (MoNanas) is a statistical/machine-learning -([SML](doc/design.md#sml)) [flow](doc/design.md#flow) composition -engine. Users can compose a sequence of algorithms to be executed by just -providing a description as an input to MoNanas. The data flow is automatically -handled by the framework. - -Easy [flow](doc/design.md#flow) composition and reusability means that -we can speed up the extraction of actionable infrastructure insight. - -### Advantages -:thumbsup: Decouple algorithm design from execution. - -:thumbsup: Reusable specification of the desired [flow](doc/design.md#flow). - -:thumbsup: Language independent [flow](doc/design.md#flow) definition. - -:thumbsup: Data source and format independent. - -:thumbsup: Easy to add new [SML](doc/design.md#sml) algorithms and # combine them with pre-existing ones in the [flow](doc/design.md#flow). - -:thumbsup: Transparently exploit data parallelism. - -### Documentation -* [MoNanas/GettingStarted](doc/getting_started.md): A starting point for users -and developers of MoNanas. - -### Repositories -Core: https://github.com/openstack/monasca-analytics.git - -## Example Use Cases - -### Integrate with Monasca -See: [MoNanas/IntegrateMonasca](doc/monasca.md) for more details integration with Monasca - -### Relevant to OpenStack -See: [MoNanas/UseCases](doc/use_cases.md) for more details use cases that are relevant to OpenStack. - -## MoNanas Design -See: [MoNanas/Design](doc/design.md) for details on MoNanas's architecture, -its functional requirements and core concepts. - -## Technologies -MoNanas uses a number of third-party technologies: -* Apache Spark (https://spark.apache.org/): Apache Spark is a fast and general -engine for large-scale data processing. -* Apache Kafka (https://kafka.apache.org/): Used by Monasca and MoNanas's Kafka -`source` and `sink`. -* Apache ZooKeeper (https://zookeeper.apache.org/): Used by Kafka. - -## Feature Release Schedule -- [x] Basic SML flow. -- [x] New algorithm "add-on" ability. -- [x] Example datasets and SML flows. -- [ ] Support end-to-end learning + data processing flows (currently, the latter part does not get updated due to Spark's immutability.) -- [ ] Refactor codes to be consistent with terms used in the documentation. -- [ ] Add a source, ingestor and transformer for Monasca. -- [ ] Model connections as objects rather than references and have driver specifics in one place. -- [ ] Expanded orchestration abilities/expressiveness. -- [ ] Container-enabled testing/deployment for non-production environments. -- [ ] Add Vitrage Sink. -- [ ] Add a ready-to-use virtual machine image (get rid of the fetch-deps.sh). - -## Contributing -There are multiple ways to contribute to the project. All are equally important -to us! -* You can have a look at the -[Monasca launchpad](https://launchpad.net/monasca) for problems that -needs to be solved (bugs/issues), and blueprints. -* You can also help us to add -[new learning algorithms](doc/dev_guide.md#add_new_algorithms). -* Finally, we are very interested in having more data sources to experiment -with. The source can either be from an existing data provider or randomly -generated. The more, the better! :) If you are interested to work on that -aspect, [you are welcome as well](doc/dev_guide.md#add_new_sources). - -For more information on setting up your development environment, see -[MoNanas/DevGuide](doc/dev_guide.md). - -For more information about Monanas, please visit the wiki page: -[Monanas wiki](https://wiki.openstack.org/wiki/Monasca/Analytics). - -And for more information about Monasca, please visit the wiki page: -[Monasca wiki](https://wiki.openstack.org/wiki/Monasca). - -## License -Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -Licensed under the Apache License, Version 2.0 (the "License"); you may not -used this file except in compliance with the License. You may obtain a copy of -the License at: -``` -http://www.apache.org/licenses/LICENSE-2.0 -``` -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations under -the License. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..46d1993 --- /dev/null +++ b/README.rst @@ -0,0 +1,10 @@ +This project is no longer maintained. + +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". + +For any further questions, please email +openstack-discuss@lists.openstack.org or join #openstack-monasca on +Freenode. \ No newline at end of file diff --git a/SUMMARY.md b/SUMMARY.md deleted file mode 100644 index 60337c3..0000000 --- a/SUMMARY.md +++ /dev/null @@ -1,7 +0,0 @@ -# Summary - -* [MoNanas](README.md) - * [Design](doc/design.md) - * [Getting Started](doc/getting_started.md) -* [Developer Guide](doc/dev_guide.md) -* [Examples](doc/examples.md) diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index ad63651..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,43 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# All Vagrant configuration is done below. The "2" in Vagrant.configure -# configures the configuration version (we support older styles for -# backwards compatibility). Please don't change it unless you know what -# you're doing. -Vagrant.configure(2) do |config| - # The most common configuration options are documented and commented below. - # For a complete reference, please see the online documentation at - # https://docs.vagrantup.com. - - # Default box - config.vm.box = "ubuntu/trusty64" - - # Setup port for Machine Learning Framework - config.vm.network "forwarded_port", guest: 3000, host: 3000 - config.vm.network "forwarded_port", guest: 4040, host: 4040 - - # Shell provisioning - config.vm.provision "shell", - inline: "echo 'export VAGRANT_ENV=1' >> /home/vagrant/.profile" - - config.vm.provision "shell" do |s| - # s.env is only available since Vagrant 1.8 - s.path = "fetch-deps.sh" - s.privileged = false - end - - # Virtual box settings - config.vm.provider "virtualbox" do |v| - v.memory = 4096 - v.cpus = 3 - end - - # Proxy settings - if Vagrant.has_plugin?("vagrant-proxyconf") - config.proxy.http = ENV['HTTP_PROXY'] || ENV['http_proxy'] - config.proxy.https = ENV['HTTPS_PROXY'] || ENV['https_proxy'] - config.proxy.no_proxy = "localhost,127.0.0.1" - end - -end diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/config/config_template.json b/config/config_template.json deleted file mode 100644 index 6f9da55..0000000 --- a/config/config_template.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "id": "my_id", - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "my_src_module" - } - }, - "ingestors": { - "ing1": { - "module": "my_ingestor_module" - } - }, - "smls": { - "sml1": { - "module": "my_sml_module" - } - }, - "voters": { - "vot1": { - "module": "my_voter_module" - } - }, - "sinks": { - "snk1": { - "module": "my_sink_module" - } - }, - "ldps": { - "ldp1": { - "module": "my_ldp_module" - } - }, - "connections": { - "src1": ["ing1", "ldp1"], - "sml1": ["vot1"], - "ing1": [], - "vot1": ["ldp1"], - "ldp1": ["snk1"], - "snk1": [] - }, - "feedback": {} -} diff --git a/config/iptables_anomalies.banana b/config/iptables_anomalies.banana deleted file mode 100644 index 1677f5f..0000000 --- a/config/iptables_anomalies.banana +++ /dev/null @@ -1,16 +0,0 @@ -############################## -# IP Tables anomalies -# - -sleep = 0.01 -src = IPTablesSource(sleep=sleep) -ing1 = IptablesIngestor() -svm = SvmOneClass() -voter = PickIndexVoter(0) -ldp1 = IptablesLDP() -stdout = StdoutSink() -sqlite = IptablesSQLiteSink() - -src -> [ing1, ldp1] -ing1 -> svm -> voter -> ldp1 -ldp1 -> [sqlite, stdout] diff --git a/config/iptables_anomalies.json b/config/iptables_anomalies.json deleted file mode 100644 index 5170fdd..0000000 --- a/config/iptables_anomalies.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "IPTablesSource", - "sleep": 0.01 - } - }, - "ingestors": { - "ing1": { - "module": "IptablesIngestor" - } - }, - "smls": { - "sml1": { - "module": "SvmOneClass", - "nb_samples": 500 - } - }, - "voters": { - "vot1": { - "module": "PickIndexVoter", - "index": 0 - } - }, - "sinks": { - "snk1": { - "module": "IptablesSQLiteSink" - }, - "snk2": { - "module": "StdoutSink" - } - }, - "ldps": { - "ldp1": { - "module": "IptablesLDP" - } - }, - "connections": { - "src1": ["ing1", "ldp1"], - "ing1": [], - "sml1": ["vot1"], - "vot1": ["ldp1"], - "ldp1": ["snk1", "snk2"], - "snk1": [], - "snk2": [] - }, - "feedback": {} -} diff --git a/config/logging.json b/config/logging.json deleted file mode 100644 index 47ccb3c..0000000 --- a/config/logging.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "standard": { - "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s" - } - }, - "handlers": { - "default": { - "level": "INFO", - "class": "logging.StreamHandler", - "formatter": "standard" - } - }, - "loggers": { - "": { - "handlers": ["default"], - "level": "INFO", - "propagate": true - } - } -} diff --git a/config/markov_source_config.banana b/config/markov_source_config.banana deleted file mode 100644 index 2c1ec98..0000000 --- a/config/markov_source_config.banana +++ /dev/null @@ -1,57 +0,0 @@ -############################## -# Markov source config example -# (cloud-like data model) -# - -src = CloudMarkovChainSource(sleep=0.01, min_event_per_burst=500) - -src.transitions.web_service = { - "run=>slow": { - "0": 0.001, - "8": 0.02, - "12": 0.07, - "14": 0.07, - "22": 0.03, - "24": 0.00 - }, - "slow=>run": { - "0": 0.99, - "8": 0.7, - "12": 0.1, - "14": 0.1, - "22": 0.8, - "24": 0.9 - }, - "stop=>run": 0.7 -} - -src.transitions.host = { - "on=>off": 0.005, - "off=>on": 0.5 -} - -src.transitions.switch = { - "on=>off": 0.01, - "off=>on": 0.7 -} - -src.triggers.support = { - "get_called" : { - "0": 0.1, - "8": 0.2, - "12": 0.8, - "14": 0.8, - "22": 0.5, - "24": 0.0 - } -} - -ing1 = CloudIngestor() -ling = LiNGAM(threshold=0.5) -voter = PickIndexVoter(0) -sink = KafkaSink(host="localhost", port=9092, topic="transformed_alerts") -ldp = CloudCausalityLDP() - -# Connections -src -> [ing1 -> ling, ldp] -ling -> voter -> ldp -> sink diff --git a/config/markov_source_config.json b/config/markov_source_config.json deleted file mode 100644 index 8e64fa3..0000000 --- a/config/markov_source_config.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "CloudMarkovChainSource", - "sleep": 0.01, - "min_event_per_burst": 500, - "transitions": { - "web_service": { - "run=>slow": { - "0": 0.001, - "8": 0.02, - "12": 0.07, - "14": 0.07, - "22": 0.03, - "24": 0.001 - }, - "slow=>run": { - "0": 0.99, - "8": 0.7, - "12": 0.1, - "14": 0.1, - "22": 0.8, - "24": 0.99 - }, - "stop=>run": 0.7 - }, - "host": { - "on=>off": 0.005, - "off=>on": 0.5 - }, - "switch": { - "on=>off": 0.01, - "off=>on": 0.7 - } - }, - "triggers": { - "support": { - "get_called": { - "0": 0.1, - "8": 0.2, - "12": 0.8, - "14": 0.8, - "22": 0.5, - "24": 0.0 - } - } - }, - "graph": { - "h1:host": ["s1"], - "h2:host": ["s1"], - "s1:switch": [], - "w1:web_service": ["h1"], - "w2:web_service": ["h2"] - } - - } - }, - "ingestors": { - "ing1": { - "module": "CloudIngestor" - } - }, - "smls": { - "sml1": { - "module": "LiNGAM", - "threshold": 0.5 - } - }, - "voters": { - "vot1": { - "module": "PickIndexVoter", - "index": 0 - } - }, - "sinks": { - "snk1": { - "module": "KafkaSink", - "host": "localhost", - "port": 9092, - "topic": "transformed_alerts" - } - }, - "ldps": { - "ldp1": { - "module": "CloudCausalityLDP" - } - }, - "connections": { - "src1": ["ing1", "ldp1"], - "sml1": ["vot1"], - "ing1": [], - "vot1": ["ldp1"], - "ldp1": ["snk1"], - "snk1": [] - }, - "feedback": {} -} \ No newline at end of file diff --git a/config/metric_experiments.banana b/config/metric_experiments.banana deleted file mode 100644 index fa34e9c..0000000 --- a/config/metric_experiments.banana +++ /dev/null @@ -1,30 +0,0 @@ -####################### -# Metric experiments -# - -# Sources -src = MonascaMarkovChainSource(sleep=0.01) - -# Sinks -stdout = StdoutSink() -file = FileSink(path="~/monasca-aggregate.log") - -# Live data processors -period = 0.1 * 2 -aggregator = MonascaAggregateLDP(func="cnt", period=period) -combiner = MonascaCombineLDP( - metric= "cpu.logical_cores_actives", - bindings= { - a: "cpu.idle_perc", - b: "cpu.total_logical_cores", - }, - lambda= "a * b", - period= period -) -derivator = MonascaDerivativeLDP(period=period) - - -# Connections -src -> aggregator -> stdout -src -> [combiner, derivator] -> stdout -[combiner, derivator] -> file diff --git a/config/metric_experiments.json b/config/metric_experiments.json deleted file mode 100644 index 63c564d..0000000 --- a/config/metric_experiments.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "MonascaMarkovChainSource", - "sleep": 0.01 - } - }, - "ingestors": {}, - "smls": {}, - "voters": {}, - "sinks": { - "snk2": { - "module": "StdoutSink" - }, - "snk3": { - "module": "FileSink", - "path": "~/monasca-aggregate.log" - } - }, - "ldps": { - "ldp3": { - "module": "MonascaAggregateLDP", - "period": 2, - "func": "cnt" - }, - "ldp4": { - "module": "MonascaCombineLDP", - "metric": "cpu.logical_cores_actives", - "period": 1, - "lambda": "a * b", - "bindings": { - "a": "cpu.idle_perc", - "b": "cpu.total_logical_cores" - } - }, - "ldp5": { - "module": "MonascaDerivativeLDP", - "period": 1 - } - }, - "connections": { - "src1": ["ldp5"], - "ldp3": [], - "ldp4": [], - "ldp5": ["snk2"], - "snk2": [], - "snk3": [] - }, - "feedback": {} -} diff --git a/config/monasca_aggregate_all_fn.banana b/config/monasca_aggregate_all_fn.banana deleted file mode 100644 index 9f4303a..0000000 --- a/config/monasca_aggregate_all_fn.banana +++ /dev/null @@ -1,13 +0,0 @@ -############################## -# Monasca aggregate all functions - -src = MonascaMarkovChainSource(sleep=0.01) -snk = KafkaSink(host="localhost", port=9092, topic="experiments") -stdout = StdoutSink() - -ldp1 = MonascaAggregateLDP(func="cnt") -ldp2 = MonascaAggregateLDP(func="max") -ldp3 = MonascaAggregateLDP(func="min") -ldp4 = MonascaAggregateLDP(func="avg") - -src -> [ldp1, ldp2, ldp3, ldp4] -> [snk, stdout] diff --git a/config/monasca_aggregate_all_fn.json b/config/monasca_aggregate_all_fn.json deleted file mode 100644 index c81754f..0000000 --- a/config/monasca_aggregate_all_fn.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "MonascaMarkovChainSource", - "sleep": 0.01 - } - }, - "ingestors": {}, - "smls": {}, - "voters": {}, - "sinks": { - "snk1": { - "module": "KafkaSink", - "host": "localhost", - "port": 9092, - "topic": "monasca_experiments" - }, - "snk2": { - "module": "StdoutSink" - } - }, - "ldps": { - "ldp1": { - "module": "MonascaAggregateLDP", - "period": 2, - "func": "max" - }, - "ldp2": { - "module": "MonascaAggregateLDP", - "period": 2, - "func": "min" - }, - "ldp3": { - "module": "MonascaAggregateLDP", - "period": 2, - "func": "avg" - }, - "ldp4": { - "module": "MonascaAggregateLDP", - "period": 2, - "func": "sum" - } - }, - "connections": { - "src1": ["ldp1", "ldp2", "ldp3", "ldp4"], - "ldp1": ["snk2"], - "ldp2": ["snk2"], - "ldp3": ["snk2"], - "ldp4": ["snk2"], - "snk1": [], - "snk2": [] - }, - "feedback": {} -} diff --git a/config_dsl.py b/config_dsl.py deleted file mode 100644 index b3efea7..0000000 --- a/config_dsl.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import os -import six -from logging import config as log_conf - -from monasca_analytics.banana.cli import interpreter - -DEFAULT_LOGGING_CONFIG_FILE = "config/logging.json" - - -def setup_logging(): - current_dir = os.path.dirname(__file__) - logging_config_file = os.path.join(current_dir, - DEFAULT_LOGGING_CONFIG_FILE) - with open(logging_config_file, "rt") as f: - config = json.load(f) - log_conf.dictConfig(config) - - -def main(): - setup_logging() - print(">>>>> DEPRECATED TOOL <<<<<") - print(">>>>> Use the banana language instead <<<<<") - print("") - print("Welcome to Monanas config command line") - print("Type help for help about commands") - inter = interpreter.DSLInterpreter() - cmd = "" - while "exit" != cmd.lower(): - cmd = six.moves.input(">> ") - if cmd != "": - try: - print(inter.execute_string(cmd)) - except Exception as e: - print("Failed : " + str(e)) - -if __name__ == "__main__": - main() diff --git a/devstack/README.md b/devstack/README.md deleted file mode 100644 index 5f15e63..0000000 --- a/devstack/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# Monasca Analytics DevStack Plugin - -The Monasca Analytics DevStack plugin currently only works on Ubuntu 16.04 (Xenial). -More Linux Distributions will be supported in the future. - -Running the Monasca Analytics DevStack plugin requires a machine with 8GB of RAM. - -Directions for installing and running Devstack can be found here: - - http://docs.openstack.org/developer/devstack/ - -To run Monasca Analytics in DevStack, do the following three steps. - -1. Clone the DevStack repo. - -``` -git clone https://opendev.org/openstack/devstack -``` - -2. Add the following to the DevStack local.conf file in the root of the devstack directory. You may - need to create the local.conf if it does not already exist. - -# BEGIN DEVSTACK LOCAL.CONF CONTENTS - -``` -[[local|localrc]] -MYSQL_PASSWORD=secretmysql -DATABASE_PASSWORD=secretdatabase -RABBIT_PASSWORD=secretrabbit -ADMIN_PASSWORD=secretadmin -SERVICE_PASSWORD=secretservice -SERVICE_TOKEN=111222333444 - -LOGFILE=$DEST/logs/stack.sh.log -LOGDIR=$DEST/logs -LOG_COLOR=False - -# This line will enable all of Monasca Analytics. -enable_plugin monasca-analytics https://opendev.org/openstack/monasca-analytics -``` - -# END DEVSTACK LOCAL.CONF CONTENTS - -3. Run './stack.sh' from the root of the devstack directory. - -To run Monasca Analytics with the bare mininum of OpenStack components you can -add the following two lines to the local.conf file. - -``` -disable_all_services -enable_service rabbit mysql key -``` - -# License - -``` -# (C) Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -``` diff --git a/devstack/files/kafka/kafka-server-start.sh b/devstack/files/kafka/kafka-server-start.sh deleted file mode 100644 index 5785bbd..0000000 --- a/devstack/files/kafka/kafka-server-start.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -if [ $# -lt 1 ]; -then - echo "USAGE: $0 [-daemon] server.properties" - exit 1 -fi -base_dir=$(dirname $0) -export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" -export KAFKA_HEAP_OPTS="-Xms256m -Xmx256m" - -EXTRA_ARGS="-name kafkaServer -loggc" - -COMMAND=$1 -case $COMMAND in - -daemon) - EXTRA_ARGS="-daemon "$EXTRA_ARGS - shift - ;; - *) - ;; -esac - -exec $base_dir/kafka-run-class.sh $EXTRA_ARGS kafka.Kafka $@ diff --git a/devstack/files/kafka/kafka.conf b/devstack/files/kafka/kafka.conf deleted file mode 100644 index 005e77c..0000000 --- a/devstack/files/kafka/kafka.conf +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -description "Kafka" - -start on runlevel [2345] -stop on runlevel [!2345] - -respawn - -limit nofile 32768 32768 - -# If zookeeper is running on this box also give it time to start up properly -pre-start script - if [ -e /etc/init.d/zookeeper ]; then - /etc/init.d/zookeeper start || true - fi -end script - -# Rather than using setuid/setgid sudo is used because the pre-start task must run as root -exec sudo -Hu kafka -g kafka KAFKA_HEAP_OPTS="-Xmx128m" /opt/kafka/bin/kafka-server-start.sh /etc/kafka/server.properties diff --git a/devstack/files/kafka/kafka.service b/devstack/files/kafka/kafka.service deleted file mode 100644 index c03d2af..0000000 --- a/devstack/files/kafka/kafka.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Apache Kafka -Requires=network.target -After=zookeepepr.service - -[Service] -Type=simple -ExecStart=/opt/kafka/bin/kafka-server-start.sh /etc/kafka/server.properties -ExecStop=/opt/kafka/bin/kafka-server-stop.sh - -[Install] -WantedBy=multi-user.target diff --git a/devstack/files/kafka/log4j.properties b/devstack/files/kafka/log4j.properties deleted file mode 100644 index b3cff00..0000000 --- a/devstack/files/kafka/log4j.properties +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -log4j.rootLogger=WARN, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender -log4j.appender.kafkaAppender.MaxFileSize=50MB -log4j.appender.kafkaAppender.MaxBackupIndex=4 -log4j.appender.kafkaAppender.File=/var/log/kafka/server.log -log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender -log4j.appender.stateChangeAppender.MaxFileSize=50MB -log4j.appender.stateChangeAppender.MaxBackupIndex=4 -log4j.appender.stateChangeAppender.File=/var/log/kafka/state-change.log -log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender -log4j.appender.controllerAppender.MaxFileSize=50MB -log4j.appender.controllerAppender.MaxBackupIndex=4 -log4j.appender.controllerAppender.File=/var/log/kafka/controller.log -log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -# Turn on all our debugging info -#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender -#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender -#log4j.logger.kafka.perf=DEBUG, kafkaAppender -#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender -#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG -log4j.logger.kafka=WARN, kafkaAppender - -# Tracing requests results in large logs -#log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender -#log4j.appender.requestAppender.MaxFileSize=50MB -#log4j.appender.requestAppender.MaxBackupIndex=4 -#log4j.appender.requestAppender.File=/var/log/kafka/kafka-request.log -#log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout -#log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n -# -#log4j.logger.kafka.network.RequestChannel$=TRACE, requestAppender -#log4j.additivity.kafka.network.RequestChannel$=false -# -#log4j.logger.kafka.network.Processor=TRACE, requestAppender -#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender -#log4j.additivity.kafka.server.KafkaApis=false -#log4j.logger.kafka.request.logger=TRACE, requestAppender -#log4j.additivity.kafka.request.logger=false - -log4j.logger.kafka.controller=TRACE, controllerAppender -log4j.additivity.kafka.controller=false - -log4j.logger.state.change.logger=TRACE, stateChangeAppender -log4j.additivity.state.change.logger=false - diff --git a/devstack/files/kafka/server.properties b/devstack/files/kafka/server.properties deleted file mode 100644 index 15787c0..0000000 --- a/devstack/files/kafka/server.properties +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################# Server Basics ############################# - -# The id of the broker. This must be set to a unique integer for each broker. -broker.id=0 - -############################# Socket Server Settings ############################# - -# The port the socket server listens on -port=9092 - -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -host.name=127.0.0.1 - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= - -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= - -# The number of threads handling network requests -num.network.threads=2 - -# The number of threads doing disk I/O -num.io.threads=2 - -# The send buffer (SO_SNDBUF) used by the socket server -socket.send.buffer.bytes=1048576 - -# The receive buffer (SO_RCVBUF) used by the socket server -socket.receive.buffer.bytes=1048576 - -# The maximum size of a request that the socket server will accept (protection against OOM) -socket.request.max.bytes=104857600 - - -############################# Log Basics ############################# - -# A comma separated list of directories under which to store log files -log.dirs=/var/kafka - -auto.create.topics.enable=false -# The number of logical partitions per topic per server. More partitions allow greater parallelism -# for consumption, but also mean more files. -num.partitions=2 - -############################# Log Flush Policy ############################# - -# Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. -# There are a few important trade-offs here: -# 1. Durability: Unflushed data may be lost if you are not using replication. -# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. -# The settings below allow one to configure the flush policy to flush data after a period of time or -# every N messages (or both). This can be done globally and overridden on a per-topic basis. - -# The number of messages to accept before forcing a flush of data to disk -log.flush.interval.messages=10000 - -# The maximum amount of time a message can sit in a log before we force a flush -log.flush.interval.ms=1000 - -############################# Log Retention Policy ############################# - -# The following configurations control the disposal of log segments. The policy can -# be set to delete segments after a period of time, or after a given size has accumulated. -# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens -# from the end of the log. - -# The minimum age of a log file to be eligible for deletion -log.retention.hours=24 - -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. -log.retention.bytes=104857600 - -# The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=104857600 - -# The interval at which log segments are checked to see if they can be deleted according -# to the retention policies -log.retention.check.interval.ms=60000 - -# By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires. -# If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction. -log.cleaner.enable=false - -############################# Zookeeper ############################# - -# Zookeeper connection string (see zookeeper docs for details). -# This is a comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". -# You can also append an optional chroot string to the urls to specify the -# root directory for all kafka znodes. -zookeeper.connect=127.0.0.1:2181 - -# Timeout in ms for connecting to zookeeper -zookeeper.connection.timeout.ms=1000000 - diff --git a/devstack/files/maven/settings.xml b/devstack/files/maven/settings.xml deleted file mode 100644 index d65b036..0000000 --- a/devstack/files/maven/settings.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - true - http - - - - 8080 - localhost - - - diff --git a/devstack/files/zookeeper/environment b/devstack/files/zookeeper/environment deleted file mode 100644 index 3d5cde7..0000000 --- a/devstack/files/zookeeper/environment +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Modified from http://packages.ubuntu.com/saucy/zookeeperd -NAME=zookeeper -ZOOCFGDIR=/etc/zookeeper/conf - -# seems, that log4j requires the log4j.properties file to be in the classpath -CLASSPATH="$ZOOCFGDIR:/usr/share/java/jline.jar:/usr/share/java/log4j-1.2.jar:/usr/share/java/xercesImpl.jar:/usr/share/java/xmlParserAPIs.jar:/usr/share/java/netty.jar:/usr/share/java/slf4j-api.jar:/usr/share/java/slf4j-log4j12.jar:/usr/share/java/zookeeper.jar" - -ZOOCFG="$ZOOCFGDIR/zoo.cfg" -ZOO_LOG_DIR=/var/log/zookeeper -USER=$NAME -GROUP=$NAME -PIDDIR=/var/run/$NAME -PIDFILE=$PIDDIR/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -JAVA=/usr/bin/java -ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" -ZOO_LOG4J_PROP="INFO,ROLLINGFILE" -JMXLOCALONLY=false -JAVA_OPTS="" diff --git a/devstack/files/zookeeper/log4j.properties b/devstack/files/zookeeper/log4j.properties deleted file mode 100644 index 3fe8ae0..0000000 --- a/devstack/files/zookeeper/log4j.properties +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# From http://packages.ubuntu.com/saucy/zookeeperd - -# ZooKeeper Logging Configuration -# - -# Format is " (, )+ - -log4j.rootLogger=${zookeeper.root.logger} - -# Example: console appender only -# log4j.rootLogger=INFO, CONSOLE - -# Example with rolling log file -#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE - -# Example with rolling log file and tracing -#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE - -# -# Log INFO level and above messages to the console -# -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=INFO -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add ROLLINGFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender -log4j.appender.ROLLINGFILE.Threshold=WARN -log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/zookeeper.log - -# Max log file size of 10MB -log4j.appender.ROLLINGFILE.MaxFileSize=10MB -# uncomment the next line to limit number of backup files -#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 - -log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout -log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n - - -# -# Add TRACEFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.TRACEFILE=org.apache.log4j.FileAppender -log4j.appender.TRACEFILE.Threshold=TRACE -log4j.appender.TRACEFILE.File=${zookeeper.log.dir}/zookeeper_trace.log - -log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout -### Notice we are including log4j's NDC here (%x) -log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/devstack/files/zookeeper/myid b/devstack/files/zookeeper/myid deleted file mode 100644 index c227083..0000000 --- a/devstack/files/zookeeper/myid +++ /dev/null @@ -1 +0,0 @@ -0 \ No newline at end of file diff --git a/devstack/files/zookeeper/zoo.cfg b/devstack/files/zookeeper/zoo.cfg deleted file mode 100644 index 079bc33..0000000 --- a/devstack/files/zookeeper/zoo.cfg +++ /dev/null @@ -1,74 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html - -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=10 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=5 -# the directory where the snapshot is stored. -dataDir=/var/lib/zookeeper -# Place the dataLogDir to a separate physical disc for better performance -# dataLogDir=/disk2/zookeeper - -# the port at which the clients will connect -clientPort=2181 - -# Maximum number of clients that can connect from one client -maxClientCnxns=60 - -# specify all zookeeper servers -# The fist port is used by followers to connect to the leader -# The second one is used for leader election - -server.0=127.0.0.1:2888:3888 - -# To avoid seeks ZooKeeper allocates space in the transaction log file in -# blocks of preAllocSize kilobytes. The default block size is 64M. One reason -# for changing the size of the blocks is to reduce the block size if snapshots -# are taken more often. (Also, see snapCount). -#preAllocSize=65536 - -# Clients can submit requests faster than ZooKeeper can process them, -# especially if there are a lot of clients. To prevent ZooKeeper from running -# out of memory due to queued requests, ZooKeeper will throttle clients so that -# there is no more than globalOutstandingLimit outstanding requests in the -# system. The default limit is 1,000.ZooKeeper logs transactions to a -# transaction log. After snapCount transactions are written to a log file a -# snapshot is started and a new transaction log file is started. The default -# snapCount is 10,000. -#snapCount=1000 - -# If this option is defined, requests will be will logged to a trace file named -# traceFile.year.month.day. -#traceFile= - -# Leader accepts client connections. Default value is "yes". The leader machine -# coordinates updates. For higher update throughput at thes slight expense of -# read throughput the leader can be configured to not accept clients and focus -# on coordination. -#leaderServes=yes - -# Autopurge every hour to avoid using lots of disk in bursts -# Order of the next 2 properties matters. -# autopurge.snapRetainCount must be before autopurge.purgeInterval. -autopurge.snapRetainCount=3 -autopurge.purgeInterval=1 diff --git a/devstack/plugin.sh b/devstack/plugin.sh deleted file mode 100755 index 93dfe8c..0000000 --- a/devstack/plugin.sh +++ /dev/null @@ -1,362 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Monasca-analytics DevStack plugin -# -# Install and start Monasca-analytics service in devstack -# -# To enable Monasca-analytics in devstack add an entry to local.conf that -# looks like -# -# [[local|localrc]] -# enable_plugin monasca-analytics https://git.openstack.org/openstack/monasca-analytics -# -# By default all Monasca services are started (see -# devstack/settings). To disable a specific service use the -# disable_service function. For example to turn off notification: -# -# disable_service monasca-notification -# -# Several variables set in the localrc section adjust common behaviors -# of Monasca (see within for additional settings): -# -# EXAMPLE VARS HERE - -# Save trace setting -XTRACE=$(set +o | grep xtrace) -set -o xtrace - -ERREXIT=$(set +o | grep errexit) -set -o errexit - -# Determine if we are running in devstack-gate or devstack. -if [[ $DEST ]]; then - # We are running in devstack-gate. - export MONASCA_BASE=${MONASCA_BASE:-"${DEST}"} - -else - # We are running in devstack. - export MONASCA_BASE=${MONASCA_BASE:-"/opt/stack"} - -fi -export MONASCA_ANALYTICS_BASE=${MONASCA_ANALYTICS_BASE:-"${MONASCA_BASE}/monasca-analytics"} - -### -function pre_install_spark { -: -} - -### -function pre_install_monasca_analytics { -: -} - -### -function unstack_monasca_analytics { - echo_summary "Unstack Monasca-analytics" - - delete_monasca_analytics_files - - sudo userdel monasca-analytics || true - sudo groupdel monasca-analytics || true - - unstack_spark -} - -### -function delete_monasca_analytics_files { - sudo rm -rf /opt/monasca/analytics || true - sudo rm -rf /etc/monasca/analytics || true - sudo rm /etc/init/monasca-analytics.conf || true - - MONASCA_ANALYTICS_DIRECTORIES=("/var/log/monasca/analytics" "/var/run/monasca/analytics" "/etc/monasca/analytics/init") - - for MONASCA_ANALYTICS_DIRECTORY in "${MONASCA_ANALYTICS_DIRECTORIES[@]}" - do - sudo rm -rf ${MONASCA_ANALYTICS_DIRECTORY} || true - done -} - -### -function unstack_spark { - echo_summary "Unstack Spark" - - delete_spark_directories - sudo rm -rf /opt/spark/download || true -} - -### -function clean_monasca_analytics { - set +o errexit - unstack_monasca_analytics - unistall_pkgs - set -o errexit -} - -### -function delete_spark_directories { - for SPARK_DIRECTORY in "${SPARK_DIRECTORIES[@]}" - do - sudo rm -rf ${SPARK_DIRECTORY} || true - done - - sudo rm -rf /var/log/spark-events || true -} - -### -function unistall_pkgs { - sudo apt-get -y purge ipython python-scipy python-numpy - sudo apt-get -y purge python-setuptools - - sudo apt-get -y purge sbt - sudo apt-key del $KEYID - sudo sed -i -e '/deb https\:\/\/dl.bintray.com\/sbt\/debian \//d' /etc/apt/sources.list.d/sbt.list - sudo dpkg -r scala - - sudo apt-get -y purge $JDK_PKG - - sudo rm -rf ~/.m2 - - sudo rm -rf $SPARK_DIR -} - -### -function install_pkg { - ## JDK - sudo -E apt-get -y install $JDK_PKG - - ## SCALA - download_through_cache $SCALA_URL $SCALA $SPARK_DOWNLOAD - sudo -E dpkg -i $SPARK_DOWNLOAD/$SCALA - echo "deb https://dl.bintray.com/sbt/debian /" | sudo -E tee -a /etc/apt/sources.list.d/sbt.list - sudo -E apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv $KEYID - sudo -E apt-get update - sudo -E apt-get -y install sbt - - ## other pkg - sudo -E apt-get -y install python-setuptools - sudo -E apt-get -y install python-numpy python-scipy ipython -} - -### -function build_spark { - ## install maven - download_through_cache $MAVEN_URL $MAVEN_TARBALL $SPARK_DOWNLOAD - sudo chown stack:stack $SPARK_DOWNLOAD/$MAVEN_TARBALL - sudo -u stack -g stack tar -xzf $SPARK_DOWNLOAD/$MAVEN_TARBALL -C $SPARK_DIR - - if [ ${http_proxy} ];then - read HTTP_PROXY_USER_NAME HTTP_PROXY_PASSWORD HTTP_PROXY_HOST<< END -`echo ${http_proxy:7} | awk -F: '{sub("@", ":");print $1, $2, $3}'` -END - if [ -z $HTTP_PROXY_HOST ];then - LENGTH_FOR_HOST=`expr match "$http_proxy" 'http://[\.A-Za-z\-]*'`-7 - sed -e '7,8d' \ - -e "s/<\/host>/${http_proxy:7:$LENGTH_FOR_HOST}<\/host>/g" \ - ${MONASCA_ANALYTICS_BASE}/devstack/files/maven/settings.xml > ~/.m2/settings.xml - else - sed -e "s/<\/username>/${HTTP_PROXY_USER_NAME}<\/username>/g" \ - -e "s/<\/password>/${HTTP_PROXY_PASSWORD}<\/password>/g" \ - -e "s/<\/host>/${HTTP_PROXY_HOST}<\/host>/g" \ - ${MONASCA_ANALYTICS_BASE}/devstack/files/maven/settings.xml > ~/.m2/settings.xml - fi - fi - - ## Build Spark - download_through_cache $SPARK_URL ${SPARK_TARBALL_NAME} $SPARK_DOWNLOAD - sudo chown stack:stack $SPARK_DOWNLOAD/${SPARK_TARBALL_NAME} - sudo -u stack -g stack tar -xzf $SPARK_DOWNLOAD/${SPARK_TARBALL_NAME} -C $SPARK_DIR - - DEVSTACK_DIR=`pwd` - cd $SPARK_DIR/spark-${SPARK_VERSION} - $SPARK_DIR/$MAVEN/bin/mvn -DskipTests clean package - sudo cp -pf ./conf/log4j.properties.template ./conf/log4j.properties - sudo sed -i 's/log4j.rootCategory=INFO/log4j.rootCategory=ERROR/g' ./conf/log4j.properties - cd $DEVSTACK_DIR -} - -### -function install_zookeeper { - if [ ! -e /etc/init.d/zookeeper ]; then - echo_summary "Install Monasca Zookeeper" - - sudo apt-get -y install zookeeperd - - sudo cp "${MONASCA_ANALYTICS_BASE}"/devstack/files/zookeeper/myid /etc/zookeeper/conf/myid - sudo cp "${MONASCA_ANALYTICS_BASE}"/devstack/files/zookeeper/environment /etc/zookeeper/conf/environment - sudo cp "${MONASCA_ANALYTICS_BASE}"/devstack/files/zookeeper/log4j.properties /etc/zookeeper/conf/log4j.properties - sudo cp "${MONASCA_ANALYTICS_BASE}"/devstack/files/zookeeper/zoo.cfg /etc/zookeeper/conf/zoo.cfg - if [[ ${SERVICE_HOST} ]]; then - sudo sed -i "s/server\.0=127\.0\.0\.1/server.0=${SERVICE_HOST}/g" /etc/zookeeper/conf/zoo.cfg - fi - - sudo mkdir -p /var/log/zookeeper || true - sudo chmod 755 /var/log/zookeeper - - sudo systemctl start zookeeper || sudo systemctl restart zookeeper - else - echo_summary "SKIP:Install Monasca Zookeeper" - fi -} - -### -function install_kafka { - if [ ! -e /etc/init/kafka.conf ];then - echo_summary "Install Monasca Kafka" - - sudo groupadd --system kafka || true - sudo useradd --system -g kafka kafka || true - - download_through_cache $KAFKA_URL $KAFKA_TARBALL $SPARK_DOWNLOAD - sudo tar -xzf $SPARK_DOWNLOAD/$KAFKA_TARBALL -C /opt - sudo ln -sf /opt/kafka_${KAFKA_VERSION} /opt/kafka - - sudo cp -f "${MONASCA_ANALYTICS_BASE}"/devstack/files/kafka/kafka-server-start.sh /opt/kafka_${KAFKA_VERSION}/bin/kafka-server-start.sh - sudo cp -f "${MONASCA_ANALYTICS_BASE}"/devstack/files/kafka/kafka.conf /etc/init/kafka.conf - - sudo chown root:root /etc/init/kafka.conf - sudo chmod 644 /etc/init/kafka.conf - - sudo mkdir -p /var/kafka || true - sudo chown kafka:kafka /var/kafka - sudo chmod 755 /var/kafka - sudo rm -rf /var/kafka/lost+found - - sudo mkdir -p /var/log/kafka || true - sudo chown kafka:kafka /var/log/kafka - sudo chmod 755 /var/log/kafka - - sudo ln -sf /opt/kafka/config /etc/kafka - sudo cp -f "${MONASCA_ANALYTICS_BASE}"/devstack/files/kafka/log4j.properties /etc/kafka/log4j.properties - sudo chown kafka:kafka /etc/kafka/log4j.properties - - sudo chmod 644 /etc/kafka/log4j.properties - sudo cp -f "${MONASCA_ANALYTICS_BASE}"/devstack/files/kafka/server.properties /etc/kafka/server.properties - sudo chown kafka:kafka /etc/kafka/server.properties - sudo chmod 644 /etc/kafka/server.properties - - if [[ ${SERVICE_HOST} ]]; then - sudo sed -i "s/host\.name=127\.0\.0\.1/host.name=${SERVICE_HOST}/g" /etc/kafka/server.properties - sudo sed -i "s/zookeeper\.connect=127\.0\.0\.1:2181/zookeeper.connect=${SERVICE_HOST}:2181/g" /etc/kafka/server.properties - fi - - sudo cp -f "${MONASCA_ANALYTICS_BASE}"/devstack/files/kafka/kafka.service /etc/systemd/system/kafka.service - sudo chmod 644 /etc/systemd/system/kafka.service - - sudo systemctl enable kafka - sudo systemctl start kafka || sudo systemctl restart kafka - else - echo_summary "SKIP:Install Monasca Kafka" - fi -} - -### -function install_spark { - echo_summary "Install Spark" - - sudo mkdir -p $SPARK_DOWNLOAD - sudo chown -R stack:stack $SPARK_DIR - sudo chmod -R 755 $SPARK_DIR - mkdir -p ~/.m2 - - sudo -E apt-get update - - install_pkg - - build_spark - - install_zookeeper - - install_kafka -} - -### -function post_config_monasca_analytics { -: -} - -### -function extra_monasca_analytics { -: -} - -function download_through_cache { - local resource_location=$1 - local resource_name=$2 - local download_dir=$3 - - if [[ ! -d ${download_dir} ]]; then - _safe_permission_operation mkdir -p ${download_dir} - _safe_permission_operation chown stack ${download_dir} - fi - - pushd ${download_dir} - if [[ ! -f ${resource_name} ]]; then - sudo -E curl -m ${DOWNLOAD_FILE_TIMEOUT} --retry 3 --retry-delay 5 ${resource_location} -o ${resource_name} - fi - popd -} - -# check for service enabled -echo_summary "Monasca-analytics plugin with service enabled = `is_service_enabled monasca-analytics`" - -if is_service_enabled monasca-analytics; then - - if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then - # Set up system services - echo_summary "Configuring Spark system services" - pre_install_spark - echo_summary "Configuring Monasca-analytics system services" - pre_install_monasca_analytics - - elif [[ "$1" == "stack" && "$2" == "install" ]]; then - # Perform installation of service source - echo_summary "Installing Spark" - install_spark - - elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then - # Configure after the other layer 1 and 2 services have been configured - echo_summary "Configuring Monasca-analytics" - post_config_monasca_analytics - - elif [[ "$1" == "stack" && "$2" == "extra" ]]; then - # Initialize and start the Monasca service - echo_summary "Initializing Monasca-analytics" - extra_monasca_analytics - fi - - if [[ "$1" == "unstack" ]]; then - echo_summary "Unstacking Monasca-analytics" - unstack_monasca_analytics - fi - - if [[ "$1" == "clean" ]]; then - # Remove state and transient data - # Remember clean.sh first calls unstack.sh - echo_summary "Cleaning Monasca-analytics" - clean_monasca_analytics - fi - -else - echo_summary "Monasca-analytics not enabled" -fi - -#Restore errexit -$ERREXIT - -# Restore xtrace -$XTRACE diff --git a/devstack/settings b/devstack/settings deleted file mode 100644 index f9e902f..0000000 --- a/devstack/settings +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -enable_service monasca-analytics - -# -# Monasca infrastructure services -# - -# databases -# MySQL is already enabled in devstack - -# -# Dependent Software Versions -# - -DOWNLOAD_FILE_TIMEOUT=${DOWNLOAD_FILE_TIMEOUT:-1800} - -# spark vars -SPARK_DIRECTORIES=("/var/spark" "/var/log/spark" "/var/run/spark/work" "/etc/spark/conf" "/etc/spark/init" ) - -JDK_PKG="openjdk-8-jre-headless openjdk-8-jdk" - -MAVEN="apache-maven-3.5.4" -MAVEN_TARBALL="$MAVEN-bin.tar.gz" -MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/$MAVEN_TARBALL" - -SCALA_VERSION=${SCALA_VERSION:-2.12} -SCALA_MIN_VERSION=${SCALA_MIN_VERSION:-.8} -SCALA="scala-${SCALA_VERSION}${SCALA_MIN_VERSION}.deb" -SCALA_URL="https://downloads.lightbend.com/scala/${SCALA_VERSION}${SCALA_MIN_VERSION}/$SCALA" -KEYID=642AC823 - -SPARK_DIR="/opt/spark" -SPARK_DOWNLOAD="$SPARK_DIR/download" -SPARK_VERSION=${SPARK_VERSION:-2.4.4} -SPARK_TARBALL_NAME="spark-${SPARK_VERSION}.tgz" -SPARK_URL="http://archive.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK_TARBALL_NAME" - -BASE_KAFKA_VERSION=${BASE_KAFKA_VERSION:-2.1.1} -KAFKA_DIR="/opt/kafka" -KAFKA_DOWNLOAD="$KAFKA_DIR/download" -KAFKA_VERSION=${KAFKA_VERSION:-${SCALA_VERSION}-${BASE_KAFKA_VERSION}} -KAFKA_TARBALL="kafka_$KAFKA_VERSION.tgz" -KAFKA_URL="https://archive.apache.org/dist/kafka/$BASE_KAFKA_VERSION/$KAFKA_TARBALL" diff --git a/doc/banana.md b/doc/banana.md deleted file mode 100644 index be1b0d1..0000000 --- a/doc/banana.md +++ /dev/null @@ -1,141 +0,0 @@ -# Banana: a configuration language for Monanas - -Welcome to Banana, a configuration language for Monanas. The language is the -key of "recipes" allowing users to reuse it to tailor solutions for their -use-cases. In other words, Banana allows us to write a recipe(s) that will be -ingested by Monanas. - -The language is fully typed. It uses type inference to avoid having to add any -typing annotation. It is still in its early stages, so more features will be -added to solve common problems. - -> Note: a valid `banana` recipe (or file) might still contain errors that -> can only be discovered at run-time. The type system, will remove -> most of them though. - -To get you started, we provide an example below, which would allow us to -understand most parts of the language. - -> TODO: Once we have a specified interface, we should use it instead as it will -> provide syntax highlighting, in-editor error indication as well as other -> features such as autocompletion. - -## Part 1: Creating components - -Here is how we create a component: - -```python -# Create an IPTablesSource with sleep parameter set to 0.01 -src = IPTablesSource(sleep=0.01) -``` - -We could also create a component without any parameter. In that case, each -parameter is initialized with a default value. - -In order to get something interesting we first create the following components: - -```python -src = IPTablesSource(sleep=0.01) -ing1 = IptablesIngestor() -svm = SvmOneClass() -voter = PickIndexVoter(0) -ldp1 = IptablesLDP() -stdout = StdoutSink() -sqlite = IptablesSQLiteSink() -``` - -## Part 2: Connecting components - -Connections can be placed anywhere in the file. They will always be processed -after everything else. - -We have created five components so far, but note that, some components can only -connected to certain types of components). For instance, a source can only -be connected to an ingestor or a live data processor. However, you can't -connect it to a statistical or machine learning algorithms as those need to get -curated data only. Try to add the following line: - -```py -src -> alg -``` - -You should see an error, saying that this is not possible: - -``` -Error: Can't connect src to alg, src can only be connected to Ingestor. -``` - -What we want is to have those connections: - -``` -+---------+ +---------+ +---------+ +---------+ +---------+ +------------+ -| src +-----> | ing1 +------> | alg +-----> | vot +---> | ldp +-----> | stdout | -+----+----+ +---------+ +---------+ +---------+ +----+----+ +------------+ - | ^ - | | - +----------------------------------------------------------------------+ -``` - -Here is how we can achieve that: - -``` -src -> [ing1 -> alg -> vot -> ldp, ldp] -ldp -> stdout -``` - -We could also write it like this: - -``` -src -> ing1 -> alg -> vot -> ldp -src -> ldp -> stdout -``` - -Or like this: - -``` -src -> ing1 -src -> ldp -ing1 -> alg -alg -> vot -vot -> ldp -ldp -> stdout -``` - -The main difference is readability and this is subjective. Use the version that -you think is more readable. - -Banana will treat all of them as being semantically identical. - -## Part 3: Sharing settings between components - -From what we described above, it is possible that we could end up with many -similar parameters across components. It would be great if we could share them. -In Banana we can declare a variable not only for components, but also for -`string`, `number` and json-like `object`. - -For instance, this is valid in banana: - -```python -sleep = 0.01 -# ... -``` - -You can also make use of arithmetic operators to perform any computation you -might require with previously declared variables or anything else. - -Some global variables, defined by the execution environment are also available. -For instance, you could define `sleep` like this: - -```python -sleep = 0.01 / $spark.BATCH_INTERVAL -# ... -``` - -> TODO: the above feature has not yet been implemented. - -Finally, Banana supports string interpolation to mix many types together: - -```python -port = 9645 -host = "localhost:" + port -``` diff --git a/doc/cli.md b/doc/cli.md deleted file mode 100644 index 9f19e74..0000000 --- a/doc/cli.md +++ /dev/null @@ -1,187 +0,0 @@ -# Command line interface to generate a JSON configuration - -A simple command line tool has been implemented in order to manage -configurations in an easier way. It is not expected to be maintained in the -long term. It is introduced to experiment with the creation of `banana`, the -configuration language of Monanas. - -This section explains what operations are currently available and how to use -them. - -> NOTE: Please refer to [Monasca/Configuration](configuration.md) for the -> structure of the JSON configuration. - -> NOTE: This tool is DEPRECATED, use [BANANA](banana.md) configuration language -> instead. - -## Usage - -* Start the Monanas cli -```bash -python $MONANAS_HOME/config_dsl.py -``` -After running this command, a simple empty configuration will be created. You -can then modify it by running the commands listed below: - -## Available commands - -You can run the following operations from the CLI in order to create, remove, connect, -disconnect and modify components from the configuration. - -### Create component - -Create a component and assign it to a variable: - -```python ->> A = IPTablesSource -``` - -This command adds a new source of type `IPTablesSource` to the configuration, -assigning it with a default configuration. It links the source component to -variable A and returns a unique ID. You can either use the variable or the ID -to refer to the instance of the `IPTablesSource` created. - -### Remove component - -Remove a component using an ID or a variable name: - -```python ->> rm A -``` - -This command removes the component referenced by `A` from the configuration. -The parameter can either be a variable or an ID associated to a component -in the configuration. The component will only be removed if it is not connected -to any other component. - -### Connect components - -Connect two components in the configuration: - -```python ->> A -> B -``` - -This command connects the component referenced by `A` with the component -referenced by `B`. Both `A` and `B` can be variables or IDs, and the connection -is directional from `A` to `B`. The connection is valid and considered only if -the associated components exist and allowed for connection. For example, -connecting a source with an ingestor is allowed, but connecting a source with -a voter is not. - -### Disconnect components - -Disconnect two components in the configuration: - -```python ->> A !-> B -``` - -This command disconnects the component `A` from component `B`. Both `A` and `B` -can be variables or IDs and the connection is directional from `A` to `B`. If -the connection doesn't exist, nothing will happen. - -### Modify component - -Modify values of the configuration of a component: - -```python ->> A.params.subparam1.subparam2 = value -``` - -This command modifies the value of the configuration parameter at the end of -the path defined by a dot notation. The configuration is validated before being -modified; hence, if the modification results in an invalid configuration, it -will not be executed. `A` can either be a variable or an ID. - -## Config presentation operations - -The following operations can be run using the tool in order to view the -current configuration, sub-configurations, and available components that can be -instantiated. - -### Print - -Print the full configuration: - -```python ->> print -``` - -This command displays the full configuration in JSON format on the screen. - -Print component type sub-configuration: - -```python ->> print connections -``` - -If a parameter is passed to the print command that corresponds to a component -type, or in general, a first level key of the configuration, only the relevant -sub-configuration that is selected will be displayed on the screen. - -Print a particular component sub-configuration: - -```python ->> print A -``` - -If a parameter is passed to the print command that corresponds to a variable -or an ID associated to a particular component, only its configuration will be -displayed on the screen. - -### List - -Print all available components: - -```python ->> list -``` - -This command displays all available components that can be add to the -configuration, organized by type. - -Print all available components of a particular type: - -```python ->> list smls -``` - -If a type is passed as a parameter to the list command, only the available -components of that type will be listed. - -## Config storage operations - -The following operations can be run from the tool in order to load and save -configurations from/to files. - -### Load - -Load a configuration from a file: - -```python ->> load filename -``` - -This command loads the configuration stored in the file 'filename', overriding -the configuration currently being manipulated in memory. - -### Save - -Save a configuration to a file: - -```python ->> save filename -``` - -This command saves the configuration being currently handled to the file -'filename', overriding the file if it existed previously. - -Save a configuration to the last file: - -```python ->> save -``` - -If no parameter is provided, the save operation saves the current -configuration being handled to the last file loaded from or saved to. diff --git a/doc/configuration.md b/doc/configuration.md deleted file mode 100644 index 8fe3334..0000000 --- a/doc/configuration.md +++ /dev/null @@ -1,123 +0,0 @@ -# Configurations -MoNanas consumes two configuration files, one for orchestrating data execution -and the other for logging. - -## Orchestrating Configuration -> Note: Please refer to [Monasca/Design](design.md) to understand the concept -of each component before creating or modifying a configuration. - -With the current implementation, a JSON file is used to configure how MoNanas -orchestrates the data execution pipeline. This comes in different parts as -follows. - -### ID (`id`) - -A unique identifier for the configuration. - -### Spark (`spark_config`) - -* `appName`: A string describing the Spark's application name. -* `streaming`: Attributes for data streaming. - * `batch_interval`: DStream's batch interval. - -### Server (`server`) - -* `port`: Port number to listen. -* `debug`: Debug mode. - -### Sources (`sources`) - -`sources` is a JSON object (equivalent to python dictionary in this case) where -the keys represent unique identifiers of sources. Each source has the following -attributes: -* `module`: The name of the python module used for connecting to the source. -* `params`: A JSON object representing the source's parameters (e.g. data model -used). - -### Ingestors (`ingestors`) - -`ingestors` is a JSON object (equivalent to python dictionary in this case) -where the keys represent unique identifiers of ingestors. Each ingestor has -the following attributes: -* `module`: The name of the python module implementing the ingestor. -* `params`: A JSON object representing the ingestor's parameters (e.g. -parameters for data conversion). - -### Aggregators (`aggregators`) - -`aggregators` is a JSON object (equivalent to python dictionary in this case) -where the keys represent unique identifiers of aggregators. Each aggregator -has the following attributes: -* `module`: The name of the python module implementing the aggregator. -* `params`: A JSON object representing the aggregator's parameters (e.g. how -data streams are aggregated). - -> Note: Currently, only one aggregator is supported - the python module is -`default_aggregator`. - -### SML Functions (`sml`) - -`sml` is a JSON object (equivalent to python dictionary in this case) -where the keys represent unique identifiers of SML functions. Each SML function -has the following attributes: -* `module`: The name of the python module implementing the SML function. -* `params`: A JSON object representing the SML function's parameters (e.g. -number of samples, confidence interval). - -### Voters (`voters`) - -`voters` is a JSON object (equivalent to python dictionary in this case) where -the keys represent unique identifiers of voters. Each voter has the following -attributes: -* `module`: The name of the python module implementing the voter. -* `params`: A JSON object representing the voter's parameters (e.g. weights, -topology). - -### Live Data Processors (`ldp`) - -`ldp` is a JSON object (equivalent to python dictionary in this case) where -the keys represent unique identifiers of live data processors. Each live data -processor has the following attributes: -* `module`: The name of the python module implementing the live data processor. -* `params`: A JSON object representing the live data processor's parameters -(e.g. mode). - -### Sinks (`sinks`) - -`sinks` is a JSON object (equivalent to python dictionary in this case) where -the keys represent unique identifiers of sinks. Each sink has the following -attributes: -* `module`: The name of the python module used for connecting to the sink. -* `params`: A JSON object representing the sink's parameters (e.g. server's -details, data format). - -### Connections - -`connections` is a JSON object (equivalent to python dictionary in this case) -where each key represents a unique identifier of the component acting as the -originating end of the data flow where its associated value is an list of -unique identifiers of components acting as terminating ends of the data flow. -The information described by `connections` can be used to represent the flow -of data execution end-to-end. - -## Logging Configuration - -### MoNanas - -MoNanas comes with a default logging configuration. To change the logging -properties including format, level, etc., simply override -`$MONANAS_HOME/config/logging.json`. - -### Spark - -By default, Spark's logging is at an INFO level. To avoid unnecessary console -output, change `log4j.rootCategory=INFO, console` to `log4j.rootCategory=ERROR, -console` in `$SPARK_HOME/conf/log4j.properties` or as preferred. - -> Note: Not required for a Vagrant generated VM. - -### Other Software/Tools - -For logging configuration of other software/tools, please refer to their -user guide. - diff --git a/doc/design.md b/doc/design.md deleted file mode 100644 index d2a7c88..0000000 --- a/doc/design.md +++ /dev/null @@ -1,106 +0,0 @@ -# MoNanas/Design - -## Overview - -MoNanas enables users to compose their data execution flow through -configuration. Each part of the configuration describes how MoNanas should -handle the data. - -![Tags](images/tags.png) - -The needs for processing data from heterogeneous sources and a way to describe -the execution declaratively (i.e. how data flow and get processed) are -increasing. An implicit cause is the requirement to get a better insight by -utilizing the enormous amount of data we have. By taking several concepts -required to achieve that, MoNanas aims to be an statistical/analytic framework -for processing data from arbitrary sources with Monasca being the core user -to makes sense out of alerts data generated or coming into the system. - -### Functional Requirements - -To understand the design principle of MoNanas, first we need to understand what -the functional requirements are. - -* A machine learning framework that ingests events/data from arbitrary sources -(single or multiple, homogeneous or heterogeneous), processes it and persists -into arbitrary data sinks. -* Composable components for events/data processing through configurations, -where additional machine learning algorithms can be easily integrated. For -example, a stream of events can be aggregated using a causality graph and -motif comparisons. -* An easy to use API for controlling and manipulating events/data streams as -well as querying for processed data. - -> #### Additional Requirements -> * Built on open-source technologies. - -## MoNanas Components - -![MoNanas Architecture](images/architecture.png) - -The diagram above illustrates MoNanas architecture with the following -components. - -### Data Sources -A data source in MoNanas refers to the source of information including alerts, -raw events or any other data that can be pushed or pulled into the system. -This can be a streaming source or readily available data available for -accessing. MoNanas provides a few commonly used connectors including Kafka. -Since MoNanas uses Spark, any sources provided by Spark can be easily utilized -in MoNanas as well. - -### Data Ingestors -Machine learning algorithms may not be implemented in such a way that it can -use data with arbitrary format as inputs. Therefore, an ingestor is required -for a specific type to convert the data into a common format, in this case, -numpy array, which can be understood by the SML functions. We provide each data -ingestor for each source available in MoNanas. - -### Data Aggregators -Currently, MoNanas supports only one aggregator (`default_aggregator`). Its -responsibility is to aggregate data from different streams (from ingestors). -However, the support for multiple aggregators and multilayered aggregations -can be easily developed in the future. - -### Statistical/Machine Learning (SML) Functions -An SML function may be a single machine learning algorithm or a specifically -composed sequence of algorithms that consumes numpy arrays and processes the -data to extract insights. Few SML functions are readily available for use -including LiNGAM and LibPGM for finding causality structures. - -### Voters -In case where multiple SML functions are used in the orchestration of the -SML flow, voters are used to determine which outcome(s) of the SML functions -are used to process the data streamed from sources/ingestors. MoNanas's default -voter is `pick` where the user simply indicate the index of the SML function -that finishes the learning process. - -### Live Data Processors -A live data processor is optional in the data execution pipeline. It takes the -outcome of a voted SML function and uses it to process the data coming from sources -or ingestors. - -### Data Sinks -In MoNanas, data sinks are not necessarily persistent storage. For example, we -can push processed data back to Kafka with a different topic for Monasca or -Vitrage consumption. Any type of data can be pushed to data sink including raw -data, ingested data, processed data and outcomes of the SML functions. - -## Statistical/Machine-Learning Algorithms -Statistical/Machine Learning (SML) algorithms are used to gain insights of -data ingested into MoNanas. It can be an arbitrary algorithm, a simple filter -or a composition of multiple algorithms (run in sequence, in parallel or a -combination of both) as long as it satisfies the data processing requirement. -The outputs of these algorithms are then discarded, aggregated or utilized, -usually by one or more live data processors to process the raw or ingested -data. - -## Data Flow Composition -Data flow composition is a essential concept in MoNanas. It allows users to -compose a data flow and execution arbitrarily based on available components as -long as the composition is valid i.e. the connections defined by the -composition are all valid. For example, connecting a source to a voter -directly is invalid but connecting a source to a live data processor or an -ingestor is valid. Each component used in the composition can be one of many -provided by MoNanas or one of the custom developed components. Diagrams of -example compositions can be found in [MoNanas/Examples](examples.md). diff --git a/doc/dev_guide.md b/doc/dev_guide.md deleted file mode 100644 index a0c0552..0000000 --- a/doc/dev_guide.md +++ /dev/null @@ -1,89 +0,0 @@ - - -# MoNanas/DevGuide - -## Development Environment - -MoNanas's repository comes with a Vagrantfile for a quick deployment on the -client system. To use vagrant, simple do the following. - -* Install [vagrant](https://www.vagrantup.com/) on your local machine. -* Clone the project from https://github.hpe.com/labs/monanas/. -* From `$MONANAS_HOME`, run `vagrant up && vagrant ssh`. This will set up a VM -using VMWare or VirtualBox and ssh onto it. Once you have logged into the -instance, following the guideline provided in -[MoNanas/GettingStarted](getting_started.md). - -At the root of the project there is a `Makefile` which provides common tasks: - -* `make test`: Run the test suite. -* `make style`: Check for pep8 compliance on the entire code base. -* `make testspec ` A handy way to test only a - `TestCase` subclass. -* `make all`: Run both `test` and `style`. -* `make start`: Start Monanas and send the `start_streaming` action via the - REST interface. - -## Adding Custom Components - -As illustrated in [Monasca/Design](design.md), MoNanas's architecture was -designed to be pluggable. Therefore, integrating a new component or a new -statistical/machine learning algorithm is very simple. The following shows a -step-by-step guide on how to add a new data source and a new learning -algorithm. A custom implementation of other components can be added in similar -fashion. - -### Add New Data Sources - -When creating a new data source, everything you need is located in `main/source` package, and new sources should be contained in that package in order to keep the convention. All you need to do is extend the class `BaseSource` in `main/source/base.py` module. - -#### Default configuration and Validation -The first step in the Data Source life-cycle is its creation and configuration validation. Also, a default configuration is needed by DSL in order to add a component to the configuration, and it can be very convenient for users to have a default configuration. Please, implement the following methods: - -* `validate_config` -It should validate the schema of the configuration passed as parameter, checking that expected parameters are there, and values have the expected type and/or values. Please, check other classes validate_config implementations in order to have examples on how to use the Schema library. Please, make this method static by annotating it with the @staticmethod decorator. - -* `get_default_config` -It should return a dictionary containing the default schema of this component. This method will be called by DSL when creating a component of this type. Please, make this method static by annotating it with the @staticmethod decorator. - -#### Main logic functions -The aim of a source class is to provide data which will then be consumed by ingestors. When MoNanas is ordered to start streaming data, the source classes will be asked to create a stream of data, and other components in the pipeline may be interested in the features of the data provided by the source class. - -* `create_dstream` -It should create a spark dstream using the Spark Streaming Context passed as parameter. Please, refer to spark documentation if you want more details about dstream object, and feel free to view implementations of this function by other source classes. - -* `get_feature_list` -It should return a list of strings in order representing the features provided by the dstream. - -#### Termination functions -When MoNanas is ordered to stop streaming data, it will call terminate_source in all the sources that are streaming. - -* `terminate_source` -It should do any necessary cleanup of the source when it is terminated. For example, if the source was running a TCP server generating traffic, at this point it may want to stop it. - - -### Add New Learning Algorithms -When adding a new algorithm, everything you need is located in: -`main/sml` package, and new algorithms should be contained in that package in order to keep the convention. All you need to do is extend the class `BaseSML` in `main/sml/base.py` module. - -#### Default configuration and Validation -Please, refer to the 'Add New Data Sources' section. - -#### Main logic functions -The aim of a SML class is to train a machine learning algorithm, or do statistics to learn something, using a batch of data provided by the aggregator. When data is available, it will be manipulated by the logic implemented in the learn_structure function; the data flow will be stopped by MoNanas when all the SMLs have consumed at least the number of samples provided by the number_of_samples_required function. - -* `learn_structure` -This is the function that implements the logic of the algorithm. The data is provided as a parameter, and it should return the structure learned from the data (e.g. causality matrix, or trained classifier object). - -* `number_of_samples_required` -this function should return the number of samples that the algorithm requires in order to provide reliable results. - -## Coding Standards -Python: All Python code conforms to the OpenStack standards at: -https://docs.openstack.org/hacking/latest/ - -* Developers must add unit tests for all logical components before merging to -master. -* Pull Requests are welcome and encouraged to ensure good code quality. Label -the PR as `ready for review` when all the features are completed. - diff --git a/doc/examples.md b/doc/examples.md deleted file mode 100644 index 8235c54..0000000 --- a/doc/examples.md +++ /dev/null @@ -1,556 +0,0 @@ -# Examples -This section shows a step-by-step guide for running examples using MoNanas. - -## Alert Fatigue Management -When there are too many alerts, devops become less productive due to alert -fatigue. Using MoNanas helps devops to focus on only what matters, in many -cases, "the root causes" of those alerts. This example shows how MoNanas -ingests alerts, aggregates them based on LiNGAM, a non-gaussian methods for -estimating causal structures, and pushes aggregated alerts back to a Kafka -server. - -![Alert Fatigue Pipeline](images/alert_fatigue_pipeline.png) - -The diagram above shows a simple data execution pipeline for consuming alerts -from Kafka server used by Monasca for messaging. However, to demonstrate this -scenario without integrating into a production environment, we use a Markov -Chain process to generate alerts (as illustrated in the diagram below) similar -to that would have been pushed to the queue by Monasca. - -![Alert Fatigue Pipeline with Markov Chain Process](images/alert_fatigue_pipeline_with_markov.png) - -### Running the Example -To test this example, perform the following steps. - -#### Configuration File -Before running MoNanas, we need to create a configuration file describing how -we want MoNanas to orchestrate the data execution (creating a pipeline). You can find the following configuration in `$MONANAS_HOME/config/markov_source_config.json`: - - -```json -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "CloudMarkovChainSource", - "params": { - "server_sleep_in_seconds": 0.01 - }, - "transitions": { - "web_service": { - "run=>slow": { - "0": 0.001, - "8": 0.02, - "12": 0.07, - "14": 0.07, - "22": 0.03, - "24": 0.001 - }, - "slow=>run": { - "0": 0.99, - "8": 0.7, - "12": 0.1, - "14": 0.1, - "22": 0.8, - "24": 0.99 - }, - "stop=>run": 0.7 - }, - "host": { - "on=>off": 0.005, - "off=>on": 0.5 - }, - "switch": { - "on=>off": 0.01, - "off=>on": 0.7 - } - }, - "triggers": { - "support": { - "get_called": { - "0": 0.1, - "8": 0.2, - "12": 0.8, - "14": 0.8, - "22": 0.5, - "24": 0.0 - } - } - }, - "graph": { - "h1:host": ["s1"], - "h2:host": ["s1"], - "s1:switch": [], - "w1:web_service": ["h1"], - "w2:web_service": ["h2"] - } - - } - }, - "ingestors": { - "ing1": { - "module": "CloudIngestor" - } - }, - "smls": { - "sml1": { - "module": "LiNGAM", - "params": { - "threshold": 0.5 - } - } - }, - "voters": { - "vot1": { - "module": "PickIndexVoter", - "params": { - "index": 0 - } - } - }, - "sinks": { - "snk1": { - "module": "KafkaSink", - "params": { - "host": "localhost", - "port": 9092, - "topic": "transformed_alerts" - } - } - }, - "ldps": { - "ldp1": { - "module": "CloudCausalityLDP" - } - }, - "connections": { - "src1": ["ing1", "ldp1"], - "sml1": ["vot1"], - "ing1": [], - "vot1": ["ldp1"], - "ldp1": ["snk1"], - "snk1": [] - }, - "feedback": {} -} -``` - -The flow of data execution is defined in `connections`. In this case, data -are ingested from `src1` where Monasca-like alerts are being generated -using Markov Chain process. The data are then ingested by `ing1` where -each entry is converted into a format suitable for machine learning algorithm. -MoNanas uses `numpy.array` as a standard format. Typically, an aggregator is -responsible for aggregating data from different ingestors but in this scenario, -there is only one ingestor, hence the implicit aggregator (not defined in the -configuration) simply forwards the data to `sml1`, which simply deduplicates -the data then uses LiNGAM algorithm to find a causality structure of the -aggregated data then passes the result (structure) to `vot1`. The voter is -configured to pick the output of the first SML function and forwards that to -`ldp1`. Here, the live data processor transforms data streamed from `src1` -using the causality structure and pushes it to standard output as well as the -specified Kafka server. - -#### Run MoNanas -As the sink of this orchestration is Kafka, we need to run Apache Zookeeper -and Apache Kafka first. - -```bash -$KAFKA_HOME/bin/zookeeper-server-start.sh \ - $KAFKA_HOME/config/zookeeper.properties - -$KAFKA_HOME/bin/kafka-server-start.sh \ - $KAFKA_HOME/config/server.properties -``` - -After that, start MoNanas as follows. - -```bash -python $MONANAS_HOME/run.py -p $SPARK_HOME -c $MONANAS_HOME/config/markov_source_config.json \ - -l $MONANAS_HOME/config/logging.json -``` - -A default logging configuration file is located in `$MONANAS_HOME/config`. -Users can override this option. Currently, `-l` option is mandatory as MoNanas -does not assume a default configuration. - -#### Start Data Execution -MoNanas exposes REST API for controlling the data execution. Use any HTTP -client to POST with the following request body to MoNanas server to start -data streaming. - -```json -{ - "action": "start_streaming" -} -``` -e.g. using curl from terminal to make a request assuming the server is running -locally. -```bash -curl -H "Content-Type: application/json" -X POST \ - -d '{"action": "start_streaming"}' \ - http://localhost:3000/api/v1/actions -``` - -#### Results -The sinks for the transformed (processed) alerts defined in the configuration -are via standard output and Kafka server. Therefore, the output will be -displayed in the console. Alternatively, users can subscribe to a queue -with the topic `transformed_alerts` using any Kafka client. A result example -(live data processing with aggregation mode based on causality structure) found -by LiNGAM algorithm is shown below. - -```json -{ - "alarm_id": "219e65b8-395a-4588-a792-799fdbeef455", - "id": "1452900361688", - "metrics": [ - "..." - ], - "new_state": "OK", - "old_state": "UNDETERMINED", - "reason": "The alarm threshold(s) have not been exceeded.", - "reason_data": "{}", - "sub_alarms": [ - "..." - ], - "timestamp": "2016-01-15T23:26:01.688Z", - "__monanas__": { - "causes": [ - "9b0129fa-7869-4436-8319-68bfa8bc88a1", - "3f4b2651-41f5-4f2a-8093-e9ad8cd5b7bf" - ] - } -} -``` - -The alert shown above has been annotated with MoNanas's specific metadata, -`__monanas__`. In this case where the mode of live data processing is -`aggregation`, IDs of the alerts caused by this one are added onto a list, -`causes`. - -> Note: The underlying causality structure is a directed acyclic graph. - -In contrast, if the `filter` mode is used, the result format would be identical to -the source input format. So you can typically use it anywhere in existing data -flow. In this mode, only root cause alerts are preserved. A result example -under this mode is shown below. - -## Disk Failure prediction using SMART data - -Hard disk drives (HDDs) are most fragile part of systems and consequences of disk failures can be difficult to recover, or even unrecoverable. -To ensure the reliability and stability of systems, it is crucial to monitor the working conditions of HDDs in real time and detect soon-to-fail HDDs by sensors. - -This example shows how MoNanas ingests SMART (self-monitoring and repair technology) data, trains a classification algorithm(Random forest), and then uses the trained algorithm to detect the likelihood failure of the HDDs. - -To train the model, we use publically available [SMART dataset](https://www.backblaze.com/b2/hard-drive-test-data.html), which is collected in the Backblaze data center. Backblaze run tens of thousands hard drives from 2013 and representing the largest public SMART dataset. - -### Running the Example -To test this example, perform the following steps. - -#### Configuration File -Before running MoNanas, we need to create a configuration file describing how -we want MoNanas to orchestrate the data execution (creating a pipeline). You can find the following configuration in `$MONANAS_HOME/config/smart.json`: - - - - -```json -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "SmartFile", - "params": { - "dir": "/var/tmp/source_data/" - } - } - }, - "ingestors": { - "ing1": { - "module": "SmartIngestor" - } - }, - "smls": { - "sml1": { - "module": "RandomForestClassifier", - } - }, - "voters": { - "vot1": { - "module": "PickIndexVoter", - "index": 0 - } - }, - "sinks": { - "snk3": { - "module": "KafkaSink", - "host": "127.0.0.1", - "port": 9092, - "topic": "my_topic" - }, - "snk2": { - "module": "StdoutSink" - } - }, - "ldps": { - "ldp1": { - "module": "Smart" - } - }, - "connections": { - "src1": ["ing1", "ldp1"], - "ing1": [], - "sml1": ["vot1"], - "vot1": ["ldp1"], - "ldp1": ["snk1", "snk2"], - "snk1": [], - "snk2": [], - "snk3": [] - }, - "feedback": {} -} -``` - - -The flow of data execution is defined in `connections`. In this case, data -are ingested from `src1` where smart data with csv format smart-data are being generated -using open dataxxxx. The data are then ingested by `ing1` where -each entry is converted into a format suitable for machine learning algorithm. -MoNanas uses `numpy.array` as a standard format. Typically, an aggregator is -responsible for aggregating data from different ingestors but in this scenario, -there is only one ingestor, hence the implicit aggregator (not defined in the -configuration) simply forwards the data to `sml1`, which uses Random Forest algorithm - to find the disk failures then passes the result to `vot1`. - -The voter is configured to pick the output of the first SML function and -forwards that to `ldp1`. Here, the live data processor transforms data streamed from `src1` -using prediction result and pushes it to standard output as well as thespecified Kafka server. - -#### Run MoNanas -Start MoNanas as follows: - - -```bash -python $MONANAS_HOME/run.py -p $SPARK_HOME -c $MONANAS_HOME/config/iptables_anomalies.json \ - -l $MONANAS_HOME/config/logging.json -``` -If you want to run your own configuration you will need to change the value of the -c parameter. - -#### Start Data Execution -MoNanas exposes a REST API for controlling the data execution. Use any HTTP -client to POST with the following request body to MoNanas server (running on your localhost, at port 3000 by default) to start data streaming. - - -```json -{ - "action": "start_streaming" -} -``` -e.g. using curl from terminal to make a request assuming the server is running -locally. - -```bash -curl -H "Content-Type: application/json" -X POST \ - -d '{"action": "start_streaming"}' \ - http://localhost:3000/api/v1/actions -``` - -#### Stop Data Execution -When you want to stop the example, you can send another HTTP POST to order MoNanas to stop streaming data. In this case, the request body should be: - - -```json -{ - "action": "stop_streaming" -} -``` -e.g. using curl from terminal to make a request assuming the server is running -locally. - -```bash -curl -H "Content-Type: application/json" -X POST \ - -d '{"action": "stop_streaming"}' \ - http://localhost:3000/api/v1/actions -``` - -#### Results - - -The sinks for the transformed (processed) alerts defined in the configuration are via standard output and Kafka server. -Therefore, if the hard drive failure is predicted, the output will be displayed in the console. Alternatively, users can subscribe to a queue with the topic "my_topic" using any Kafka client. - -A result example is shown below. The value of the `DiskFailure` is identifier specified in input, so you can change the value as you like, as long as it's constant and unique across all data. - -```json -{ -"DiskFailure": "ST3000DM001_disk001" -} -``` - - - - -## Anomalies in Rule Firing Patterns - -Some attacks can be recognized by patterns of rules being triggered in an anomalous fashion. For example, a Ping flood, a denial of service attack that consist in sending multiple pings to a system, would trigger IPTable rules handling the Ping much more often than if the system is not being attacked. - -This example shows how MoNanas ingests iptables firing data, trains an anomaly detection algorithm (SVM One Class), and then uses the trained algorithm to detect when the system is being attacked, which is detected as an anomaly. Finally the data is stored in a SQLite database via the SQLite sink. - -![IPTables Anomaly Detection Pipeline](images/iptables_arch.png) - -The diagram above shows a simple data execution pipeline for generating and consuming IPTables triggering data. The source is a TCP server that generates mock data, triggering IPTables according to the logic provided by this Markov Chain: - -![IPTables Markov Chain Source](images/iptables_src.png) - -As you can see, there are 3 states: `stopped`, `normal`, and `attack`. Each transition marked in the diagram can happen with some probability each time period: we made sure that the system stays in `normal` state most of the time. -The system triggers 12 possible IPTables, classified in four families: `ssh`, `concrete IPs`, `http`, and `ping` with different probabilities according to the state of the system, in order to model possible ping flood attacks: - -1) State `stopped`: nothing is triggered -2) State `normal`: higher probability for http traffic, but some of the other rules will be triggered. -3) State `attack`: very high probability of triggering `ping` traffic in every timestamp, keeping the rest of probabilities as they were in `normal` state. - -The ingestor consumes data from the markov chain IPTable source, and counts the number of times that an IPTable of each family has been triggered in every time window, generating a 4D array per time window (one dimension per IPTable type). - -The SML block is an anomaly detection algorithm which is based in SVM (Support Vector Machines). In the first phase of MoNanas execution, it trains the classifier, learning the border in the space of arrays generated by the ingestor between anomalies and non-anomalies. - -After the pick voter selects the only available SML, the LDP module is ready to find anomalies in real time in phase 2. It uses the feature names returned by the IPTables Markov chain source class, which are the IPTable type represented in each position of the arrays. When new data of triggered IPTables arrive to the IPTables LDP, it is classified as anomaly or non-anomaly, and the dstream is modified with this information. - -Finally the SQLite sink saves the classified data to a SQLite database. - -### Running the Example -To test this example, perform the following steps. - -#### Configuration File -Before running MoNanas, we need to create a configuration file describing how -we want MoNanas to orchestrate the data execution (creating a pipeline). You can find the following configuration in `$MONANAS_HOME/config/iptables_anomalies.json`: - -```json -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "IPTablesSource", - "params": { - "server_sleep_in_seconds": 0.01 - } - } - }, - "ingestors": { - "ing1": { - "module": "IptablesIngestor" - } - }, - "smls": { - "sml1": { - "module": "SvmOneClass" - } - }, - "voters": { - "vot1": { - "module": "PickIndexVoter", - "params": { - "index": 0 - } - } - }, - "sinks": { - "snk1": { - "module": "IptablesSQLiteSink" - }, - "snk2": { - "module": "StdoutSink" - } - }, - "ldps": { - "ldp1": { - "module": "IptablesLDP" - } - }, - "connections": { - "src1": ["ing1", "ldp1"], - "ing1": [], - "sml1": ["vot1"], - "vot1": ["ldp1"], - "ldp1": ["snk1", "snk2"], - "snk1": [], - "snk2": [] - }, - "feedback": {} -} -``` - -The flow of data execution is defined in `connections`, with the origin being the keys of the dictionary, and the destinations are each element of the list associated to each origin. -As you can see, the flow described in the previous section is mapped to the configuration, except for the ingestor, which is not explicitly connected; the reason for this is that all ingestors are connected to the aggregator, which is connected to all SMLs, so no explicit connection needs to be defined. - -#### Run MoNanas -Start MoNanas as follows: - -```bash -python $MONANAS_HOME/run.py -p $SPARK_HOME -c $MONANAS_HOME/config/iptables_anomalies.json \ - -l $MONANAS_HOME/config/logging.json -``` -If you want to run your own configuration you will need to change the value of the -c parameter. - -#### Start Data Execution -MoNanas exposes a REST API for controlling the data execution. Use any HTTP -client to POST with the following request body to MoNanas server (running on your localhost, at port 3000 by default) to start data streaming. - -```json -{ - "action": "start_streaming" -} -``` -e.g. using curl from terminal to make a request assuming the server is running -locally. -```bash -curl -H "Content-Type: application/json" -X POST \ - -d '{"action": "start_streaming"}' \ - http://localhost:3000/api/v1/actions -``` - -#### Stop Data Execution -When you want to stop the example, you can send another HTTP POST to order MoNanas to stop streaming data. In this case, the request body should be: - -```json -{ - "action": "stop_streaming" -} -``` -e.g. using curl from terminal to make a request assuming the server is running -locally. -```bash -curl -H "Content-Type: application/json" -X POST \ - -d '{"action": "stop_streaming"}' \ - http://localhost:3000/api/v1/actions -``` - -#### Results -By default, the pre-selected logging level defined in `$MONANAS_HOME/config/logging.json` lets you see relevant information in the screen during this execution of this example. -The classified data generated by the LDP module is saved in a SQLite database located by default in `sqlite_sink.db` file; please feel free to inspect the database to see what was classified as anomaly and what was classified as normal traffic. diff --git a/doc/getting_started.md b/doc/getting_started.md deleted file mode 100644 index fcfbea9..0000000 --- a/doc/getting_started.md +++ /dev/null @@ -1,68 +0,0 @@ -# MoNanas/GettingStarted - -This section serves as a user guide and helps you to get started with MoNanas. -For developer guide, see: [MoNanas/DevGuide](dev_guide.md). - -> Note: `$XXX_HOME` indicates the root directory where the corresponding -project resides. For example, `$KAFKA_HOME` refers to where Kafka's source -files have been extracted to. - -## Pre-requisites - -* Python (>= v.2.7.6) -* Apache Spark (>= v.1.6.1) -* Apache Kafka (>= v.0.8.2.1) - this comes with Apache Zookeeper. -* Vagrant (for development environment or testing MoNanas only). - -> Note: Newer versions of these software/tools should be compatible, but are -not tested so nothing is guaranteed. - -## Installation - -* Clone MoNanas repo: https://github.com/openstack/monasca-analytics - -### Everything on Host - -The easiest way to install everything on a physical host is through our -provided script, `fetch-deps.sh` located in `$MONANAS_HOME`. - -### Everything on VM - -MoNanas comes with a Vagrantfile for quick deployment. For more information, -see: [MoNanas/DevGuide](dev_guide.md). - -## Usage - -* Start Apache ZooKeeper1 -```bash -$KAFKA_HOME/bin/zookeeper-server-start.sh \ - $KAFKA_HOME/config/zookeeper.properties -``` -* Start Apache Kafka1 -```bash -$KAFKA_HOME/bin/kafka-server-start.sh \ - $KAFKA_HOME/config/server.properties -``` -* Start MoNanas with your configuration -```bash -python $MONANAS_HOME/run.py -p $SPARK_HOME -c \ - -l -``` -e.g. -```bash -python $HOME/monanas/run.py -p $HOME/spark -c $HOME/monanas/config/config.json \ - -l $HOME/monanas/config/logging.json -``` - -> Note: -1. Only when `KafkaSource` or `KafkaSink` is used in your processes. - -## Configurations -MoNanas consumes two configuration files, one for orchestrating data execution -and the other for logging. A Domain Specific Language (DSL) has been implemented in order to manipulate configurations. -Please, see [MoNanas/Configuration](configuration.md) for more details on MoNanas configuration; -and [MoNanas/Banana](banana.md) for more details on `banana` the language to write recipes. - -## Examples - -To run examples and see how MoNanas works step-by-step, see: [MoNanas/Examples](examples.md) diff --git a/doc/images/alert_fatigue_pipeline.png b/doc/images/alert_fatigue_pipeline.png deleted file mode 100644 index a9eea10..0000000 Binary files a/doc/images/alert_fatigue_pipeline.png and /dev/null differ diff --git a/doc/images/alert_fatigue_pipeline_with_markov.png b/doc/images/alert_fatigue_pipeline_with_markov.png deleted file mode 100644 index bc8b0cf..0000000 Binary files a/doc/images/alert_fatigue_pipeline_with_markov.png and /dev/null differ diff --git a/doc/images/architecture.png b/doc/images/architecture.png deleted file mode 100644 index 2a27463..0000000 Binary files a/doc/images/architecture.png and /dev/null differ diff --git a/doc/images/disk_failure_prediction.png b/doc/images/disk_failure_prediction.png deleted file mode 100644 index 1ca0d62..0000000 Binary files a/doc/images/disk_failure_prediction.png and /dev/null differ diff --git a/doc/images/disk_failure_prediction_with_data.png b/doc/images/disk_failure_prediction_with_data.png deleted file mode 100644 index f36249c..0000000 Binary files a/doc/images/disk_failure_prediction_with_data.png and /dev/null differ diff --git a/doc/images/got_tons_of_alarms.png b/doc/images/got_tons_of_alarms.png deleted file mode 100644 index aae9496..0000000 Binary files a/doc/images/got_tons_of_alarms.png and /dev/null differ diff --git a/doc/images/iptables_arch.png b/doc/images/iptables_arch.png deleted file mode 100755 index 629418f..0000000 Binary files a/doc/images/iptables_arch.png and /dev/null differ diff --git a/doc/images/iptables_src.png b/doc/images/iptables_src.png deleted file mode 100755 index 6369c83..0000000 Binary files a/doc/images/iptables_src.png and /dev/null differ diff --git a/doc/images/monanas-logo.png b/doc/images/monanas-logo.png deleted file mode 100644 index 35c918d..0000000 Binary files a/doc/images/monanas-logo.png and /dev/null differ diff --git a/doc/images/save_long_investigation_works.png b/doc/images/save_long_investigation_works.png deleted file mode 100644 index 1c8892d..0000000 Binary files a/doc/images/save_long_investigation_works.png and /dev/null differ diff --git a/doc/images/tags.png b/doc/images/tags.png deleted file mode 100644 index 1b46d5a..0000000 Binary files a/doc/images/tags.png and /dev/null differ diff --git a/doc/monasca.md b/doc/monasca.md deleted file mode 100644 index ad48e51..0000000 --- a/doc/monasca.md +++ /dev/null @@ -1,213 +0,0 @@ -# Integrate with Monasca -This section shows a step-by-step guide for integration MoNanas with Monasca. - -## Alarm management - -For example, let's say you have a some cloud system built on OpenStack, -and all services have been monitored by Monasca. - -Operators got tons of alarms when a trouble happened -even if there is actual trouble in one service. - -![Got tons of alarms](images/got_tons_of_alarms.png) - -Integrating MoNanas and Monasca eliminates unnecessary alarms. -And save the operator from long investigation works! - -![Save long investigation works](images/save_long_investigation_works.png) - - -### Running the MoNanas -To manage alarms from Monasca, perform the following steps. - - -#### Alarm Definition in Monasca. - -Refer to [Monanas API](https://github.com/openstack/monasca-api/blob/master/docs/monasca-api-spec.md). - -#### Configuration File in MoNanas - -Before running MoNanas, we need to create a configuration file describing how -we want MoNanas to orchestrate the data execution (creating a pipeline). - -You can edit configuration in `$MONANAS_HOME/config/markov_source_config.json` like the following. - -```json -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": false - }, - "sources": { - "src1": { - "module": "TextFileSource", - "params": { - "dir": "/tmp/source_data" - }, - "features": [ - "api_response.time.5sec.cinder1", - "api_response.time.5sec.cinder2", - "api_response.time.5sec.heat1", - "api_response.time.5sec.heat2", - "api_response.time.5sec.keystone1", - "api_response.time.5sec.keystone2", - "api_response.time.5sec.neutron1", - "api_response.time.5sec.neutron2", - "api_response.time.5sec.nova1", - "api_response.time.5sec.nova2" - ] - } - }, - "ingestors": { - "ing1": { - "module": "CloudIngestor" - } - }, - "smls": { - "sml1": { - "module": "LiNGAM", - "threshold": 0.5 - } - }, - "voters": { - "vot1": { - "module": "PickIndexVoter", - "index": 0 - } - }, - "sinks": { - "snk1": { - "module": "FileSink2", - "params": { - "dir": "/var/tmp/result/markov", - "file_prefix": "result" - } - }, - "snk2": { - "module": "StdoutSink" - } - }, - "ldps": { - "ldp1": { - "module": "CloudCausalityLDP" - } - }, - "connections": { - "src1": ["ing1", "ldp1"], - "sml1": ["vot1"], - "ing1": [], - "vot1": ["ldp1"], - "ldp1": ["snk1"], - "snk1": [], - "snk2": [] - }, - "feedback": {} -} -``` - -The flow of data execution is defined in `connections`. In this case, data -are ingested from `src1`. The data are then ingested by `ing1` where -each entry is converted into a format suitable for machine learning algorithm. -MoNanas uses `numpy.array` as a standard format. Typically, an aggregator is -responsible for aggregating data from different ingestors but in this scenario, -there is only one ingestor, hence the implicit aggregator (not defined in the -configuration) simply forwards the data to `sml1`, which simply deduplicates -the data then uses LiNGAM algorithm to find a causality structure of the -aggregated data then passes the result (structure) to `vot1`. The voter is -configured to pick the output of the first SML function and forwards that to -`ldp1`. Here, the live data processor transforms data streamed from `src1` -using the causality structure and pushes it to standard output as well as the -specified Kafka server. - -#### Run MoNanas - -As the sink of this orchestration is Kafka, we need to run Apache Zookeeper -and Apache Kafka first. - -```bash -$KAFKA_HOME/bin/zookeeper-server-start.sh \ - $KAFKA_HOME/config/zookeeper.properties - -$KAFKA_HOME/bin/kafka-server-start.sh \ - $KAFKA_HOME/config/server.properties -``` - -After that, start MoNanas as follows. - -```bash -python $MONANAS_HOME/run.py -p $SPARK_HOME -c $MONANAS_HOME/config/markov_source_config.json \ - -l $MONANAS_HOME/config/logging.json -``` - -A default logging configuration file is located in `$MONANAS_HOME/config`. -Users can override this option. Currently, `-l` option is mandatory as MoNanas -does not assume a default configuration. - -#### Start Data Execution - -MoNanas exposes REST API for controlling the data execution. Use any HTTP -client to POST with the following request body to MoNanas server to start -data streaming. - -```json -{ - "action": "start_streaming" -} -``` -e.g. using curl from terminal to make a request assuming the server is running -locally. -```bash -curl -H "Content-Type: application/json" -X POST \ - -d '{"action": "start_streaming"}' \ - http://localhost:3000/api/v1/actions -``` - -#### Results - -The sinks for the transformed (processed) alerts defined in the configuration -are via standard output and Kafka server. Therefore, the output will be -displayed in the console. Alternatively, users can subscribe to a queue -with the topic `transformed_alerts` using any Kafka client. A result example -(live data processing with aggregation mode based on causality structure) found -by LiNGAM algorithm is shown below. - -```json -{ - "alarm": { - "state": "ALARM", - "alarm_definition": { - "name": "api_response.time.5sec.nova2" - } - }, - "__monanas__": - [ - { - "causes": [ - "api_response.time.5sec.neutron2", - "api_response.time.5sec.cinder1", - "api_response.time.5sec.nova1", - "api_response.time.5sec.keystone1", - "api_response.time.5sec.keystone2" - ] - }, - { - "results": [ - "api_response.time.5sec.cinder2", - "api_response.time.5sec.heat1", - "api_response.time.5sec.heat2", - "api_response.time.5sec.nova1", - "api_response.time.5sec.neutron1", - "api_response.time.5sec.keystone1" - ] - } - ] -} -``` - - diff --git a/doc/use_cases.md b/doc/use_cases.md deleted file mode 100644 index 6134a25..0000000 --- a/doc/use_cases.md +++ /dev/null @@ -1,20 +0,0 @@ -# Example Use Cases - -Below are few use cases that are relevant to OpenStack. However, MoNanas -enables you to add your own [data ingestors](doc/dev_guide.md#ingestors). - -| Example | Alert Fatigue Management | Anomaly Detection | -|:------------------------------|:-------------------------|:------------------| -| **Dataset** | Synthetic, but representative, set of Monasca alerts that are processed in a stream manner. This alert set represents alerts that are seen in a data center consisting of several racks, enclosures and nodes. | `iptables` rules together with the number of times they are fired in a time period. | -| **Parsing** | Monasca alert parser. | Simple parser extracting period and number of fire events per rule. | -| **SML algorithm flow** | `filter(bad_formatted) -> filter(duplicates) -> aggregate() >> aggregator` aggregation can utilize conditional independence causality, score-based causality, linear algebra causality. | `detect_anomaly() >> aggregator` anomaly detection could be based on SVM, trend, etc. | -| **Output** | Directed acyclic alert graph with potential root causes at the top. | Rule set with an anomalous number of firing times in a time period. | -| **:information_source: Note** | Even though this could be consumed directly by devops, the usage of [Vitrage MoNanas Sink](doc/getting_started.md#vitrage_sink) is recommended. The output of this module can speed up creation of a [Vitrage](https://wiki.openstack.org/wiki/Vitrage) entity graph to do further analysis on it. | None. | - -`->` indicates a sequential operation in the flow. - -`//` indicates beginning of group of operations running in parallel. - -`-` indicates operations running in parallel. - -`>>` indicates end of group of operations running in parallel. diff --git a/fetch-deps.sh b/fetch-deps.sh deleted file mode 100644 index 300e123..0000000 --- a/fetch-deps.sh +++ /dev/null @@ -1,92 +0,0 @@ -# This file is intended to be used by vagrant only. -if [ "$VAGRANT_ENV" ]; then - - # Create a temporary folder - mkdir ~/tmp - cd ~/tmp - - # Java 8 - sudo -E apt-get update - sudo -E apt-get -y install openjdk-7-jre-headless openjdk-7-jdk - - # Maven - curl ftp://mirror.reverse.net/pub/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz > mvn.tar.gz - tar -xzf mvn.tar.gz - sudo mv apache-maven-3.3.9 /usr/local/apache-maven-3.3.9 - echo 'export PATH=/usr/local/apache-maven-3.3.9/bin:$PATH' >> $HOME/.profile - source $HOME/.profile - mkdir ~/.m2 - LENGTH_FOR_HOST=`expr match "$HTTP_PROXY" 'http://[\.A-Za-z\-]*'`-7 - echo ' - - - '" - true - http - ${HTTP_PROXY:7:$LENGTH_FOR_HOST} - 8080 - localhost - - " > ~/.m2/settings.xml - - - # Scala - curl https://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.deb > scala.deb - sudo dpkg -i scala.deb - - # Scala build tool - echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list - sudo -E apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 642AC823 - sudo -E apt-get update - sudo -E apt-get -y install sbt - - # Spark - curl http://apache.claz.org/spark/spark-1.6.1/spark-1.6.1.tgz > spark.tgz - echo "-------------------------" - echo "unzip spark to ~/spark" - mkdir ~/spark/ - tar -xzf spark.tgz - mv spark-1.6.1/ ~/spark/spark-1.6.1 - cd ~/spark/spark-1.6.1 - mvn -DskipTests clean package - cd ~/tmp - # config for logging in spark - cp ~/spark/spark-1.6.1/conf/log4j.properties.template ~/spark/spark-1.6.1/conf/log4j.properties - sed -i 's/log4j.rootCategory=INFO/log4j.rootCategory=ERROR/g' ~/spark/spark-1.6.1/conf/log4j.properties - - # Kafka - mkdir ~/kafka - curl http://apache.arvixe.com/kafka/0.9.0.0/kafka_2.11-0.9.0.0.tgz > kafka.tgz - echo "-------------------------" - echo "unzip kafka to ~/kafka" - tar -xzf kafka.tgz -C ~/kafka - mv ~/kafka/kafka_2.11-0.9.0.0/* ~/kafka - rm -r ~/kafka/kafka_2.11-0.9.0.0 - - # Python dependencies - sudo -E apt-get -y install python-pip python-setuptools - sudo -E pip install pep8 - - # Remove temporary folder - echo "-------------------------" - echo "Removing temporary folder" - rm -r ~/tmp - - # Monanas dependencies - echo "-------------------------" - echo "Get Monanas deps" - cd /vagrant - sudo -E apt-get -y install python-numpy python-scipy ipython - sudo -E -H python setup.py develop - - # Environment setup - set -v - echo 'export SPARK_HOME=~/spark/spark-1.6.1' >> $HOME/.profile - echo 'export KAFKA_HOME=~/kafka' >> $HOME/.profile - echo 'export PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.9-src.zip:$PYTHONPATH' >> $HOME/.profile - set +v - -else - echo "This file is intended to be used by a vagrant vm only." -fi diff --git a/monasca_analytics/__init__.py b/monasca_analytics/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/banana/README.md b/monasca_analytics/banana/README.md deleted file mode 100644 index 89f2028..0000000 --- a/monasca_analytics/banana/README.md +++ /dev/null @@ -1,40 +0,0 @@ -## Banana configuration language - -This module contains everything related to Banana. In each -sub-module (sub-folder) you will find a `README.md` file -that describes: - - * Purpose of the module. - * The current status of the implementation. - * How testing is done. - -The compiler is split in passes. Each pass performs some -transformations and / or generates more data. Only the last -step has side-effects on the Monanas instance. - -Each sub-module roughly maps to one pass run by the compiler. - -### Passes - -The Banana compiler runs the following passes: - - * `parse`, parse the input and build an [AST](./grammar/README.md). - * `typeck`, type check the input. - * `deadpathck`, remove dead path in the connections. - * `eval`, evaluate the AST generated. - -Each pass makes some assumptions about the state of -the data, and in particular that the previous passes -have run successfully. While this is made obvious by -the arguments required to run some passes, it is less -so for others. - -Generally, things to remember: - - * Changing the ordering of passes is more likely to - break things. - * New passes are free to modify the AST / TypeTable. - * New passes should not break invariants. - -For more information on passes, have a look in their -specific `README.md` file. \ No newline at end of file diff --git a/monasca_analytics/banana/__init__.py b/monasca_analytics/banana/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/banana/bytecode/__init__.py b/monasca_analytics/banana/bytecode/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/banana/bytecode/assembler.py b/monasca_analytics/banana/bytecode/assembler.py deleted file mode 100644 index 544c4b5..0000000 --- a/monasca_analytics/banana/bytecode/assembler.py +++ /dev/null @@ -1,1030 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This file is an adaptation of BytecodeAssembler: -# -# http://peak.telecommunity.com/DevCenter/BytecodeAssembler#toc -# -# It has been adapted to match the requirements of monasca_analytics - -import six -import sys - -from array import array -from dis import cmp_op -from dis import EXTENDED_ARG -from dis import hasfree -from dis import hasjabs -from dis import hasjrel -from dis import haslocal -from dis import hasname -from dis import HAVE_ARGUMENT -from dis import opname -from types import CodeType -from types import FunctionType - -from monasca_analytics.banana.bytecode.decorators import decorate_assignment -from monasca_analytics.banana.bytecode.symbols import Symbol - -opcode = {} -for op in range(256): - _name = opname[op] - if _name.startswith('<'): - continue - if _name.endswith('+0'): - opcode[_name[:-2]] = op - opcode[_name] = op - -# globals().update(opcode) # opcodes are now importable at will - -# Flags from code.h -CO_OPTIMIZED = 0x0001 # use LOAD/STORE_FAST instead of _NAME -CO_NEWLOCALS = 0x0002 # only cleared for module/exec code -CO_VARARGS = 0x0004 -CO_VARKEYWORDS = 0x0008 -CO_NESTED = 0x0010 # ??? -CO_GENERATOR = 0x0020 -CO_NOFREE = 0x0040 # set if no free or cell vars -CO_GENERATOR_ALLOWED = 0x1000 # unused -CO_FUTURE_DIVISION = 0x2000 -CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # Python 2.5+ only -CO_FUTURE_WITH_STATEMENT = 0x8000 # Python 2.5+ only - -# opcode -LOAD_DEREF = opcode["LOAD_DEREF"] -STORE_DEREF = opcode["STORE_DEREF"] -LOAD_FAST = opcode["LOAD_FAST"] -STORE_FAST = opcode["STORE_FAST"] -DELETE_FAST = opcode["DELETE_FAST"] -YIELD_VALUE = opcode["YIELD_VALUE"] -LOAD_CONST = opcode["LOAD_CONST"] -CALL_FUNCTION = opcode["CALL_FUNCTION"] -# CALL_FUNCTION_VAR = opcode["CALL_FUNCTION_VAR"] -CALL_FUNCTION_VAR = opcode["CALL_FUNCTION"] -CALL_FUNCTION_KW = opcode["CALL_FUNCTION_KW"] -# CALL_FUNCTION_VAR_KW = opcode["CALL_FUNCTION_VAR_KW"] -CALL_FUNCTION_VAR_KW = opcode["CALL_FUNCTION_KW"] -BUILD_TUPLE = opcode["BUILD_TUPLE"] -BUILD_LIST = opcode["BUILD_LIST"] -UNPACK_SEQUENCE = opcode["UNPACK_SEQUENCE"] -RETURN_VALUE = opcode["RETURN_VALUE"] -BUILD_SLICE = opcode["BUILD_SLICE"] -if six.PY2: - DUP_TOPX = opcode["DUP_TOPX"] -else: - # DUP_TOPX no longer in use from python3.3 - DUP_TOP_TWO = opcode["DUP_TOP_TWO"] - -RAISE_VARARGS = opcode["RAISE_VARARGS"] -MAKE_FUNCTION = opcode["MAKE_FUNCTION"] -# MAKE_CLOSURE = opcode["MAKE_CLOSURE"] -MAKE_CLOSURE = opcode["MAKE_FUNCTION"] -FOR_ITER = opcode["FOR_ITER"] -JUMP_IF_TRUE_OR_POP = opcode["JUMP_IF_TRUE_OR_POP"] -JUMP_IF_FALSE_OR_POP = opcode["JUMP_IF_FALSE_OR_POP"] -JUMP_FORWARD = opcode["JUMP_FORWARD"] -JUMP_ABSOLUTE = opcode["JUMP_ABSOLUTE"] -SETUP_EXCEPT = opcode["SETUP_EXCEPT"] -SETUP_FINALLY = opcode["SETUP_FINALLY"] -SETUP_LOOP = opcode["SETUP_LOOP"] -BREAK_LOOP = opcode["BREAK_LOOP"] -CONTINUE_LOOP = opcode["CONTINUE_LOOP"] -POP_BLOCK = opcode["POP_BLOCK"] -LIST_APPEND = opcode["LIST_APPEND"] -COMPARE_OP = opcode["COMPARE_OP"] -LOAD_NAME = opcode["LOAD_NAME"] -STORE_NAME = opcode["STORE_NAME"] -DELETE_NAME = opcode["DELETE_NAME"] - -fast_to_deref = { - LOAD_FAST: LOAD_DEREF, - STORE_FAST: STORE_DEREF, -} - -deref_to_deref = dict([(k, k) for k in hasfree]) - - -def with_name(f, name): - try: - f.__name__ = name - return f - except (TypeError, AttributeError): - return FunctionType( - f.func_code, f.func_globals, name, f.func_defaults, f.func_closure - ) - - -class Const(object): - """Wrapper to ensure constants are hashable even if mutable""" - - __slots__ = 'value', 'hash', 'hashable' - - def __init__(self, value_): - self.value = value_ - try: - self.hash = hash(value_) - except TypeError: - self.hash = hash(id(value_)) - self.hashable = False - else: - self.hashable = True - - def __repr__(self): - return "Const(%s)" % repr(self.value) - - def __hash__(self): - return self.hash - - def __eq__(self, other): - if type(other) is not Const: - return False - if self.hashable: - return self.value == other.value - else: - return self.value is other.value - - def __ne__(self, other): - return not self == other - - def __call__(self, code): - code.LOAD_CONST(self.value) - - -class Node(tuple): - """Base class for AST nodes""" - __slots__ = [] - - -def nodetype(*mixins, **kw): - - def callback(frame, name, func, old_locals): - def __new__(cls, *args, **kw): - result = func(*args, **kw) - if type(result) is tuple: - return tuple.__new__(cls, (cls,) + result) - else: - return result - - def __repr__(self): - r = self.__class__.__name__ + tuple.__repr__(self[1:]) - if len(self) == 2: - return r[:-2] + ')' # nix trailing ',' - return r - - def __call__(self, code): - return func(*(self[1:] + (code,))) - - import inspect - args = inspect.getargspec(func)[0] - - d = dict( - __new__=__new__, __repr__=__repr__, __doc__=func.__doc__, - __module__=func.__module__, __args__=args, __slots__=[], - __call__=__call__ - ) - for p, a in enumerate(args[:-1]): # skip 'code' argument - if isinstance(a, str): - d[a] = property(lambda self, p=p + 1: self[p]) - - d.update(kw) - return type(name, mixins + (Node,), d) - - return decorate_assignment(callback) - - -@nodetype() -def Local(name, code=None): - if code is None: - return name, - if name in code.co_cellvars or name in code.co_freevars: - return code.LOAD_DEREF(name) - elif code.co_flags & CO_OPTIMIZED: - return code.LOAD_FAST(name) - else: - return code.LOAD_NAME(name) - - -@nodetype() -def Getattr(ob, name, code=None): - try: - name = const_value(name) - except NotAConstant: - return Call(Const(getattr), [ob, name]) - if code is None: - return fold_args(Getattr, ob, name) - code(ob) - code.LOAD_ATTR(name) - - -@nodetype() -def Call(fn, args=(), kwargs=(), star=None, dstar=None, fold=True, code=None): - if code is None: - data = ( - fn, tuple(args), tuple(kwargs), star or (), dstar or (), fold - ) - if fold and (args or kwargs or star or dstar): - return fold_args(Call, *data) - else: - return data - - code(fn, *args) - for k, v in kwargs: - code(k, v) - - argc = len(args) - kwargc = len(kwargs) - - if star: - if dstar: - code(star, dstar) - return code.CALL_FUNCTION_VAR_KW(argc, kwargc) - else: - code(star) - return code.CALL_FUNCTION_VAR(argc, kwargc) - else: - if dstar: - code(dstar) - return code.CALL_FUNCTION_KW(argc, kwargc) - else: - return code.CALL_FUNCTION(argc, kwargc) - - -class _Pass(Symbol): - def __call__(self, code=None): - pass - - def __nonzero__(self): - return False -Pass = _Pass('Pass', __name__) - - -class NotAConstant(Exception): - """The supplied value is not a constant expression tree""" - - -def const_value(value_): - """Return the constant value -- if any -- of an expression tree - - Raises NotAConstant if the value or any child of the value are - not constants. - """ - t = type(value_) - if t is Const: - value_ = value_.value - elif t is tuple: - t = tuple(map(const_value, value_)) - if t == value_: - return value_ - return t - elif generate_types.get(t) != Code.LOAD_CONST: - raise NotAConstant(value_) - return value_ - - -def fold_args(f, *args): - """Return a folded ``Const`` or an argument tuple""" - - try: - for arg in args: - if arg is not Pass and arg is not None: - const_value(arg) - except NotAConstant: - return args - else: - c = Code() - f(*args + (c,)) - c.RETURN_VALUE() - return Const(eval(c.code())) - - -@nodetype() -def LocalAssign(name, code=None): - if code is None: - return name, - if name in code.co_cellvars or name in code.co_freevars: - return code.STORE_DEREF(name) - elif code.co_flags & CO_OPTIMIZED: - return code.STORE_FAST(name) - else: - return code.STORE_NAME(name) - - -class Code(object): - co_argcount = 0 - co_stacksize = 0 - co_flags = CO_OPTIMIZED | CO_NEWLOCALS # typical usage - co_filename = '' - co_name = '' - co_firstlineno = 0 - co_freevars = () - co_cellvars = () - _last_lineofs = 0 - _ss = 0 - _tmp_level = 0 - - def __init__(self): - self.co_code = array('B') - self.co_consts = [None] - self.co_names = [] - self.co_varnames = [] - self.co_lnotab = array('B') - self.emit = self.co_code.append - self.blocks = [] - self.stack_history = [] - - def emit_arg(self, op, arg): - emit = self.emit - if arg > 0xFFFF: - emit(EXTENDED_ARG) - emit((arg >> 16) & 255) - emit((arg >> 24) & 255) - emit(op) - emit(arg & 255) - emit((arg >> 8) & 255) - - def locals_written(self): - vn = self.co_varnames - hl = dict.fromkeys([STORE_FAST, DELETE_FAST]) - return dict.fromkeys([vn[arg] for ofs, op, arg in self if op in hl]) - - def set_lineno(self, lno): - if not self.co_firstlineno: - self.co_firstlineno = self._last_line = lno - return - - append = self.co_lnotab.append - incr_line = lno - self._last_line - incr_addr = len(self.co_code) - self._last_lineofs - if not incr_line: - return - - assert incr_addr >= 0 and incr_line >= 0 - - while incr_addr > 255: - append(255) - append(0) - incr_addr -= 255 - - while incr_line > 255: - append(incr_addr) - append(255) - incr_line -= 255 - incr_addr = 0 - - if incr_addr or incr_line: - append(incr_addr) - append(incr_line) - - self._last_line = lno - self._last_lineofs = len(self.co_code) - - def YIELD_VALUE(self): - self.stackchange(stack_effects[YIELD_VALUE]) - self.co_flags |= CO_GENERATOR - return self.emit(YIELD_VALUE) - - def LOAD_CONST(self, const): - self.stackchange(0, 1) - pos = 0 - hashable = True - try: - hash(const) - except TypeError: - hashable = False - while 1: - try: - arg = self.co_consts.index(const, pos) - it = self.co_consts[arg] - except ValueError: - arg = len(self.co_consts) - self.co_consts.append(const) - break - else: - if type(it) is type(const) and (hashable or it is const): - break - pos = arg + 1 - continue - return self.emit_arg(LOAD_CONST, arg) - - def CALL_FUNCTION(self, argc=0, kwargc=0, op=CALL_FUNCTION, extra=0): - self.stackchange(1 + argc + 2 * kwargc + extra, 1) - emit = self.emit - emit(op) - emit(argc) - emit(kwargc) - - def CALL_FUNCTION_VAR(self, argc=0, kwargc=0): - self.CALL_FUNCTION( - argc, - kwargc, - CALL_FUNCTION_VAR, - 1) # 1 for *args - - def CALL_FUNCTION_KW(self, argc=0, kwargc=0): - self.CALL_FUNCTION( - argc, - kwargc, - CALL_FUNCTION_KW, - 1) # 1 for **kw - - def CALL_FUNCTION_VAR_KW(self, argc=0, kwargc=0): - self.CALL_FUNCTION( - argc, - kwargc, - CALL_FUNCTION_VAR_KW, - 2) # 2 *args,**kw - - def BUILD_TUPLE(self, count): - self.stackchange(count, 1) - self.emit_arg(BUILD_TUPLE, count) - - def BUILD_LIST(self, count): - self.stackchange(count, 1) - self.emit_arg(BUILD_LIST, count) - - def UNPACK_SEQUENCE(self, count): - self.stackchange(1, count) - self.emit_arg(UNPACK_SEQUENCE, count) - - def RETURN_VALUE(self): - self.stackchange(1, 0) - self.emit(RETURN_VALUE) - self.stack_unknown() - - def BUILD_SLICE(self, count): - assert count in (2, 3), "Invalid number of arguments for BUILD_SLICE" - self.stackchange(count, 1) - self.emit_arg(BUILD_SLICE, count) - - if six.PY2: - def DUP_TOPX(self, count): - self.stackchange(count, count * 2) - self.emit_arg(DUP_TOPX, count) - else: - def DUP_TOP_TWO(self, count): - self.stackchange(count, count * 2) - self.emit_arg(DUP_TOP_TWO, count) - - def RAISE_VARARGS(self, argc): - assert 0 <= argc <= 3, "Invalid number of arguments for RAISE_VARARGS" - self.stackchange(argc, 0) - self.emit_arg(RAISE_VARARGS, argc) - - def MAKE_FUNCTION(self, ndefaults): - self.stackchange(1 + ndefaults, 1) - self.emit_arg(MAKE_FUNCTION, ndefaults) - - def MAKE_CLOSURE(self, ndefaults, freevars): - if sys.version >= '2.5': - freevars = 1 - self.stackchange(1 + freevars + ndefaults, 1) - self.emit_arg(MAKE_CLOSURE, ndefaults) - - def here(self): - return len(self.co_code) - - def set_stack_size(self, size): - if size < 0: - raise AssertionError("Stack underflow") - if size > self.co_stacksize: - self.co_stacksize = size - bytes = len(self.co_code) - len(self.stack_history) + 1 - if bytes > 0: - self.stack_history.extend([self._ss] * bytes) - self._ss = size - - def get_stack_size(self): - return self._ss - - stack_size = property(get_stack_size, set_stack_size) - - def stackchange(self, inputs, outputs): - if self._ss is None: - raise AssertionError("Unknown stack size at this location") - self.stack_size -= inputs # check underflow - self.stack_size += outputs # update maximum height - - def stack_unknown(self): - self._ss = None - - def branch_stack(self, location, expected): - if location >= len(self.stack_history): - if location > len(self.co_code): - raise AssertionError("Forward-looking stack prediction!", - location, len(self.co_code) - ) - actual = self.stack_size - if actual is None: - self.stack_size = actual = expected - self.stack_history[location] = actual - else: - actual = self.stack_history[location] - if actual is None: - self.stack_history[location] = actual = expected - - if actual != expected: - raise AssertionError( - "Stack level mismatch: actual=%s expected=%s" - % (actual, expected) - ) - - def jump(self, op, arg=None): - def jump_target(offset): - target = offset - if op not in hasjabs: - target = target - (posn + 3) - if target < 0: - raise AssertionError("Relative jumps can't go backwards") - if target > 0xFFFF: - target = offset - (posn + 6) - return target - - def backpatch(offset): - target = jump_target(offset) - if target > 0xFFFF: - raise AssertionError("Forward jump span must be <64K bytes") - self.patch_arg(posn, 0, target) - self.branch_stack(offset, old_level) - - if op == FOR_ITER: - old_level = self.stack_size = self.stack_size - 1 - self.stack_size += 2 - else: - old_level = self.stack_size - self.stack_size -= (op in (JUMP_IF_TRUE_OR_POP, JUMP_IF_FALSE_OR_POP)) - posn = self.here() - if arg is not None: - self.emit_arg(op, jump_target(arg)) - self.branch_stack(arg, old_level) - lbl = None - else: - self.emit_arg(op, 0) - - def lbl(code=None): - backpatch(self.here()) - if op in (JUMP_FORWARD, JUMP_ABSOLUTE, CONTINUE_LOOP): - self.stack_unknown() - return lbl - - def COMPARE_OP(self, op): - self.stackchange(2, 1) - self.emit_arg(COMPARE_OP, compares[op]) - - def setup_block(self, op): - jmp = self.jump(op) - self.blocks.append((op, self.stack_size, jmp)) - return jmp - - def SETUP_EXCEPT(self): - ss = self.stack_size - self.stack_size = ss + 3 # simulate the level at "except:" time - self.setup_block(SETUP_EXCEPT) - self.stack_size = ss # restore the current level - - def SETUP_FINALLY(self): - ss = self.stack_size - self.stack_size = ss + 3 # allow for exceptions - self.stack_size = ss + 1 # simulate the level after the None is pushed - self.setup_block(SETUP_FINALLY) - self.stack_size = ss # restore original level - - def SETUP_LOOP(self): - self.setup_block(SETUP_LOOP) - - def POP_BLOCK(self): - if not self.blocks: - raise AssertionError("Not currently in a block") - - why, level, fwd = self.blocks.pop() - self.emit(POP_BLOCK) - - if why != SETUP_LOOP: - if why == SETUP_FINALLY: - self.LOAD_CONST(None) - fwd() - else: - self.stack_size = level - 3 # stack level resets here - else_ = self.JUMP_FORWARD() - fwd() - return else_ - else: - return fwd - - if 'JUMP_IF_TRUE_OR_POP' not in opcode: - def JUMP_IF_TRUE_OR_POP(self, address=None): - lbl = self.JUMP_IF_TRUE(address) - self.POP_TOP() - return lbl - - globals()['JUMP_IF_TRUE_OR_POP'] = -1 - - if 'JUMP_IF_FALSE_OR_POP' not in opcode: - def JUMP_IF_FALSE_OR_POP(self, address=None): - lbl = self.JUMP_IF_FALSE(address) - self.POP_TOP() - return lbl - - globals()['JUMP_IF_FALSE_OR_POP'] = -1 - - if 'JUMP_IF_TRUE' not in opcode: - def JUMP_IF_TRUE(self, address=None): - self.DUP_TOP() - return self.POP_JUMP_IF_TRUE(address) - else: - globals()['POP_JUMP_IF_TRUE'] = -1 - - if 'JUMP_IF_FALSE' not in opcode: - def JUMP_IF_FALSE(self, address=None): - self.DUP_TOP() - return self.POP_JUMP_IF_FALSE(address) - else: - globals()['POP_JUMP_IF_FALSE'] = -1 - - if 'LIST_APPEND' in opcode and LIST_APPEND >= HAVE_ARGUMENT: - def LIST_APPEND(self, depth): - self.stackchange(depth + 1, depth) - self.emit_arg(LIST_APPEND, depth) - - def assert_loop(self): - for why, level, fwd in self.blocks: - if why == SETUP_LOOP: - return - raise AssertionError("Not inside a loop") - - def BREAK_LOOP(self): - self.assert_loop() - self.emit(BREAK_LOOP) - self.stack_unknown() - - def CONTINUE_LOOP(self, label): - self.assert_loop() - if self.blocks[-1][0] == SETUP_LOOP: - op = JUMP_ABSOLUTE # more efficient if not in a nested block - else: - op = CONTINUE_LOOP - return self.jump(op, label) - - def __call__(self, *args): - last = None - for ob in args: - if callable(ob): - last = ob(self) - else: - try: - f = generate_types[type(ob)] - except KeyError: - raise TypeError("Can't generate", ob) - else: - last = f(self, ob) - return last - - def return_(self, ob=None): - return self(ob, Code.RETURN_VALUE) - - @classmethod - def from_function(cls, function, copy_lineno=False): - code = cls.from_code(function.func_code, copy_lineno) - return code - - @classmethod - def from_code(cls, code, copy_lineno=False): - import inspect - self = cls.from_spec(code.co_name, *inspect.getargs(code)) - if copy_lineno: - self.set_lineno(code.co_firstlineno) - self.co_filename = code.co_filename - self.co_freevars = code.co_freevars # XXX untested! - return self - - @classmethod - def from_spec(cls, name='', args=(), var=None, kw=None): - self = cls() - self.co_name = name - self.co_argcount = len(args) - self.co_varnames.extend(args) - if var: - self.co_varnames.append(var) - self.co_flags |= CO_VARARGS - if kw: - self.co_varnames.append(kw) - self.co_flags |= CO_VARKEYWORDS - - def tuple_arg(args): - self.UNPACK_SEQUENCE(len(args)) - for arg in args: - if not isinstance(arg, six.string_types): - tuple_arg(arg) - else: - self.STORE_FAST(arg) - - for narg, arg in enumerate(args): - if not isinstance(arg, six.string_types): - dummy_name = '.' + str(narg) - self.co_varnames[narg] = dummy_name - self.LOAD_FAST(dummy_name) - tuple_arg(arg) - - return self - - def patch_arg(self, offset, oldarg, newarg): - code = self.co_code - if (oldarg > 0xFFFF) != (newarg > 0xFFFF): - raise AssertionError("Can't change argument size", oldarg, newarg) - code[offset + 1] = newarg & 255 - code[offset + 2] = (newarg >> 8) & 255 - if newarg > 0xFFFF: - newarg >>= 16 - code[offset - 2] = newarg & 255 - code[offset - 1] = (newarg >> 8) & 255 - - def nested(self, name='', args=(), var=None, kw=None, cls=None): - if cls is None: - cls = Code - code = cls.from_spec(name, args, var, kw) - code.co_filename = self.co_filename - return code - - def __iter__(self): - i = 0 - extended_arg = 0 - code = self.co_code - n = len(code) - while i < n: - op = code[i] - if op >= HAVE_ARGUMENT: - oparg = code[i + 1] + code[i + 2] * 256 + extended_arg - extended_arg = 0 - if op == EXTENDED_ARG: - extended_arg = oparg * 65536 - i += 3 - continue - yield i, op, oparg - i += 3 - else: - yield i, op, None - i += 1 - - def makefree(self, names): - nowfree = dict.fromkeys(self.co_freevars) - newfree = [n for n in names if n not in nowfree] - if newfree: - self.co_freevars += tuple(newfree) - self._locals_to_cells() - - def makecells(self, names): - nowcells = dict.fromkeys(self.co_cellvars + self.co_freevars) - newcells = [n for n in names if n not in nowcells] - if newcells: - if not (self.co_flags & CO_OPTIMIZED): - raise AssertionError("Can't use cellvars in unoptimized scope") - cc = len(self.co_cellvars) - nc = len(newcells) - self.co_cellvars += tuple(newcells) - if self.co_freevars: - self._patch( - deref_to_deref, - dict([(n + cc, n + cc + nc) for n in range( - len(self.co_freevars))]) - ) - self._locals_to_cells() - - def _locals_to_cells(self): - freemap = dict( - [(n, p) for p, n in enumerate(self.co_cellvars + self.co_freevars)] - ) - argmap = dict( - [(p, freemap[n]) for p, n in enumerate(self.co_varnames) - if n in freemap] - ) - if argmap: - for ofs, op, arg in self: - if op == DELETE_FAST and arg in argmap: - raise AssertionError( - "Can't delete local %r used in nested scope" - % self.co_varnames[arg] - ) - self._patch(fast_to_deref, argmap) - - def _patch(self, opmap, argmap=None): - if argmap is None: - argmap = {} - code = self.co_code - for ofs, op, arg in self: - if op in opmap: - if arg in argmap: - self.patch_arg(ofs, arg, argmap[arg]) - elif arg is not None: - continue - code[ofs] = opmap[op] - - def code(self, parent=None): - if self.blocks: - raise AssertionError("%d unclosed block(s)" % len(self.blocks)) - - flags = self.co_flags & ~CO_NOFREE - if parent is not None: - locals_written = self.locals_written() - self.makefree([ - n for n in self.co_varnames[ - self.co_argcount + - ((self.co_flags & CO_VARARGS) == CO_VARARGS) + - ((self.co_flags & CO_VARKEYWORDS) == CO_VARKEYWORDS): - ] if n not in locals_written - ]) - - if not self.co_freevars and not self.co_cellvars: - flags |= CO_NOFREE - elif parent is not None and self.co_freevars: - parent.makecells(self.co_freevars) - - if six.PY2: - return CodeType( - self.co_argcount, len(self.co_varnames), - self.co_stacksize, flags, self.co_code.tostring(), - tuple(self.co_consts), tuple(self.co_names), - tuple(self.co_varnames), - self.co_filename, self.co_name, self.co_firstlineno, - self.co_lnotab.tostring(), self.co_freevars, self.co_cellvars - ) - else: - kwonlyargcount = 0 - return CodeType( - self.co_argcount, kwonlyargcount, len(self.co_varnames), - self.co_stacksize, flags, self.co_code.tobytes(), - tuple(self.co_consts), tuple(self.co_names), - tuple(self.co_varnames), - self.co_filename, self.co_name, self.co_firstlineno, - self.co_lnotab.tobytes(), self.co_freevars, self.co_cellvars - ) - -for op in hasfree: - if not hasattr(Code, opname[op]): - def do_free(self, varname, op=op): - self.stackchange(stack_effects[op][0], stack_effects[op][1]) - try: - arg = list(self.co_cellvars + self.co_freevars).index(varname) - except ValueError: - raise NameError("Undefined free or cell var", varname) - self.emit_arg(op, arg) - setattr(Code, opname[op], with_name(do_free, opname[op])) - -compares = {} -for value, cmp_name in enumerate(cmp_op): - compares[value] = value - compares[cmp_name] = value -compares['<>'] = compares['!='] - -for op in hasname: - if not hasattr(Code, opname[op]): - def do_name(self, name, op=op): - self.stackchange(stack_effects[op][0], stack_effects[op][1]) - try: - arg = self.co_names.index(name) - except ValueError: - arg = len(self.co_names) - self.co_names.append(name) - self.emit_arg(op, arg) - if op in (LOAD_NAME, STORE_NAME, DELETE_NAME): - # Can't use optimized local vars, so reset flags - self.co_flags &= ~CO_OPTIMIZED - setattr(Code, opname[op], with_name(do_name, opname[op])) - -for op in haslocal: - if not hasattr(Code, opname[op]): - def do_local(self, varname, op=op): - if not self.co_flags & CO_OPTIMIZED: - raise AssertionError( - "co_flags must include CO_OPTIMIZED to use fast locals" - ) - self.stackchange(stack_effects[op][0], stack_effects[op][1]) - try: - arg = self.co_varnames.index(varname) - except ValueError: - arg = len(self.co_varnames) - self.co_varnames.append(varname) - self.emit_arg(op, arg) - setattr(Code, opname[op], with_name(do_local, opname[op])) - -for op in hasjrel + hasjabs: - if not hasattr(Code, opname[op]): - def do_jump(self, address=None, op=op): - self.stackchange(stack_effects[op][0], stack_effects[op][1]) - return self.jump(op, address) - setattr(Code, opname[op], with_name(do_jump, opname[op])) - - -def gen_map(code, ob): - code.BUILD_MAP(0) - for k, v in ob.items(): - code.DUP_TOP() - code(k, v) - code.ROT_THREE() - code.STORE_SUBSCR() - - -def gen_tuple(code, ob): - code(*ob) - return code.BUILD_TUPLE(len(ob)) - - -def gen_list(code, ob): - code(*ob) - return code.BUILD_LIST(len(ob)) - -generate_types = { - int: Code.LOAD_CONST, - bool: Code.LOAD_CONST, - CodeType: Code.LOAD_CONST, - str: Code.LOAD_CONST, - six.text_type: Code.LOAD_CONST, - complex: Code.LOAD_CONST, - float: Code.LOAD_CONST, - type(None): Code.LOAD_CONST, - tuple: gen_tuple, - list: gen_list, - dict: gen_map, -} -if six.PY2: - generate_types[long] = Code.LOAD_CONST - - -class _se(object): - """Quick way of defining static stack effects of opcodes""" - POP_TOP = END_FINALLY = POP_JUMP_IF_FALSE = POP_JUMP_IF_TRUE = 1, 0 - ROT_TWO = 2, 2 - ROT_THREE = 3, 3 - ROT_FOUR = 4, 4 - DUP_TOP = 1, 2 - UNARY_POSITIVE = UNARY_NEGATIVE = UNARY_NOT = UNARY_CONVERT = \ - UNARY_INVERT = GET_ITER = LOAD_ATTR = IMPORT_FROM = 1, 1 - - BINARY_POWER = BINARY_MULTIPLY = BINARY_DIVIDE = BINARY_FLOOR_DIVIDE = \ - BINARY_TRUE_DIVIDE = BINARY_MODULO = BINARY_ADD = BINARY_SUBTRACT = \ - BINARY_SUBSCR = BINARY_LSHIFT = BINARY_RSHIFT = BINARY_AND = \ - BINARY_XOR = BINARY_OR = COMPARE_OP = 2, 1 - - INPLACE_POWER = INPLACE_MULTIPLY = INPLACE_DIVIDE = \ - INPLACE_FLOOR_DIVIDE = INPLACE_TRUE_DIVIDE = INPLACE_MODULO = \ - INPLACE_ADD = INPLACE_SUBTRACT = INPLACE_LSHIFT = INPLACE_RSHIFT = \ - INPLACE_AND = INPLACE_XOR = INPLACE_OR = 2, 1 - - SLICE_0, SLICE_1, SLICE_2, SLICE_3 = \ - (1, 1), (2, 1), (2, 1), (3, 1) - STORE_SLICE_0, STORE_SLICE_1, STORE_SLICE_2, STORE_SLICE_3 = \ - (2, 0), (3, 0), (3, 0), (4, 0) - DELETE_SLICE_0, DELETE_SLICE_1, DELETE_SLICE_2, DELETE_SLICE_3 = \ - (1, 0), (2, 0), (2, 0), (3, 0) - - STORE_SUBSCR = 3, 0 - DELETE_SUBSCR = STORE_ATTR = 2, 0 - DELETE_ATTR = STORE_DEREF = 1, 0 - PRINT_EXPR = PRINT_ITEM = PRINT_NEWLINE_TO = IMPORT_STAR = 1, 0 - RETURN_VALUE = YIELD_VALUE = STORE_NAME = STORE_GLOBAL = STORE_FAST = 1, 0 - PRINT_ITEM_TO = LIST_APPEND = 2, 0 - - LOAD_LOCALS = LOAD_CONST = LOAD_NAME = LOAD_GLOBAL = LOAD_FAST = \ - LOAD_CLOSURE = LOAD_DEREF = IMPORT_NAME = BUILD_MAP = 0, 1 - - EXEC_STMT = BUILD_CLASS = 3, 0 - JUMP_IF_TRUE = JUMP_IF_FALSE = \ - JUMP_IF_TRUE_OR_POP = JUMP_IF_FALSE_OR_POP = 1, 1 - - -if sys.version >= "2.5": - _se.YIELD_VALUE = 1, 1 - -stack_effects = [(0, 0)] * 256 - -for opcode_name in opcode: - op = opcode[opcode_name] - opcode_name = opcode_name.replace('+', '_') - - if hasattr(_se, opcode_name): - # update stack effects table from the _se class - stack_effects[op] = getattr(_se, opcode_name) - - if not hasattr(Code, opcode_name): - # Create default method for Code class - if op >= HAVE_ARGUMENT: - def do_op(self, arg, op=op, se=stack_effects[op]): - self.stackchange(se[0], se[1]) - self.emit_arg(op, arg) - else: - def do_op(self, op=op, se=stack_effects[op]): - self.stackchange(se[0], se[1]) - self.emit(op) - - setattr(Code, opcode_name, with_name(do_op, opcode_name)) diff --git a/monasca_analytics/banana/bytecode/decorators.py b/monasca_analytics/banana/bytecode/decorators.py deleted file mode 100644 index b5a3475..0000000 --- a/monasca_analytics/banana/bytecode/decorators.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This file is an adaptation of BytecodeAssembler: -# -# http://peak.telecommunity.com/DevCenter/BytecodeAssembler#toc -# -# It has been adapted to match the requirements of monasca_analytics - -import sys - - -def enclosing_frame(frame=None, level=3): - """Get an enclosing frame that skips DecoratorTools callback code""" - frame = frame or sys._getframe(level) - while frame.f_globals.get('__name__') == __name__: - frame = frame.f_back - return frame - - -def decorate_assignment(callback, depth=2, frame=None): - """Invoke 'callback(frame,name,value,old_locals)' on next assign in 'frame' - - The frame monitored is determined by the 'depth' argument, which gets - passed to 'sys._getframe()'. When 'callback' is invoked, 'old_locals' - contains a copy of the frame's local variables as they were before the - assignment took place, allowing the callback to access the previous value - of the assigned variable, if any. The callback's return value will become - the new value of the variable. 'name' is the name of the variable being - created or modified, and 'value' is its value (the same as - 'frame.f_locals[name]'). - - This function also returns a decorator function for forward-compatibility - with Python 2.4 '@' syntax. Note, however, that if the returned decorator - is used with Python 2.4 '@' syntax, the callback 'name' argument may be - 'None' or incorrect, if the 'value' is not the original function (e.g. - when multiple decorators are used). - """ - frame = enclosing_frame(frame, depth + 1) - oldtrace = [frame.f_trace] - old_locals = frame.f_locals.copy() - - def tracer(frm, event, arg): - if event == 'call': - # We don't want to trace into any calls - if oldtrace[0]: - # ...but give the previous tracer a chance to, if it wants - return oldtrace[0](frm, event, arg) - else: - return None - - try: - if frm is frame and event != 'exception': - # Aha, time to check for an assignment... - for k, v in frm.f_locals.items(): - if k not in old_locals or old_locals[k] is not v: - break - else: - # No luck, keep tracing - return tracer - - # Got it, fire the callback, then get the heck outta here... - frm.f_locals[k] = callback(frm, k, v, old_locals) - - finally: - # Give the previous tracer a chance to run before we return - if oldtrace[0]: - # And allow it to replace our idea of the "previous" tracer - oldtrace[0] = oldtrace[0](frm, event, arg) - - uninstall() - return oldtrace[0] - - def uninstall(): - # Unlink ourselves from the trace chain. - frame.f_trace = oldtrace[0] - sys.settrace(oldtrace[0]) - - # Install the trace function - frame.f_trace = tracer - sys.settrace(tracer) - - def do_decorate(f): - # Python 2.4 '@' compatibility; call the callback - uninstall() - frame = sys._getframe(1) - return callback( - frame, getattr(f, '__name__', None), f, frame.f_locals - ) - - return do_decorate diff --git a/monasca_analytics/banana/bytecode/symbols.py b/monasca_analytics/banana/bytecode/symbols.py deleted file mode 100644 index d3df7f6..0000000 --- a/monasca_analytics/banana/bytecode/symbols.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This file is an adaptation of BytecodeAssembler: -# -# http://peak.telecommunity.com/DevCenter/BytecodeAssembler#toc -# -# It has been adapted to match the requirements of monasca_analytics - -"""Symbolic global constants, like 'None', 'NOT_FOUND', etc.""" - - -class Symbol(object): - - """Symbolic global constant""" - - __slots__ = ['_name', '_module'] - __name__ = property(lambda self: self._name) - __module__ = property(lambda self: self._module) - - def __init__(self, symbol, module_name): - self.__class__._name.__set__(self, symbol) - self.__class__._module.__set__(self, module_name) - - def __reduce__(self): - return self._name - - def __setattr__(self, attr, val): - raise TypeError("Symbols are immutable") - - def __repr__(self): - return self.__name__ - - __str__ = __repr__ - - -NOT_GIVEN = Symbol("NOT_GIVEN", __name__) -NOT_FOUND = Symbol("NOT_FOUND", __name__) diff --git a/monasca_analytics/banana/cli/__init__.py b/monasca_analytics/banana/cli/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/banana/cli/const.py b/monasca_analytics/banana/cli/const.py deleted file mode 100644 index ccf2286..0000000 --- a/monasca_analytics/banana/cli/const.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -CREATE = "create" -CONNECT = "connect" -DISCONNECT = "disconnect" -LOAD = "load" -SAVE_AS = "save_as" -SAVE = "save" -REMOVE = "remove" -MODIFY = "modify" -PRINT = "print" -LIST = "list" -HELP = "help" diff --git a/monasca_analytics/banana/cli/dsl.py b/monasca_analytics/banana/cli/dsl.py deleted file mode 100644 index a58f84d..0000000 --- a/monasca_analytics/banana/cli/dsl.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import json -import logging -import os - -import six -import voluptuous - -from monasca_analytics.config import const -from monasca_analytics.config import validation -from monasca_analytics.exception import dsl as err -import monasca_analytics.util.common_util as cu - - -logger = logging.getLogger(__name__) - -MODULE = "module" - -""" -DEPRECATED: Preferred way is to now use the Banana language instead. -""" - - -class MonanasDSL(object): - - def __init__(self, config_file_path=None): - """Constructor with an optional configuration file path - - If the configuration file is provided, it will be loaded at - MonanasDLS creation time. - If no configuration file path is provided, the default base - configuration defined in config.const will be used. - - :type config_file_path: str - :param config_file_path: path to the file containing the configuration - that will be loaded at MonansDSL creation - """ - if config_file_path: - self.load_configuration(config_file_path) - else: - self._config = const.get_default_base_config() - self._init_ids_dictionary() - - def add_component(self, _component_config): - """Add the component configuration passed as parameter - - Add it to MonanasDSL configuration, and return a new unique ID - generated for it. - The configuration passed as parameter is validated, raising exceptions - if the module does not exist or the configuration is invalid. - - :type _component_config: dict - :param _component_config: configuration of the component to be added - :rtype: str - :returns: Component ID for the added component - :raises: MonanasNoSuchClassError -- if the defined class doesn't - exist or is not of a valid type - :raises: SchemaError -- if the configuration is not valid for - the class. - """ - if type(_component_config) == str: - _component_config = json.loads(_component_config) - clz = cu.get_class_by_name(_component_config[MODULE]) - clz.validate_config(_component_config) - comp_type = cu.get_component_type(_component_config[MODULE]) - comp_id = self._generate_id(comp_type) - self._config[comp_type][comp_id] = _component_config - self._config[const.CONNECTIONS][comp_id] = [] - return comp_id - - def modify_component(self, comp_id, params_path, value): - """Overrides the value of the configuration path of a component - - Modifies the configuration of the component defined by comp_id, - following the path in the dictionary defined by params_path, and - assigning the value value. - - :type comp_id: str - :param comp_id: ID of the component to be modified - :type params_path: list - :param params_path: parameters path to modify in the config - :type value: str | int | float - :param value: new value to be assigned, will be parsed - according to the expected configuration - :rtype: bool - :returns: True if the component was modified (or if the modification - result was the same as the existing configuration), - False otherwise - :raises: SchemaError -- if the new configuration would not be valid - """ - comp_type = self._get_type_by_id(comp_id) - if not comp_type: - return False - new_conf = copy.deepcopy(self._config[comp_type][comp_id]) - logger.debug("Modifying " + comp_id + ", existing config = " + - str(new_conf)) - clz = cu.get_class_by_name(new_conf[MODULE]) - for var_type in [str, int, float]: - try: - parsed_value = var_type(value) - except ValueError as e: - logger.debug(str(e)) - continue - new_conf = self._modify_dictionary(new_conf, params_path, - parsed_value) - try: - clz.validate_config(new_conf) - logger.debug("New validated config = " + str(new_conf)) - self._config[comp_type][comp_id] = new_conf - return True - except voluptuous.Invalid as e: - logger.debug(str(e)) - continue - return False - - def remove_component(self, component_id): - """Remove a component from the configuration. - - Removes from the configuration the component whose ID matches the - one passed as parameter. - - :type component_id: str - :param: component_id: ID of the component to be removed - :rtype: bool - :returns: True if the component was removed, False otherwise - :raises: DSLExistingConnection -- if at least a connection exists with - the component as origin or destination. - """ - if self._is_connected(component_id): - raise err.DSLExistingConnection("Cannot remove component " + - component_id + - " because it is still connected") - for comp_type in const.components_types: - if component_id in self._config[comp_type].keys(): - del self._config[comp_type][component_id] - del self._config[const.CONNECTIONS][component_id] - return True - return False - - def connect_components(self, origin_id, dest_id): - """Connect the two components passed as parameter. - - Being the origin the first one and the destination the second one. - If the connection already existed, False is returned. If we created a - new connection, True is returned. - If any of the components is not defined, a DSLInexistentComponent - exception is raised. - - :type origin_id: str - :param origin_id: ID of the component at the origin of the connection - :type dest_id: str - :param dest_id: ID of the component at the destination of the - connection - :rtype: bool - :returns: True if the components were connected, False if - the connection already existed - :raises: DSLInexistentComponent -- if either the origin or the - destination are not defined in the configuration - """ - if not self._component_defined(origin_id): - raise err.DSLInexistentComponent(origin_id) - if not self._component_defined(dest_id): - raise err.DSLInexistentComponent(dest_id) - if dest_id in self._config[const.CONNECTIONS][origin_id]: - return False - if not self._validate_connection_by_ids(origin_id, dest_id): - raise err.DSLInvalidConnection("Connection from " + origin_id + - " to " + dest_id + - " is not allowed") - self._config[const.CONNECTIONS][origin_id].append(dest_id) - return True - - def disconnect_components(self, origin_id, dest_id): - """Disconnect the component dest_id from origin_id - - If the connection from origin_id to dest_id exists, it is removed, - and the function returns true. - If it didn't exist, the function returns false and nothing happens - - :type origin_id: str - :param origin_id: ID of the component at the origin of the connection - :type dest_id: str - :param dest_id: ID of the component at the destination of the - connection - :rtype: bool - :returns: True if the components were already disconnected, - False if the connection didn't exist in the first place - """ - if origin_id in self._config[const.CONNECTIONS]: - if dest_id in self._config[const.CONNECTIONS][origin_id]: - self._config[const.CONNECTIONS][origin_id].remove(dest_id) - return True - return False - - def load_configuration(self, config_file_path): - """Load a configuration from the file passed as parameter - - :type config_file_path: str - :param config_file_path: file path containing the - configuration that will be loaded - """ - self._config = cu.parse_json_file(config_file_path) - self._init_ids_dictionary() - - def _init_ids_dictionary(self): - self.ids_by_type = { - const.SOURCES: ("src", 0), - const.INGESTORS: ("ing", 0), - const.SMLS: ("sml", 0), - const.VOTERS: ("vot", 0), - const.SINKS: ("snk", 0), - const.LDPS: ("ldp", 0), - const.CONNECTIONS: ("connections", 0)} - - def save_configuration(self, config_file_path, overwrite_file=True): - """Save the configuration to the file passed as parameter - - :type config_file_path: str - :param config_file_path: file path where the configuration - will be saved - :type overwrite_file: bool - :param overwrite_file: True will overwrite the file if it exists, - False will make the function return without - saving. - :rtype: bool - :returns: True if the configuration was saved, - False otherwise - """ - if os.path.exists(config_file_path) and\ - os.stat(config_file_path).st_size > 0 and\ - overwrite_file is False: - return False - with open(config_file_path, "w") as f: - str_config = json.dumps(self._config) - f.write(str_config) - return True - - def _generate_id(self, comp_type): - """Generate a new ID for the component type passed as parameter - - After the ID is generated, the last max checked number is stored - in the ids_by_type dictionary - - :type comp_type: str - :param comp_type: type of component for which the ID will - be created - :raises: KeyError -- if the comp_type does not correspond to a - component type of the configuration - """ - typ, num = self.ids_by_type[comp_type] - num += 1 - while(typ + str(num) in self._config[comp_type].keys()): - num += 1 - self.ids_by_type[comp_type] = (typ, num) - return typ + str(num) - - def _get_type_by_id(self, component_id): - """Gets the type of a copmonent from its ID - - :type component_id: str - :param component_id: ID of a component - :rtype: str - :returns: type of component for the ID passed as parameter - """ - for comp_type in const.components_types: - if component_id in self._config[comp_type]: - return comp_type - - def _component_defined(self, component_id): - """Check if a component is defined in the configuration - - :type component_id: str - :param component_id: ID of a component - :rtype: bool - :returns: True if the component is defined in the configuration - """ - comp_type = self._get_type_by_id(component_id) - if not comp_type: - return False - return True - - def _is_connected(self, component_id): - """Check if a component is connected - - Both in and out connections will be considered: if there is any - connection with component_id as either source or destination, this - function will return true. Empty connections lists are ignored. - - :type component_id: str - :param component_id: ID of a component - :rtype: bool - :returns: True if the component is connected to another component - according to the configuration, False otherwise - """ - cons = self._config[const.CONNECTIONS] - for origin_id, dest_ids in six.iteritems(cons): - if dest_ids == []: - continue - if origin_id == component_id: - return True - for dest_id in dest_ids: - if dest_id == component_id: - return True - return False - - def _validate_connection_by_ids(self, origin_id, dest_id): - """Validate the connection from origin_id to dest_id - - The connection from the component with ID = origin_id - to the component with ID = dest_id is validated according to the - valid connections dictionary defined in the validation module. - - :type origin_id: str - :param origin_id: ID of the origin component - :type dest_id: str - :param dest_id: ID of the destination component - :rtype: bool - :returns: True if the connection is allowed, False otherwise - """ - origin_type = self._get_type_by_id(origin_id) - dest_type = self._get_type_by_id(dest_id) - if dest_type in validation.valid_connection_types[origin_type]: - return True - return False - - def _modify_dictionary(self, target_dict, params_path, value): - """Override the value at the end of a path in the target_dictionary - - Modify the dictionary passed as first parameter, assigning the value - passed as last parameter to the key path defined by params_path - - :type target_dict: dict - :param target_dict: target to be modified - :type params_path: list[str] - :param params_path: hierarchy of keys to navigate in - the dictionary, pointing the leave to modify - :type value: object - :param value: Value that will be assigned to the path defined - by params_path in the dictionary - :rtype dict - :returns: Modified dictionary - """ - aux = target_dict - for i, param in enumerate(params_path): - if param not in aux.keys(): - aux[param] = {} - if i == len(params_path) - 1: - aux[param] = value - else: - aux = aux[param] - return target_dict diff --git a/monasca_analytics/banana/cli/interpreter.py b/monasca_analytics/banana/cli/interpreter.py deleted file mode 100644 index d1027b9..0000000 --- a/monasca_analytics/banana/cli/interpreter.py +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import json -import logging - -from monasca_analytics.banana.cli import const as dsl_const -from monasca_analytics.banana.cli import dsl -from monasca_analytics.banana.cli import parser -from monasca_analytics.config import const as config_const -from monasca_analytics.exception import dsl as err -import monasca_analytics.util.common_util as cu - -logger = logging.getLogger(__name__) - - -class DSLInterpreter(object): - - def __init__(self): - self.file_in_use = None - self.dsl = dsl.MonanasDSL() - self.mappings = {} - - def execute_string(self, str_program): - """Parse and execute the command/s in the string passed as parameter - - :type str_program: str - :param str_program: command to be executed - :rtype: str - :returns: execution result - """ - info = parser.get_parser().parseString(str_program) - return self.execute(info) - - def execute_file(self, file_program): - """Parse and execute the command/s in the file passed as parameter - - :type file_program: str - :param file_program: path to the file containing the - command to be executed - :rtype: str - :returns: execution result - """ - info = parser.get_parser().parseFile(file_program) - return self.execute(info) - - def execute(self, info): - """Execute parsed command/s - - :type info: dict - :param info: containing the parsed instructions - :rtype: str - :returns: execution result - """ - for cmd in info: - for key in cmd.keys(): - if key == dsl_const.CREATE: - return self.create(cmd[key][0], cmd[key][1]) - elif key == dsl_const.CONNECT: - return self.connect(cmd[key][0], cmd[key][1]) - elif key == dsl_const.DISCONNECT: - return self.disconnect(cmd[key][0], cmd[key][1]) - elif key == dsl_const.LOAD: - return self.load(cmd[key][0]) - elif key == dsl_const.SAVE_AS: - return self.save(cmd[key][0]) - elif key == dsl_const.SAVE: - return self.save() - elif key == dsl_const.REMOVE: - return self.remove(cmd[key][0]) - elif key == dsl_const.MODIFY: - return self.modify( - cmd[key][0], cmd[key][1:-1], cmd[key][-1]) - elif key == dsl_const.PRINT: - if len(cmd[key]) > 0: - return self.prnt(cmd[key][0]) - else: - return self.prnt_all() - elif key == dsl_const.LIST: - if len(cmd[key]) > 0: - return self.list(cmd[key][0]) - else: - return self.list_all() - elif key == dsl_const.HELP: - return self.help() - else: - return logger.warn("Wrong command" + str(cmd)) - - def create(self, varname, modulename): - """Add a module defined by modulename in the configuration - - :type varname: str - :param varname: name of the variable representing - the new component - :rtype: str - :returns: new component ID - """ - clz = cu.get_class_by_name(modulename) - conf = copy.deepcopy(clz.get_default_config()) - comp_id = self.dsl.add_component(conf) - self.mappings[varname] = comp_id - return comp_id - - def connect(self, origin_varname, dest_varname): - """Connect two components - - :type origin_varname: str - :param origin_varname: variable name or ID of the source - component of the connection - :type dest_varname: str - :param dest_varname: variable name or ID of the destination - component of the connection - :rtype: bool - :returns: True if the connection was performed, - False otherwise - """ - origin_id = self._get_id(origin_varname) - dest_id = self._get_id(dest_varname) - return self.dsl.connect_components(origin_id, dest_id) - - def _get_id(self, name_or_id): - """Get the ID from a name or ID - - :type name_or_id: str - :param name_or_id: variable name or ID - :rtype: str - :returns: ID - """ - if name_or_id in self.mappings.keys(): - return self.mappings[name_or_id] - for comp_type in config_const.components_types: - if name_or_id in self.dsl._config[comp_type]: - return name_or_id - raise err.DSLInterpreterException("undefined variable: " + name_or_id) - - def disconnect(self, origin_varname, dest_varname): - """Disconnect two components - - :type origin_varname: str - :param origin_varname: variable name or ID of the source - component of the connection - :type dest_varname: str - :param dest_varname: variable name or ID of the destination - component of the connection - :rtype: bool - :returns: True if the components were disconnected, - False otherwise - """ - origin_id = self._get_id(origin_varname) - dest_id = self._get_id(dest_varname) - return self.dsl.disconnect_components(origin_id, dest_id) - - def load(self, filepath): - """Load configuration from a file - - :type filepath: str - :param filepath: path to the file to be loaded - """ - self.dsl.load_configuration(filepath) - self.file_in_use = filepath - - def save(self, filepath=None): - """Save configuration to a file - - :type filepath: str - :param filepath: (Optional) path to the file where the configuration - will be saved. If the path is not provided, the last - file used for saving or loading will be used. - """ - if not filepath: - filepath = self.file_in_use - saved = self.dsl.save_configuration(filepath, overwrite_file=True) - if saved: - self.file_in_use = filepath - return saved - - def remove(self, varname): - """Remove a variable or ID from the configuration - - :type varname: str - :param varname: variable name or ID mapped to the component - that will be removed from the configuration - """ - remove_id = self._get_id(varname) - return self.dsl.remove_component(remove_id) - - def modify(self, varname, params, value): - """Override the value of the configuration path of a component - - :type varname: str - :param varname: variable name or ID mapped to the component - :type params: list - :param params: path to be modified in the configuration - :type value: float | int | str - :param value: value to assign - """ - modify_id = self._get_id(varname) - return self.dsl.modify_component(modify_id, params, value) - - def prnt(self, varname): - """Print the configuration of the module/s defined by varname - - If varname is a variable or ID associated to a particular component, - the configuration of that component will be printed. If it is a type - of components, the configurations of all components of that type - will be printed. - - :type varname: str - :param varname: variable, ID or type to be printed - :rtype: str - :returns: requested configuration in string format - """ - if varname in self.dsl._config.keys(): - return self._json_print(self.dsl._config[varname]) - itemId = self._get_id(varname) - for k in config_const.components_types: - if itemId in self.dsl._config[k]: - return self._json_print(self.dsl._config[k][itemId]) - - def prnt_all(self): - """Print the the whole configuration - - :rtype: str - :returns: whole configuration in string format - """ - return self._json_print(self.dsl._config) - - def _json_print(self, jstr): - """Format Json as a clean string""" - return json.dumps(jstr, indent=4, separators=(',', ': ')) - - def list(self, key): - """List the available components of the type passed as parameter""" - ret = "" - if key in config_const.components_types: - for name in cu.get_available_class_names(key): - ret += "- " + name + "\n" - return ret - - def list_all(self): - """List all available components grouped by type""" - ret = "" - for key in config_const.components_types: - ret += "- " + key + "\n" - for name in cu.get_available_class_names(key): - ret += " - " + name + "\n" - return ret - - def help(self): - return """ -Available commands - - print: prints current configuration - - list: shows available modules - - load: loads a config from a file - - save: saves a config to a file - - = : instantiates module , referenced by - - ->: connects the module to the module - - !->: disconnects the module from the module - - rm : removes the module corresponding to - - exit: finishes the execution of monanas command line -""" diff --git a/monasca_analytics/banana/cli/parser.py b/monasca_analytics/banana/cli/parser.py deleted file mode 100644 index 50f50ad..0000000 --- a/monasca_analytics/banana/cli/parser.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pyparsing as p - -from monasca_analytics.banana.cli import const - -EQUALS = p.Literal("=").suppress() -CONNECT = p.Literal("->").suppress() -DISCONNECT = p.Literal("!->").suppress() -LPAREN = p.Literal("(").suppress() -RPAREN = p.Literal(")").suppress() -LOAD = p.CaselessKeyword("load").suppress() -SAVE = p.CaselessKeyword("save").suppress() -REMOVE = p.CaselessKeyword("rm").suppress() -PRINT = p.CaselessKeyword("print").suppress() -LIST = p.CaselessKeyword("list").suppress() -HELP = p.CaselessKeyword("help").suppress() -DOT = p.Literal(".").suppress() -VARNAME = p.Word(p.alphas + "_", p.alphanums + "_") -PARAMETER = p.Word(p.alphanums + "_-") -MODULE_NAME = p.Word(p.alphanums + "_-") -VALUE = p.Word(p.alphanums + "_-.") -PATH = p.Word(p.alphanums + "_-/\.") - -cmd_create = (VARNAME + EQUALS + MODULE_NAME) -cmd_connect = (VARNAME + CONNECT + VARNAME) -cmd_disconnect = (VARNAME + DISCONNECT + VARNAME) -cmd_modify = (VARNAME + p.OneOrMore(DOT + PARAMETER) + EQUALS + VALUE) -cmd_load = (LOAD + p.Optional(LPAREN) + PATH + p.Optional(RPAREN)) -cmd_save = (SAVE + p.Optional(LPAREN) + p.Optional(RPAREN)) -cmd_save_as = (SAVE + p.Optional(LPAREN) + PATH + p.Optional(RPAREN)) -cmd_remove = (REMOVE + p.Optional(LPAREN) + VARNAME + p.Optional(RPAREN)) -cmd_print = (PRINT + p.Optional(LPAREN) + p.Optional(VARNAME) + - p.Optional(RPAREN)) -cmd_list = (LIST + p.Optional(LPAREN) + p.Optional(VARNAME) + - p.Optional(RPAREN)) -cmd_help = (HELP + p.Optional(LPAREN) + p.Optional(RPAREN)) - -bnfLine = cmd_create(const.CREATE) | cmd_connect(const.CONNECT) | \ - cmd_disconnect(const.DISCONNECT) | cmd_load(const.LOAD) | \ - cmd_save_as(const.SAVE_AS) | cmd_save(const.SAVE) | \ - cmd_remove(const.REMOVE) | cmd_modify(const.MODIFY) | \ - cmd_print(const.PRINT) | cmd_list(const.LIST) | cmd_help(const.HELP) - -bnfComment = "#" + p.restOfLine - -bnf = p.OneOrMore(p.Group(bnfLine)) -bnf.ignore(bnfComment) - - -def get_parser(): - return bnf diff --git a/monasca_analytics/banana/deadpathck/README.md b/monasca_analytics/banana/deadpathck/README.md deleted file mode 100644 index 0326592..0000000 --- a/monasca_analytics/banana/deadpathck/README.md +++ /dev/null @@ -1,22 +0,0 @@ -## Dead path checker - -Dead path checking is about removing paths in the pipeline that -lead to nothing. For instance, if there's no source or no sink -in a path. This pass is the only one that modifies the AST. - -This is the third step of the pipeline: -``` - +---------------------+ +---------------------+ - | | | | - ---> | AST & TypeTable | ---- deadpathck ---> | AST' & TypeTable' | ---> - | | | | - +---------------------+ +---------------------+ -``` - -### Current status: - -* [x] Remove branches that are dead from the list of connections. -* [x] Remove the components from the collected list of components. -* [ ] Remove statements that are dead code: - - [ ] Do not instantiate components. - - [ ] Do not compute expressions for unused variables. diff --git a/monasca_analytics/banana/deadpathck/__init__.py b/monasca_analytics/banana/deadpathck/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/banana/deadpathck/config.py b/monasca_analytics/banana/deadpathck/config.py deleted file mode 100644 index 3f97728..0000000 --- a/monasca_analytics/banana/deadpathck/config.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monasca_analytics.banana.deadpathck.dag as dag -import monasca_analytics.banana.emitter as emit -import monasca_analytics.banana.grammar.ast as ast -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.exception.banana as exception - - -import six - - -def deadpathck(banana_file, type_table, emitter=emit.PrintEmitter()): - """ - Perform dead path elimination on the provided AST. - This allow to remove branches and components that - are not connected to a Sink. - :type banana_file: ast.BananaFile - :param banana_file: The AST tree we will clean. - :type type_table monasca_analytics.banana.typeck.type_table.TypeTable - :param type_table: The TypeTable of the provided AST. - :type emitter: emit.Emitter - :param emitter: Emitter for reporting warnings. - """ - # Check that first argument is a banana file. Mainly - # an excuse to remove the F401 warning. - if not isinstance(banana_file, ast.BananaFile): - raise Exception("Expected BananaFile as first argument.") - - # Look first for all branch that are "dead" - connections = banana_file.connections # type: ast.Connection - - # If there are no connections everything is considered - # as dead. - if connections is None: - class EmptyConnections(object): - connections = [] - connections = EmptyConnections() - - # Collect the nodes and connect them. - dag_nodes = {} - # Create all the nodes - for ident in banana_file.components.keys(): - dag_nodes[ident] = dag.DagNode(type_table.get_type(ident)) - # Connect them - for ident_from, ident_to in connections.connections: - dag_from = dag_nodes[ident_from] - dag_to = dag_nodes[ident_to] - dag_from.children.append(dag_to) - dag_to.parents.append(dag_from) - - # Start from every sources and for each, check if the path is dead - for node in dag_nodes.values(): - if isinstance(node.typec, type_util.Source): - node.visit() - - # We can now remove all the components that are "dead" - # from the list of connections - for ident, node in six.iteritems(dag_nodes): - if not node.is_alive(): - emitter.emit_warning( - ident.span, - "Dead code found, this component is not in a path " - "starting from a 'Source' and ending with a 'Sink'." - ) - banana_file.components.pop(ident) - connections.connections = [edge for edge in connections.connections - if edge[0] != ident and - edge[1] != ident] - - # TODO(Joan): We could also remove them from the statements. - # TODO(Joan): But for this we need a dependency graph between - # TODO(Joan): statements to make sure we don't break the code. - - -def contains_at_least_one_path_to_a_sink(banana_file, type_table): - """ - Check that there's at least one path to a sink in the list - of components. - To run this pass, you need to make sure you - have eliminated all dead path first. - - :type banana_file: ast.BananaFile - :param banana_file: The AST to check. - :type type_table monasca_analytics.banana.typeck.type_table.TypeTable - :param type_table: The TypeTable of the provided AST. - :raise: Raise an exception if there's no Sink. - """ - def is_sink(comp): - type_comp = type_table.get_type(comp) - return isinstance(type_comp, type_util.Sink) - - def is_src(comp): - type_comp = type_table.get_type(comp) - return isinstance(type_comp, type_util.Source) - - comp_vars = banana_file.components.keys() - at_least_one_sink = len(list(filter(is_sink, comp_vars))) > 0 - at_least_one_source = len(list(filter(is_src, comp_vars))) > 0 - - if not at_least_one_sink: - raise exception.BananaNoFullPath("Sink") - if not at_least_one_source: - raise exception.BananaNoFullPath("Source") diff --git a/monasca_analytics/banana/deadpathck/dag.py b/monasca_analytics/banana/deadpathck/dag.py deleted file mode 100644 index 6df8dd0..0000000 --- a/monasca_analytics/banana/deadpathck/dag.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monasca_analytics.banana.typeck.type_util as type_util - - -class DagNode(object): - - def __init__(self, typec): - """ - Create a DAGNode. - - :param typec: The type of the node. - """ - self.parents = [] - self.children = [] - self.typec = typec - self._visited = False - self._seen_sink = False - - def visit(self): - """ - Visit this nodes and all of its connections. - """ - if not self._visited: - self._visited = True - if isinstance(self.typec, type_util.Sink): - self.visit_parents() - return - for child in self.children: - child.visit() - - def visit_parents(self): - """ - Visit the parent to tell them that we've seen a Sink. - """ - if not self._seen_sink: - self._seen_sink = True - for parent in self.parents: - parent.visit_parents() - - def is_alive(self): - return self._visited and self._seen_sink diff --git a/monasca_analytics/banana/emitter.py b/monasca_analytics/banana/emitter.py deleted file mode 100644 index 4b81954..0000000 --- a/monasca_analytics/banana/emitter.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import six - - -@six.add_metaclass(abc.ABCMeta) -class Emitter(object): - """ - Emitter base class to emit errors and warnings. - Typically errors will be collected and then send - over the network as an http response but for tests - and debugging, a `PrintEmitter` can be used instead. - """ - - @abc.abstractmethod - def emit_warning(self, span, message): - """ - Emit a warning. - :type span: monasca_analytics.banana.grammar.base_ast.Span - :param span: Span associated with the message. - :type message: str - :param message: message to emit with the warning level. - """ - pass - - @abc.abstractmethod - def emit_error(self, span, message): - """ - Emit an error - :type span: monasca_analytics.banana.grammar.base_ast.Span - :param span: Span associated with the message. - :type message: str - :param message: message to emit with the error level. - """ - pass - - -class PrintEmitter(Emitter): - """ - Print warnings and errors to the console. - """ - def emit_warning(self, span, message): - print("WARNING at line:{}".format(span.get_lineno())) - print("WARNING: {}".format(message)) - - def emit_error(self, span, message): - print("ERROR at line:{}".format(span.get_lineno())) - print("ERROR: {}".format(message)) - - -class JsonEmitter(Emitter): - """ - Print warnings and errors in a Json object. - """ - def __init__(self): - self.result = { - "errors": [], - "warnings": [], - } - - def emit_error(self, span, message): - error = JsonEmitter._gen_message_structure(span, message) - self.result["errors"].append(error) - - def emit_warning(self, span, message): - warning = JsonEmitter._gen_message_structure(span, message) - self.result["warnings"].append(warning) - - @staticmethod - def _gen_message_structure(span, message): - spanrange = span.get_range() - return { - "startLineNumber": spanrange[0][0], - "startColumn": spanrange[0][1], - "endLineNumber": spanrange[1][0], - "endColumn": spanrange[1][1], - "byteRange": [span.lo, span.hi], - "message": message - } diff --git a/monasca_analytics/banana/eval/README.md b/monasca_analytics/banana/eval/README.md deleted file mode 100644 index fc5c25e..0000000 --- a/monasca_analytics/banana/eval/README.md +++ /dev/null @@ -1,50 +0,0 @@ -## Interpreter / Evaluator - -This folder contains everything related to the evaluation -of banana files. - -This pass makes some assumptions: it is valid to create -all the components and connecting them won't throw any errors. - -Some components might need to be created in order to -check if they are valid. For instance, when a DNS lookup is -involved. In such cases, an error will be thrown during -the interpretation. However, the general intention is to move -the checks out of the evaluation as much as possible. We want to -avoid at all cost an half-working pipeline as it could have -side-effects on external data sources by corrupting them or -feeding them with incorrect data. - -The execution environment (e.g Spark) might also reject the -pipeline during an evaluation for some reason. However, this is -less likely to happen as the `deadpathck` pass removes -components and paths that would lead to errors. - -This is the last step of the pipeline: -``` - +---------------------+ - | | - ---> | AST & TypeTable | ---- interpret ---> Done - | | - +---------------------+ -``` - -### Current status - -* [x] Evaluate expressions -* [x] Create components -* [x] Connect components -* [x] Restart the pipeline - -### Tests - -All tests are located in `test/banana/eval`. We only try -to evaluate valid files, so for this pass there's only a -`should_pass` directory. - -#### Available instruction - -* `# LHS_EQ `: This instruction - compares the evaluation of the left hand side of the previous - expression with the provided string. If they are not equal, - the test will fail. \ No newline at end of file diff --git a/monasca_analytics/banana/eval/__init__.py b/monasca_analytics/banana/eval/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/banana/eval/config.py b/monasca_analytics/banana/eval/config.py deleted file mode 100644 index 1e4b278..0000000 --- a/monasca_analytics/banana/eval/config.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import operator - -import monasca_analytics.banana.eval.ctx as ctx -import monasca_analytics.banana.eval.old_style as into -import monasca_analytics.banana.grammar.ast as ast -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.config.connection as connection -import monasca_analytics.config.const as conf_const -import monasca_analytics.exception.banana as exception -import monasca_analytics.util.common_util as introspect - -import six - - -logger = logging.getLogger(__name__) - - -def eval_ast(ast_root, type_table, driver): - """ - Evaluate the provided AST by instantiating - the appropriate components and connecting them - together using the Driver interface. - :type ast_root: ast.BananaFile - :param ast_root: AST to evaluate. - :type type_table: monasca_analytics.banana.typeck.type_table.TypeTable - :param type_table: the TypeTable (used to create configurations) - :type driver: monasca_analytics.spark.driver.DriverExecutor - :param driver: Driver that will manage the created - components and connect them together. - """ - logger.debug("Creating the config dictionary from the AST...") - _config = conf_const.get_default_base_config() - - try: - logger.debug("Creating components according to banana config...") - components = eval_create_components(ast_root.statements, type_table) - convert_connections(ast_root.connections, _config) - logger.debug("Done creating components. Creating link data...") - # Pre-process to convert to old style components - components_old_style = into.into_old_conf_dict(components) - links = connection.connect_components(components_old_style, _config) - logger.debug("Done connecting components. Successful instantiation") - except Exception as ex: - logger.error("Failed to instantiate components") - logger.error("Reason : " + str(ex)) - return - - # Restart Spark using the new config - logger.debug("Stop pipeline") - driver.stop_pipeline() - logger.debug("Set new links") - driver.set_links(links) - logger.debug("Start pipeline") - driver.start_pipeline() - - -def convert_connections(connections, output_config): - """ - Augment the output_config object with the list of - connections - :type connections: ast.Connection - :param connections: The list of connections. - :type output_config: dict - :param output_config: Config where the links will be written. - """ - output_config[conf_const.CONNECTIONS] = connections.connections_cache - - -def eval_create_components(statements, type_table): - """ - Convert the provided AST into the old dict configuration. - :type statements: list[(ast.ASTNode, ast.ASTNode)] - :param statements: The AST to process - :type type_table: monasca_analytics.banana.typeck.type_table.TypeTable - :param type_table: the type table. - :rtype: dict[str, Component] - :return: Returns the component keyed by name. - """ - context = ctx.EvaluationContext() - - eval_statements_generic( - statements, - type_table, - context - ) - return context.get_components() - - -def eval_statements_generic( - statements, - type_table, - context, - cb=lambda *a, **k: None): - """ - Eval the list of statements, and call the cb after evaluating - each statement providing it with the type of the value, the - left hand side ast node, and the computed value. - :type statements: list[(ast.ASTNode, ast.ASTNode)] - :param statements: The AST to process - :type type_table: monasca_analytics.banana.typeck.type_table.TypeTable - :param type_table: the type table. - :type context: ctx.EvaluationContext - :param context: evaluation context that will collect - all intermediary results. - :type cb: (type_util.IsType, ast.ASTNode, object) -> None - :param cb: Callback called after each statement evaluation. - """ - stmt_index = 0 - for stmt in statements: - lhs, rhs = stmt - expected_type = type_table.get_type(lhs, stmt_index + 1) - stmt_index += 1 - # Provide the expected type - value = eval_rhs(context, rhs, expected_type) - # Call the cb with the expected_type of the value - # The lhs node and the value - cb(expected_type, lhs, value) - # Store result if referenced later. - context.set_variable(lhs, value) - - -def eval_rhs(context, ast_node, expected_type): - """ - Eval the right hand side node. - :type context: ctx.EvaluationContext - :param context: Evaluation context. - :type ast_node: ast.ASTNode - :param ast_node: the node to evaluate. - :type expected_type: type_util.IsType - :param expected_type: The expected type of this computation. - :return: Returns the result of this evaluation. - """ - if isinstance(ast_node, ast.StringLit): - return ast_node.inner_val() - if isinstance(ast_node, ast.Ident): - return context.get_variable(ast_node.inner_val()) - if isinstance(ast_node, ast.JsonObj): - return eval_object(context, ast_node, expected_type) - if isinstance(ast_node, ast.Number): - return ast_node.val - if isinstance(ast_node, ast.DotPath): - variable_name = ast_node.varname.inner_val() - prop = map(lambda x: x.inner_val(), ast_node.properties) - return context.get_prop_of_variable(variable_name, prop) - if isinstance(ast_node, ast.Expr): - return eval_expr(context, ast_node, expected_type) - if isinstance(ast_node, ast.Component): - return eval_comp(context, ast_node, expected_type) - raise Exception("Unhandled ast value type {}!!".format(ast_node)) - - -def eval_comp(context, comp, expected_type): - """ - Instantiate the given component, computing - the required config. - :type context: ctx.EvaluationContext - :param context: Evaluation context. - :type comp: ast.Component - :param comp: the node to evaluate. - :type expected_type: type_util.IsType - :param expected_type: The expected type of this computation. - :return: Returns the instantiated component. - """ - arguments = {} - # Compute arguments - for arg in comp.args: - arg_name = ast.DotPath(arg.arg_name.span, arg.arg_name, []) - arg_value = eval_rhs(context, arg.value, expected_type[arg_name]) - arguments[arg.arg_name.inner_val()] = arg_value - # Lookup component - component_type = introspect.get_class_by_name(comp.type_name.val) - # Get default config for the component - conf = component_type.get_default_config() - # Update modified params - for k, val in six.iteritems(arguments): - conf[k] = val - # Delay evaluation until we do the assign - return component_type, conf - - -def eval_object(context, obj, expected_type): - """ - Evaluate the provided object - :type context: ctx.EvaluationContext - :param context: Evaluation context. - :type obj: ast.JsonObj - :param obj: The expression to evaluate - :type expected_type: type_util.IsType - :param expected_type: The expected type of this computation. - :return: Returns the computed value - """ - result = expected_type.default_value() - for name, val in six.iteritems(obj.props): - subtype = expected_type[name] - ctx.set_property(result, name, eval_rhs(context, val, subtype)) - return result - - -def eval_expr(context, expr, expected_type): - """ - Eval the provided expression - :type context: ctx.EvaluationContext - :param context: Evaluation context. - :type expr: ast.Expr - :param expr: The expression to evaluate - :type expected_type: type_util.IsType - :param expected_type: The expected type of this computation. - :rtype: str | float - :return: Returns the computed value - """ - if len(expr.expr_tree) == 1: - return eval_rhs(context, expr.expr_tree[0], expected_type) - - if isinstance(expected_type, type_util.Number): - result = 0 - cast_func = float - elif isinstance(expected_type, type_util.String): - result = "" - cast_func = str - else: - raise exception.BananaEvalBug( - "Expected type for an expression can only be " - "'TypeNumber' or 'TypeString', got '{}'".format( - str(expected_type)) - ) - current_operator = operator.add - for el in expr.expr_tree: - if isinstance(el, six.string_types) and el in ['+', '-', '*', '/']: - current_operator = get_op_func(el) - else: - value = eval_rhs(context, el, expected_type) - value = cast_func(value) - result = current_operator(result, value) - return result - - -def get_op_func(op_str): - if op_str == '+': - return operator.add - if op_str == '-': - return operator.sub - if op_str == '*': - return operator.mul - if op_str == '/': - return operator.div - raise exception.BananaEvalBug( - "Unknown operator '{}'".format(op_str) - ) diff --git a/monasca_analytics/banana/eval/ctx.py b/monasca_analytics/banana/eval/ctx.py deleted file mode 100644 index b6bff2e..0000000 --- a/monasca_analytics/banana/eval/ctx.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monasca_analytics.banana.grammar.ast as ast -import monasca_analytics.exception.banana as exception - - -class EvaluationContext(object): - """Evalutation context for an AST evaluation""" - - def __init__(self): - """ - Construct an evaluation context using the type table - as based for the variable needed to be created. - """ - self._variables = {} - self._components = {} - - def set_variable(self, name, value): - """Set the variable value.""" - if isinstance(value, tuple) and len(value) == 2: - comp_type, config = value - comp_name = name.varname.inner_val() - self._components[comp_name] = comp_type( - comp_name, - config - ) - self._variables[comp_name] = config - elif not set_property(self._variables, name, value): - raise exception.BananaEvalBug( - "set_variable can only be used with DotPath or Ident." - ) - - def get_components(self): - return self._components - - def get_variable(self, name): - """Returns the variable value.""" - return self._variables[name] - - def get_prop_of_variable(self, name, prop): - """Returns the sub property of the given variable name.""" - variable = self._variables[name] - for el in prop: - variable = variable[el] - return variable - - -def set_property(root_object, name, value): - """ - Set the property name of the root_object to value. - :type root_object: dict - :param root_object: The root object - :type name: ast.DotPath | ast.Ident | ast.StringLit - :param name: Name of - :param value: - :return: Returns true if succeeded. - """ - if isinstance(name, ast.Ident) or isinstance(name, ast.StringLit): - root_object[name.inner_val()] = value - return True - elif isinstance(name, ast.DotPath): - _create_as_many_object_as_needed(root_object, name, value) - return True - return False - - -def _create_as_many_object_as_needed(root_object, dot_path, value): - """ - Create as many object as needed to be able to access the - nested property. - - :type root_object: dict - :param root_object: The root object - :type dot_path: ast.DotPath - :param dot_path: Dot Path to use. - :type value: object - :param value: Any value to set. - """ - name = dot_path.varname.inner_val() - - if len(dot_path.properties) == 0: - root_object[name] = value - return - - if name in root_object: - variable = root_object[name] - else: - variable = {} - - current_var = variable - last_index = len(dot_path.properties) - 1 - - for index, subpath in enumerate(dot_path.properties): - subpath_name = subpath.inner_val() - if index != last_index: - if subpath_name in current_var: - current_var = current_var[subpath_name] - else: - new_object = {} - current_var[subpath_name] = new_object - current_var = new_object - else: - current_var[subpath_name] = value - - root_object[name] = variable diff --git a/monasca_analytics/banana/eval/old_style.py b/monasca_analytics/banana/eval/old_style.py deleted file mode 100644 index 74526d8..0000000 --- a/monasca_analytics/banana/eval/old_style.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monasca_analytics.config.const as conf_const -import monasca_analytics.ingestor.base as ingestor -import monasca_analytics.ldp.base as ldp -import monasca_analytics.sink.base as sink -import monasca_analytics.sml.base as sml -import monasca_analytics.source.base as source -import monasca_analytics.voter.base as voter - - -import six - - -def into_old_conf_dict(components): - """ - Convert the provided dict of components - keyed by ids into a dict keyed by component - type. This is the data structure used to do - the validation of JSON configuration (the old format). - - :type components: dict[str, object] - :param components: The dictionary of components. - :return: Returns the old conf dictionary. - """ - return { - conf_const.INGESTORS: - dict(filter(lambda x: isinstance(x[1], ingestor.BaseIngestor), - six.iteritems(components))), - conf_const.VOTERS: - dict(filter(lambda x: isinstance(x[1], voter.BaseVoter), - six.iteritems(components))), - conf_const.SINKS: - dict(filter(lambda x: isinstance(x[1], sink.BaseSink), - six.iteritems(components))), - conf_const.LDPS: - dict(filter(lambda x: isinstance(x[1], ldp.BaseLDP), - six.iteritems(components))), - conf_const.SOURCES: - dict(filter(lambda x: isinstance(x[1], source.BaseSource), - six.iteritems(components))), - conf_const.SMLS: - dict(filter(lambda x: isinstance(x[1], sml.BaseSML), - six.iteritems(components))), - } diff --git a/monasca_analytics/banana/grammar/README.md b/monasca_analytics/banana/grammar/README.md deleted file mode 100644 index deeff47..0000000 --- a/monasca_analytics/banana/grammar/README.md +++ /dev/null @@ -1,63 +0,0 @@ -## Grammar - -This folder is all about the definition of the `banana` grammar. -The grammar purpose is to convert the input, text, into an -abstract syntax tree (AST). - -This is the first step of the pipeline: -``` - +--------+ +---------+ - | | | | - | Text | --- grammar ---> | AST | ---> - | | | | - +--------+ +---------+ -``` - -The module `ast.py` contains all the possible `ASTNode` which -itself is defined in `base_ast.py`. - -### Current status - -* [x] Parsing connections such as `a -> b`, `a -> [b, c]`, - `[a, b] -> [c, d]` -* [x] Parsing numbers -* [x] Parsing string literals -* [ ] Parsing booleans -* [x] Parsing assignments where the left hand side can be a property - or an identifier. -* [x] Parsing assignments where the right hand side is a number, a - string literal, a property or an identifier. -* [x] Parsing components arguments using a constructor-like syntax. -* [ ] Parsing ingestors generators (for JSON dialect) -* [ ] Parsing imports such as `from ldp.monasca import *` -* [ ] Parsing disconnections such as `a !-> b` *(requires imports)* - -### Tests - -All test regarding the grammar (i.e. the syntax and the way -the AST is built) is defined in `test/banana/grammar`. - -This folder looks like this: - -``` -test/banana/grammar -├── should_fail -│   ├── ... -│   └── file.banana -├── should_pass -│   ├── ... -│   └── file.banana -└── test_config.py -``` - -The `test_config` generates one test for each file in the -`should_pass` and `should_fail` directories. - -Test can assert various things using instructions below. - -#### Available instruction - -* `# RAISE `: Check that `exception-name` is raised. -* `# STMT_EQ ` Check the AST of statements. -* `# AST_EQ ` Check the full AST. -* `# CONN_EQ ` Check the AST of connections. \ No newline at end of file diff --git a/monasca_analytics/banana/grammar/__init__.py b/monasca_analytics/banana/grammar/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/banana/grammar/ast.py b/monasca_analytics/banana/grammar/ast.py deleted file mode 100644 index d9920ec..0000000 --- a/monasca_analytics/banana/grammar/ast.py +++ /dev/null @@ -1,704 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import monasca_analytics.banana.emitter as emit -import monasca_analytics.banana.grammar.base_ast as base -import monasca_analytics.exception.banana as exception -import monasca_analytics.util.string_util as strut -import pyparsing as p - - -import six - - -ASTNode = base.ASTNode -Span = base.Span - - -class BananaFile(object): - - def __init__(self, emitter=emit.PrintEmitter()): - self.imports = [] - self._emitter = emitter - # Components is a dict where keys - # are the name of the var and - # values are of type Component - self.components = dict() - # Statement are component - # creation or variable creation - self.statements = [] - self.connections = None - - def add_component_ctor(self, dot_path, component): - """ - Add a component if the component - does not already exists. If it does exists, - then this raises an Error. - :type dot_path: DotPath - :param dot_path: Name of the variable or property path - :type component: Component - :param component: AST part that will contains all properties - """ - if not isinstance(dot_path, DotPath): - raise exception.BananaAssignmentError( - dot_path.span, component.span) - if not len(dot_path.properties) == 0: - raise exception.BananaAssignmentError( - dot_path.span, component.span) - if dot_path.varname in self.components: - other_dot_path = filter( - lambda x: x == dot_path, - self.components.keys())[0] - no_effect_str = dot_path.span.str_from_to(component.span) - collision_str = other_dot_path.span.str_from_to( - self.components[other_dot_path].span) - self._emitter.emit_warning( - dot_path.span, - "Statement has no effect: '{}'".format(no_effect_str) - ) - self._emitter.emit_warning( - other_dot_path.span, - "It collides with: '{}'".format(collision_str) - ) - else: - self.components[dot_path.varname] = component - self.statements.append((dot_path, component)) - - def add_assignment(self, dot_path, ast_value): - """ - Add an assignment to a property or a variable. - We don't check at this point whether or not the variable - has collision with a component. - This will be done during the name resolution pass. - :type dot_path: DotPath - :param dot_path: Name of the variable or property - :type ast_value: Ident | JsonObj | StringLit | Number | DotPath - :param ast_value: Ast node this variable is assigned to. - """ - if not isinstance(dot_path, DotPath): - raise exception.BananaAssignmentError( - dot_path.span, - ast_value.span - ) - self.statements.append((dot_path, ast_value)) - - def add_connections(self, connections): - """ - Add a new set of connections between components - This function performs the same checks as the one - performs when components are connected. It warns - on redundant connections. - :type connections: Connection - :param connections: AST node that contains the collected connections. - """ - if self.connections is None: - self.connections = connections - else: - self.connections.merge_and_reset_inputs_outputs( - connections, - self._emitter - ) - - def statements_to_str(self): - return "{ " + ', '.join( - ['{} = {}'.format(x[0], x[1]) for x in self.statements] - ) + ' }' - - def __str__(self): - res = "BananaFile { " - res += 'components: { ' - res += strut.dict_to_str(self.components) - res += ' }, ' - res += 'statements: ' + self.statements_to_str() - res += 'connections: { ' - res += str(self.connections) - res += " } }" - return res - - -def make_span(s, l, t): - - def compute_hi(init_loc, tokens): - hi = init_loc - for tok in tokens: - if isinstance(tok, ASTNode): - hi = max(hi, tok.span.hi) - elif isinstance(tok, six.string_types): - hi += len(tok) - elif isinstance(tok, p.ParseResults): - hi = max(hi, compute_hi(init_loc, tok)) - else: - raise exception.BananaGrammarBug( - "Couldn't create span for: {}".format(tok) - ) - return hi - - if len(t) > 0: - span_hi = compute_hi(l, t) - return Span(s, l, span_hi) - else: - return Span(s, l, 2) - - -class Number(ASTNode): - def __init__(self, span, val): - """ - Construct a Number ast node. - :type span: Span - :param span: Span for this number - :type val: str - :param val: Value for this number - """ - super(Number, self).__init__(span) - self.val = float(val) - - def __str__(self): - return "Number< {} >".format(self.val) - - def into_unmodified_str(self): - return str(self.val) - - -class StringLit(ASTNode): - def __init__(self, span, val): - """ - Construct a StringLit ast node. - :type span: Span - :param span: Span for this string - :type val: str - :param val: Value for this string - """ - super(StringLit, self).__init__(span) - self.val = val - - def __hash__(self): - return hash(self.inner_val()) - - def __eq__(self, other): - return (isinstance(other, StringLit) or isinstance(other, Ident))\ - and self.inner_val() == other.inner_val() - - def __ne__(self, other): - return not self.__eq__(other) - - def __str__(self): - return "StringLit< {} >".format(self.val) - - def into_unmodified_str(self): - return self.val - - def inner_val(self): - return self.val.strip()[1:-1] - - -class Ident(ASTNode): - def __init__(self, span, val): - """ - Construct an Ident ast node. - :type span: Span - :param span: Span for this identifier - :type val: str - :param val: Value of this identifier - """ - super(Ident, self).__init__(span) - self.val = val - - def __hash__(self): - return hash(self.val) - - def __eq__(self, other): - return (isinstance(other, StringLit) or isinstance(other, Ident))\ - and self.val == other.inner_val() - - def __ne__(self, other): - return not self.__eq__(other) - - def __str__(self): - return "Ident< {} >".format(self.val) - - def into_unmodified_str(self): - return self.val - - def inner_val(self): - return self.val - - -class DotPath(ASTNode): - def __init__(self, span, varname, properties): - """ - :type span: Span - :param span: Span for this dotpath. - :type varname: Ident | StringLit - :param varname: Name of the variable being changed. - :type properties: list[StringLit | Ident] - :param properties: Properties being accessed. - """ - super(DotPath, self).__init__(span) - self.varname = varname - self.properties = properties - - def next_dot_path(self): - """ - Assuming the properties length is more than zero. - This returns the dot path where the varname has been - dropped. So given 'a.b.c' the returned dot path will - be 'b.c'. - - :rtype: DotPath - :return: Returns the next dot path. - """ - return DotPath( - self.span.new_with_lo(self.properties[0].span.lo), - self.properties[0], - self.properties[1:] - ) - - def into_unmodified_str(self): - arr = [self.varname.into_unmodified_str()] - arr.extend([x.into_unmodified_str() for x in self.properties]) - return '.'.join(arr) - - def __str__(self): - arr = [str(self.varname)] - arr.extend([str(x) for x in self.properties]) - return 'DotPath< {} >'.format('.'.join(arr)) - - def __key(self): - return self.into_unmodified_str().replace('"', '') - - def __eq__(self, other): - return self.__key() == other.__key() - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.__key()) - - -class Expr(ASTNode): - def __init__(self, span, expr_tree): - """ - Construct an expression - :type span: Span - :param span: Span for the expression. - :type expr_tree: p.ParseResults - ;:param expr_tree: The tree generated by pyparsing.infixNotation - """ - super(Expr, self).__init__(span) - # We don't use this tree at this point. - # During typecheck we will make sure - # that the expression can evaluate - # Finally during evaluation, we will evaluate - # the final result. - if isinstance(expr_tree, p.ParseResults): - expr_tree = expr_tree.asList() - if isinstance(expr_tree, list): - for i in range(0, len(expr_tree)): - if isinstance(expr_tree[i], list): - expr_tree[i] = Expr(span, expr_tree[i]) - self.expr_tree = expr_tree - else: - self.expr_tree = [expr_tree] - - def into_unmodified_str(self): - # TODO(Joan): reconstruct the original expression - return 'expression' - - def __str__(self): - return "Expr< {} >".format(strut.array_to_str(self.expr_tree)) - - -class JsonObj(ASTNode): - def __init__(self, span, tokens): - super(JsonObj, self).__init__(span) - self.props = {} - last_prop = None - if len(tokens) > 0: - for toks in tokens: - for tok in toks: - if isinstance(tok, DotPath): - if last_prop is None: - last_prop = tok - else: - self._set_new_prop(last_prop, tok) - last_prop = None - elif isinstance(tok, StringLit): - if last_prop is None: - last_prop = tok - else: - self._set_new_prop(last_prop, tok) - last_prop = None - elif isinstance(tok, list): - if last_prop is None: - raise p.ParseFatalException( - "Bug Found in JsonObj!" - ) - self._set_new_prop( - last_prop, - JsonObj.dictify_array(tok) - ) - last_prop = None - else: - if last_prop is None: - raise p.ParseFatalException( - "Bug Found in JsonObj!" - ) - self._set_new_prop(last_prop, tok) - last_prop = None - - def _set_new_prop(self, prop, token): - if prop in self.props: - raise exception.BananaJsonObjShadowingError(self.span, prop.span) - else: - self.props[prop] = token - - def into_unmodified_str(self): - # TODO(Joan): improve this for error reporting - return str(self.props) - - def __str__(self): - return "JsonObj< {} >".format(strut.dict_to_str(self.props)) - - @staticmethod - def dictify_array(tok): - new_array = [] - for el in tok: - if isinstance(el, list): - new_array.append(JsonObj.dictify_array(el)) - else: - new_array.append(el) - return new_array - - -def into_connection(ast_node): - """ - Convert an ast node into a Connection node. - :type ast_node: Connection | Ident - :param ast_node: The ast node to convert. - :rtype: Connection - :return: Returns a Connection node - """ - if isinstance(ast_node, Connection): - return ast_node - elif isinstance(ast_node, Ident): - return Connection( - ast_node.span, - [ast_node], - [ast_node] - ) - else: - raise p.ParseFatalException("Bug found!") - - -class Connection(ASTNode): - - def __init__(self, span, inputs=None, outputs=None, connections=None): - """ - Create a connection object. - :type span: Span - :param span: Span for this connection - :type inputs: list[Ident] - :param inputs: Input ast nodes of the connection - :type outputs: list[Ident] - :param outputs: Outputs nodes - :type connections: list[(Ident, Ident)] - :param connections: The list of connections aggregated so far. - """ - super(Connection, self).__init__(span) - if inputs is None: - inputs = [] - if outputs is None: - outputs = [] - if connections is None: - connections = [] - self.inputs = inputs - self.outputs = outputs - self.connections = connections - self.connections_cache = {} - self._build_connection_cache() - - def connect_to(self, other_con, emitter): - """ - Connect this connection to the other one. - After this function has been executed, the other_con - object can be dropped. - :type other_con: Connection - :param other_con: Other connection to connect to. - :type emitter: emit.Emitter - :param emitter: Emitter. - """ - self.span.hi = max(other_con.span.hi, self.span.hi) - self.span.lo = min(other_con.span.lo, self.span.lo) - old_outputs = self.outputs - self.outputs = other_con.outputs - - # Generate new connections - for old_output in old_outputs: - for other_input in other_con.inputs: - self._check_and_connect(old_output, other_input, emitter) - - # Preserve old connections - self._merge_connections(other_con, emitter) - - def merge_all(self, tokens, emitter): - """ - Merge all the tokens with this class - :type tokens: list[list[Connection | Ident]] - :param tokens: List of list of tokens - :type emitter: emit.Emitter - :param emitter: Emitter to report errors - """ - if len(tokens) == 1: - if len(tokens[0]) > 0: - for tok in tokens[0]: - other_con = into_connection(tok) - self.merge_with(other_con, emitter) - - def merge_and_reset_inputs_outputs(self, other_con, emitter): - """ - Merge this connection with other_con and reset inputs / outputs - as they're no longer necessary. - :type other_con: Connection - :param other_con: the other connection we are gonna merge with. - :type emitter: emit.Emitter - :param emitter: Emitter to report errors - """ - self.inputs = [] - self.outputs = [] - self._merge_connections(other_con, emitter) - - def merge_with(self, other_con, emitter): - """ - Merge the provided connection with this one. - :type other_con: Connection - :param other_con: Connection to merge with self. - :type emitter: emit.Emitter - :param emitter: Emitter to report errors - """ - def extend(into, iterable, what): - for other_thing in iterable: - if len(list(filter(lambda x: x.val == other_thing.val, - into))) > 0: - emitter.emit_warning( - other_thing.span, - "{} {} already present".format( - what, other_thing.val - ) - ) - else: - into.append(other_thing) - extend(self.inputs, other_con.inputs, 'Input') - extend(self.outputs, other_con.outputs, 'Output') - self._merge_connections(other_con, emitter) - - def _merge_connections(self, other_con, emitter): - """ - Merge only the connections field from other_con into self. - :type other_con: Connection - :param other_con: Connection to merge with self. - :type emitter: emit.Emitter - :param emitter: Emitter to report errors - """ - for ident_from, ident_to in other_con.connections: - self._check_and_connect(ident_from, ident_to, emitter) - - def _check_and_connect(self, ident_from, ident_to, emitter): - """ - Check if the connection does not already exists and if it does not, - add it to the list of connections. Otherwise report a warning and - do nothing. - - :type ident_from: Ident - :param ident_from: The 'From' node of the directed edge. - :type ident_to: Ident - :param ident_to: The 'To' node of the directed edge we are creating. - :type emitter: emit.Emitter - :param emitter: Emitter to report errors. - """ - if ident_from.val in self.connections_cache: - if ident_to.val in self.connections_cache[ident_from.val]: - emitter.emit_warning( - ident_to.span, - "Connection from '{}' to '{}'" - " is already present" - .format(ident_from.val, ident_to.val) - ) - return - self.connections_cache[ident_from.val].append(ident_to.val) - else: - self.connections_cache[ident_from.val] = [ident_to.val] - self.connections.append((ident_from, ident_to)) - - def _build_connection_cache(self): - """ - Build a cache of connections keyed by where they start from. - """ - for ident_from, ident_to in self.connections: - if ident_from.val not in self.connections_cache: - self.connections_cache[ident_from.val] = [] - if ident_to.val not in self.connections_cache: - self.connections_cache[ident_to.val] = [] - self.connections_cache[ident_from.val].append(ident_to.val) - # Sanity check - for _, vals in self.connections_cache: - if len(set(vals)) != len(vals): - raise p.ParseFatalException("Bug found in Connection!!") - - def into_unmodified_str(self): - # TODO(Joan): improve this - return "connection" - - def __str__(self): - res = "Connection<" - res += " {} ".format([(str(x[0]), str(x[1])) - for x in self.connections]) - res += ">" - return res - - -class Assignment(ASTNode): - - def __init__(self, span, dot_path, value): - """ - Construct an assignment AST node. - :type span: Span - :param span: the span of the assignment. - :type dot_path: DotPath - :param dot_path: the left hand side of the assignment. - :type value: Component | Number | StringLit | JsonObj | DotPath | Expr - :param value: the right hand side of the assignment. - """ - super(Assignment, self).__init__(span) - if (isinstance(value, Component) or - isinstance(value, JsonObj) or - isinstance(value, Number) or - isinstance(value, StringLit) or - isinstance(value, Ident) or - isinstance(value, DotPath) or - isinstance(value, Expr)) and\ - isinstance(dot_path, DotPath): - self.lhs = dot_path - self.rhs = value - else: - raise exception.BananaGrammarBug( - 'Impossible assignment found with' - ' left hand side: {} and' - ' right hand side: {}' - .format(type(dot_path), type(value)) - ) - - def into_unmodified_str(self): - return "{} = {}".format(self.lhs.into_unmodified_str(), - self.rhs.into_unmodified_str()) - - def __str__(self): - return "{} = {}".format(str(self.lhs), str(self.rhs)) - - -class ComponentCtorArg(ASTNode): - - def __init__(self, span, value, arg_name=None): - """ - Construct an argument for a component ctor - :type span: Span - :param span: Span of the argument. - :type value: Number | StringLit | JsonObj | DotPath | Expr - :param value: Value for the argument - :type arg_name: Ident - :param arg_name: Name of the argument - """ - super(ComponentCtorArg, self).__init__(span) - if (isinstance(value, JsonObj) or - isinstance(value, Number) or - isinstance(value, StringLit) or - isinstance(value, Ident) or - isinstance(value, DotPath) or - isinstance(value, Expr)) and ( - isinstance(arg_name, Ident) or - arg_name is None): - self.arg_name = arg_name - self.value = value - else: - # This code should be unreachable. - # The grammar as defined should prevent us from - # seeing an arg value or a value of the incorrect type - raise exception.BananaGrammarBug( - 'Impossible constructor argument found with' - ' left hand side: {} and' - ' right hand side: {}' - .format(type(arg_name), type(value)) - ) - - def into_unmodified_str(self): - return "{} = {}".format(self.arg_name.into_unmodified_str(), - self.value.into_unmodified_str()) - - def __str__(self): - if self.arg_name is not None: - return "{} = {}".format(self.arg_name, self.value) - else: - return "{}".format(self.value) - - -class Component(ASTNode): - - def __init__(self, span, type_name=None, args=None): - """ - Construct a component - :type span: Span - :param span: Span of this component - :type type_name: Ident - :param type_name: Name of this component - :type args: list[ComponentCtorArg] - :param args: List of arguments - """ - super(Component, self).__init__(span) - if args is None: - args = [] - self.type_name = type_name - self.args = args - - def set_ctor(self, type_name): - """ - Set the constructor name of that component - :type type_name: Ident - :param type_name: Name of that constructor - """ - self.type_name = type_name - - def add_arg(self, arg): - """ - Add an argument to that component constructor. - :type arg: ComponentCtorArg - :param arg: Argument to add to that component. - """ - self.args.append(arg) - - def into_unmodified_str(self): - return self.type_name.into_unmodified_str() + "(" + \ - ', '.join(map(lambda x: x.into_unmodified_str(), self.args)) +\ - ")" - - def __str__(self): - res = "" - res += "Component {" - res += " type_name: {},".format(self.type_name) - arr = ', '.join(map(lambda x: str(x), self.args)) - res += " args: {}".format("[" + arr + "]") - res += "}" - return res diff --git a/monasca_analytics/banana/grammar/base_ast.py b/monasca_analytics/banana/grammar/base_ast.py deleted file mode 100644 index 6310888..0000000 --- a/monasca_analytics/banana/grammar/base_ast.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import six - - -@six.add_metaclass(abc.ABCMeta) -class ASTNode(object): - """ - Base class of all ast nodes - """ - def __init__(self, span): - """ - Construct an ASTNode - :type span: Span - :param span: span for this AST node. - """ - self.span = span - - @abc.abstractmethod - def into_unmodified_str(self): - """ - Returns a simple name for this ASTNode. It should be minimalist - and user oriented. No span info, no debug info. - :rtype: str - :returns: A simple name for that ast node. - """ - pass - - def __ne__(self, other): - return not self.__eq__(other) - - -class Span(object): - """ - Represent a region of code, used for error reporting. - Position are absolute within the file. - """ - - def __init__(self, text, lo, hi): - """ - :type text: str | None - :param text: Full text of the file - :type lo: int - :param lo: position of the beginning of the region - :type hi: int - :param hi: position of the end of the region - """ - self._text = text - self.lo = lo - self.hi = hi - - def __str__(self): - if self._text is not None: - return self._text[self.lo:self.hi] - else: - return '?SPAN?' - - def new_with_lo(self, lo_val): - """ - Construct a new Span with an new value for - lo. - - :type lo_val: int - :param lo_val: New value for lo. - :rtype: Span - :return: Returns a new span - """ - return Span(self._text, lo_val, self.hi) - - def str_from_to(self, to_span): - """ - Returns a string that start at self and stops at to_span. - :type to_span: Span - :param to_span: Span to stop at. - :rtype: six.string_types - :return: Returns the string encapsulating both - """ - return self._text[self.lo:to_span.hi] - - def get_line(self): - """ - Returns the line for associated with this span. - """ - if self._text is not None: - splitted = self._text.splitlines() - current_pos = 0 - for line in splitted: - if current_pos < self.lo < len(line) + current_pos: - return line.strip() - else: - current_pos += len(line) - else: - return '?LINE?' - - def get_range(self): - """ - Returns the start and end (line number, column number) of this span. - """ - if self._text is not None: - splitted = self._text.splitlines() - current_pos = 0 - startlineno = 0 - startcolno = 0 - endlineno = 0 - endcolno = 0 - for lineno in range(0, len(splitted)): - line = splitted[lineno] - if current_pos <= self.lo <= len(line) + current_pos: - startlineno = lineno + 1 - startcolno = self.lo - current_pos + 1 - if current_pos <= self.hi <= len(line) + current_pos: - endlineno = lineno + 1 - endcolno = self.hi - current_pos + 1 - current_pos += len(line) + 1 - return (startlineno, startcolno), (endlineno, endcolno) - else: - return (0, 0), (0, 0) - - def get_lineno(self): - """ - Returns the line number of this span. - """ - if self._text is not None: - splitted = self._text.splitlines() - current_pos = 0 - lineno = 0 - for _ in range(0, len(splitted)): - line = splitted[lineno] - if current_pos < self.lo < len(line) + current_pos: - return lineno + 1 - else: - current_pos += len(line) - lineno += 1 - return lineno - else: - return -1 - -DUMMY_SPAN = Span(None, 0, 0) - - -def from_pyparsing_exception(parse_fatal_exception): - """ - Convert the provided ParseFatalException into a Span. - - :type parse_fatal_exception: pyparsing.ParseBaseException - :param parse_fatal_exception: Exception to convert. - :rtype: Span - :return: Returns the span mapping to that fatal exception. - """ - return Span( - parse_fatal_exception.pstr, - parse_fatal_exception.loc, - parse_fatal_exception.loc + 1 - ) diff --git a/monasca_analytics/banana/grammar/config.py b/monasca_analytics/banana/grammar/config.py deleted file mode 100644 index 9dc468c..0000000 --- a/monasca_analytics/banana/grammar/config.py +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pyparsing as p - -import monasca_analytics.banana.emitter as emit -import monasca_analytics.banana.grammar.ast as ast -import monasca_analytics.banana.grammar.const as const -import monasca_analytics.exception.banana as exception - - -# This file describe the grammar for the banana config file. -# It make use of one sub grammar for certain configuration -# that requires expressions (see expression.py file) - - -def banana_grammar(emitter=emit.PrintEmitter()): - """ - Generate a banana parser that can be then used to - parse a banana content. It build an AST on which - operation can then be applied. - :return: Return a banana parser - :rtype: BananaScopeParser - """ - # Should debug - debug_grammar = False - - # Actions - def action_str_lit(s, l, t): - return ast.StringLit(ast.make_span(s, l, t), t[0]) - - def action_num_lit(s, l, t): - return ast.Number(ast.make_span(s, l, t), t[0]) - - def action_ident(s, l, t): - return ast.Ident(ast.make_span(s, l, t), t[0]) - - def action_expr(s, l, t): - if len(t) != 1: - raise exception.BananaGrammarBug( - 'Bug found in the grammar for expression,' - ' Please report this bug.' - ) - if isinstance(t[0], ast.Expr): - return t[0] - return ast.Expr(ast.make_span(s, l, t), t[0]) - - def action_dot_path(s, l, t): - # First token is the name of the variable - # The rest is the property path - if isinstance(t[0], ast.StringLit) and len(t[1:]) == 0: - return t[0] - return ast.DotPath(ast.make_span(s, l, t), t[0], t[1:]) - - def action_json_obj(s, l, t): - return ast.JsonObj(ast.make_span(s, l, t), t) - - def action_parse_ctor_arg(s, l, t): - if len(t) > 1: - return ast.ComponentCtorArg(ast.make_span(s, l, t), t[1], t[0]) - else: - return ast.ComponentCtorArg(ast.make_span(s, l, t), t[0]) - - def action_parse_comp_ctor(s, l, tokens): - comp = ast.Component(ast.make_span(s, l, tokens)) - for tok in tokens: - if isinstance(tok, ast.Ident): - comp.set_ctor(tok) - elif isinstance(tok, ast.ComponentCtorArg): - comp.add_arg(tok) - else: - raise exception.BananaGrammarBug( - 'Bug found in the grammar, Please report this bug' - ) - return comp - - def action_assignment(s, l, t): - return ast.Assignment(ast.make_span(s, l, t), t[0], t[1]) - - def action_create_connections(s, l, t): - ast_conn = ast.into_connection(t[0]) - ast_conn.span = ast.make_span(s, l, t) - for i in range(1, len(t)): - next_conn = ast.into_connection(t[i]) - ast_conn.connect_to(next_conn, emitter) - return ast_conn - - def action_merge_connections(s, l, t): - ast_conn = ast.Connection(ast.make_span(s, l, t)) - ast_conn.merge_all(t, emitter) - return ast_conn - - def action_root_ast(s, l, tokens): - root = ast.BananaFile(emitter) - for tok in tokens: - if isinstance(tok, ast.Assignment): - if isinstance(tok.rhs, ast.Component): - root.add_component_ctor(tok.lhs, tok.rhs) - else: - root.add_assignment(tok.lhs, tok.rhs) - elif isinstance(tok, ast.Connection): - root.add_connections(tok) - else: - raise exception.BananaGrammarBug( - 'Bug found in the grammar, Please report this bug.' - ) - return root - - # TODO(Joan): Remove once it is no longer needed - def print_stmt(s, l, t): - print("\nPRINT AST") - print((l, [str(x) for x in t])) - print("END PRINT AST\n") - - def action_unimplemented(s, l, t): - raise exception.BananaGrammarBug("unimplemented code reached") - - # Tokens - equals = p.Literal("=").suppress().setName('"="').setDebug(debug_grammar) - arrow = p.Literal("->").suppress().setName('"->"').setDebug(debug_grammar) - lbra = p.Literal("[").suppress().setName('"["').setDebug(debug_grammar) - rbra = p.Literal("]").suppress().setName('"]"').setDebug(debug_grammar) - colon = p.Literal(":").suppress().setName('":"') - comma = p.Literal(",").suppress().setName(",") - less = p.Literal("<").suppress().setName('"<"') - greater = p.Literal(">").suppress().setName('">"') - lbrace = p.Literal("{").suppress().setName('"{"').setDebug(debug_grammar) - rbrace = p.Literal("}").suppress().setName('"}"').setDebug(debug_grammar) - lpar = p.Literal("(").suppress().setName('"("') - rpar = p.Literal(")").suppress().setName('")"') - - # Keywords - ing = p.Literal("ing").suppress() - imp = p.Literal("import").suppress() - fro = p.Literal("from").suppress() - - # String Literal, Numbers, Identifiers - string_lit = p.quotedString()\ - .setParseAction(action_str_lit)\ - .setName(const.STRING_LIT) - number_lit = p.Regex(r'\d+(\.\d*)?([eE]\d+)?')\ - .setParseAction(action_num_lit)\ - .setName(const.NUMBER) - ident = p.Word(p.alphas + "_", p.alphanums + "_")\ - .setParseAction(action_ident)\ - .setName(const.IDENT) - - # Path for properties - dot_prop = ident | string_lit - dot_path = p.delimitedList(dot_prop, ".")\ - .setParseAction(action_dot_path)\ - .setName(const.DOT_PATH)\ - .setDebug(debug_grammar) - - # Expressions - - # Here to simplify the logic, we can match directly - # against ident and string_lit to avoid having to deal - # only with dot_path. It also allow to remove the confusion - # where '"a"' could be interpreted as a dot_path and would thus - # be the same as 'a'. With the following, the first we - # always be type-checked as a String whereas the latter will - # be as the type of the variable. - expr = p.infixNotation(number_lit | dot_path, [ - (p.oneOf('* /'), 2, p.opAssoc.LEFT), - (p.oneOf('+ -'), 2, p.opAssoc.LEFT), - ], lpar=lpar, rpar=rpar) - expr.setParseAction(action_expr)\ - .setName(const.EXPR)\ - .setDebug(debug_grammar) - - # Json-like object (value are much more) - json_obj = p.Forward() - json_value = p.Forward() - json_array = p.Group( - lbra + p.Optional(p.delimitedList(json_value)) + rbra - ) - json_array.setDebug(debug_grammar) - json_array.setName(const.JSON_ARRAY) - json_value <<= expr | json_obj | json_array - json_value.setDebug(debug_grammar)\ - .setName(const.JSON_VALUE) - json_members = p.delimitedList(p.Group(dot_path + colon - json_value)) +\ - p.Optional(comma) - json_members.setDebug(debug_grammar)\ - .setName(const.JSON_MEMBERS) - json_obj <<= p.Dict(lbrace + p.Optional(json_members) - rbrace) - json_obj.setParseAction(action_json_obj)\ - .setName(const.JSON_OBJ)\ - .setDebug(debug_grammar) - - # Component constructor - arg = (ident + equals - (expr | json_obj)) | expr | json_obj - arg.setParseAction(action_parse_ctor_arg) - params = p.delimitedList(arg) - comp_ctor = ident + lpar - p.Optional(params) + rpar - comp_ctor.setParseAction(action_parse_comp_ctor)\ - .setName(const.COMP_CTOR)\ - .setDebug(debug_grammar) - - # Assignments - assignment = dot_path + equals - (comp_ctor | expr | json_obj) - assignment.setParseAction(action_assignment) - - # Connections - connection = p.Forward() - array_of_connection = p.Group( - lbra + p.Optional(p.delimitedList(connection)) + rbra - ) - array_of_connection.setParseAction(action_merge_connections) - last_expr = ident | array_of_connection - this_expr = p.Forward() - match_expr = p.FollowedBy(last_expr + arrow - last_expr) + \ - (last_expr + p.OneOrMore(arrow - last_expr)) - this_expr <<= match_expr | last_expr - connection <<= this_expr - - match_expr.setDebug(debug_grammar)\ - .setName(const.CONNECTION) \ - .setParseAction(action_create_connections) - - # Definitions - definition = ing - less - string_lit - greater - ident - lbrace - rbrace - definition.setDebug(debug_grammar)\ - .setName(const.DEFINITION)\ - .setParseAction(action_unimplemented) - - # Import directive - module_def = (imp - ident) | fro - ident - imp - ident - module_def.setDebug(debug_grammar)\ - .setName(const.MOD_IMPORT)\ - .setParseAction(action_unimplemented) - - # Comments - comments = "#" + p.restOfLine - - statement = assignment | \ - match_expr | \ - definition | \ - module_def - - statement.setName(const.STATEMENT) - statement.setDebug(debug_grammar) - statement.setParseAction(print_stmt) - - # Grammar - grammar = p.OneOrMore(statement).ignore(comments) - grammar.setParseAction(action_root_ast) - - return BananaScopeParser(grammar) - - -class BananaScopeParser(object): - """ - Aggregate and resolve conflicts as everything was define - within the same scope. Usefull for have cpp "include"-like - functionality when importing another file. - """ - - def __init__(self, grammar): - self._grammar = grammar - - def parse(self, string): - """ - Parse the given input string. - :type string: str - :param string: Input string. - :rtype: ast.BananaFile - :return: Returns the ast root. - """ - tree = self._grammar.parseString(string, parseAll=True)[0] - return tree diff --git a/monasca_analytics/banana/grammar/const.py b/monasca_analytics/banana/grammar/const.py deleted file mode 100644 index 167196a..0000000 --- a/monasca_analytics/banana/grammar/const.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -ASSIGNMENT = "assignment" -CONNECTION = "connection" -DEFINITION = "definition" -MOD_IMPORT = "mod_import" -DOT_PATH = "path" -NUMBER = "number" -STRING_LIT = "string_lit" -IDENT = "ident" -EXPR = "expression" -JSON_OBJ = "jsonlike_obj" -STATEMENT = "statement" -JSON_ARRAY = "jsonlike_array" -JSON_VALUE = "jsonlike_value" -JSON_MEMBERS = "jsonlike_members" -COMP_CTOR = "component_ctor" diff --git a/monasca_analytics/banana/pass_manager.py b/monasca_analytics/banana/pass_manager.py deleted file mode 100644 index 36a4ca5..0000000 --- a/monasca_analytics/banana/pass_manager.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pyparsing as p - -import monasca_analytics.banana.deadpathck.config as deadpathck -import monasca_analytics.banana.emitter as emit -import monasca_analytics.banana.eval.config as ev -import monasca_analytics.banana.eval.ctx as ctx -import monasca_analytics.banana.grammar.base_ast as span_util -import monasca_analytics.banana.grammar.config as grammar -import monasca_analytics.banana.typeck.config as typeck -import monasca_analytics.exception.banana as exception - - -def execute_banana_string(banana, driver=None, emitter=emit.PrintEmitter()): - """ - Execute the provided banana string. - It will run the parse phase, and the typechecker. - :type banana: str - :param banana: The string to parse and type check. - :type driver: monasca_analytics.spark.driver.DriverExecutor | None - :param driver: Driver that will manage the created - components and connect them together. - :type emitter: emit.Emitter - :param emitter: Emitter for reporting errors/warning. - """ - try: - # Convert the grammar into an AST - parser = grammar.banana_grammar(emitter) - ast = parser.parse(banana) - # Compute the type table for the given AST - type_table = typeck.typeck(ast) - # Remove from the tree path that are "dead" - deadpathck.deadpathck(ast, type_table, emitter) - # Check that there's at least one path to be executed - deadpathck.contains_at_least_one_path_to_a_sink(ast, type_table) - # Evaluate the script - if driver is not None: - ev.eval_ast(ast, type_table, driver) - except exception.BananaException as err: - emitter.emit_error(err.get_span(), str(err)) - except p.ParseSyntaxException as err: - emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg) - except p.ParseFatalException as err: - emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg) - except p.ParseException as err: - emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg) - - -def try_compute_type_table(banana): - """ - Compute the type table for the provided banana string - if possible. Does not throw any exception if it fails. - :type banana: str - :param banana: The string to parse and type check. - """ - try: - # Convert the grammar into an AST - parser = grammar.banana_grammar() - ast = parser.parse(banana) - # Compute the type table for the given AST - return typeck.typeck(ast) - except exception.BananaException: - return None - except p.ParseSyntaxException: - return None - except p.ParseFatalException: - return None - except p.ParseException: - return None - - -def compute_type_table(banana): - """ - Compute the type table for the provided banana string - if possible. - :type banana: str - :param banana: The string to parse and type check. - """ - # Convert the grammar into an AST - parser = grammar.banana_grammar() - ast = parser.parse(banana) - # Compute the type table for the given AST - return typeck.typeck(ast) - - -def compute_evaluation_context(banana, cb=lambda *a, **k: None): - """ - Compute the evaluation context for the provided - banana string. - :type banana_str: str - :param banana_str: The string to parse and type check. - :param cb: Callback called after each statement - """ - parser = grammar.banana_grammar() - ast = parser.parse(banana) - type_table = typeck.typeck(ast) - context = ctx.EvaluationContext() - - def custom_cb(_type, lhs, value): - cb(context, _type, lhs, value) - - ev.eval_statements_generic(ast.statements, type_table, context, custom_cb) diff --git a/monasca_analytics/banana/typeck/README.md b/monasca_analytics/banana/typeck/README.md deleted file mode 100644 index 226b1ed..0000000 --- a/monasca_analytics/banana/typeck/README.md +++ /dev/null @@ -1,85 +0,0 @@ -## Type-checker - -This folder is all about the type checking of `banana` files. -The type checker purpose is to verify that components exist, -the type of local variable matches the requirements of components -parameters and assignments between them are correct. It also -checks that connections between components are valid. - -The biggest difference between the old `validation` of the JSON -format is that we have more information available. We can warn -users when they make mistakes and point at the exact locations -using `Span`. Also, the type table generated is used by other passes -to perform other static analyses. - -This is the second step of the pipeline: -``` - +-------+ +---------------------+ - | | | | - ---> | AST | ---- typeck ---> | AST & TypeTable | ---> - | | | | - +-------+ +---------------------+ -``` - -The module `type_util.py` contains all the possible types that are -known by the type checker. The `TypeTable` built lives in the -`type_table.py` module. - -### Current status - -* [x] Type check numbers -* [x] Type check string literals -* [x] Type check variable assignments -* [x] Type check component assignments -* [x] Type check component parameters -* [x] Type check connections -* [x] Resolve variable names -* [ ] Resolve imports -* [ ] Type check disconnections - -### Tests - -All tests for the type checker (i.e. making sure that -inferred types are correct and that errors are raised in -appropriate situation) lives in `test/banana/typeck`. - -This folder looks like this: - -``` -test/banana/typeck -├── should_fail -│   ├── ... -│   └── file.banana -├── should_pass -│   ├── ... -│   └── file.banana -└── test_typeck_config.py -``` - -The `test_typeck_config`, generates one test for each file -in the `should_pass` and `should_fail` directories. - -For each generated test, we basically run the following passes: - -* `grammar`: convert the input text into an AST. -* `typeck`: run the type checker. - -Tests can assert various things in `banana` comments: - - - In the `should_fail` directory, a test is expected to use - the `RAISE` instruction to specify a type of exceptions - that should be raised by the test. - - In the `should_pass` directory, a test is expected to not - raised any exception **and** to specify the state of the - `TypeTable` when the type checker has type checked everything - in the file. This is done with the `TYPE_TABLE_EQ` instruction. - - -#### Available instruction - -* `# RAISE `: Check that `exception-name` is raised. -* `# TYPE_TABLE_EQ ` -* `# NEW_TEST`: This instruction splits the file into two tests. However, - the expected exception or type table should still be the same. It - allows us to verify what should be semantically equivalent in the - grammar. \ No newline at end of file diff --git a/monasca_analytics/banana/typeck/__init__.py b/monasca_analytics/banana/typeck/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/banana/typeck/config.py b/monasca_analytics/banana/typeck/config.py deleted file mode 100644 index 2af0d78..0000000 --- a/monasca_analytics/banana/typeck/config.py +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monasca_analytics.ingestor.base as ingestor -import monasca_analytics.ldp.base as ldp -import monasca_analytics.sink.base as sink -import monasca_analytics.sml.base as sml -import monasca_analytics.source.base as source -import monasca_analytics.voter.base as voter - -import monasca_analytics.banana.grammar.ast as ast -import monasca_analytics.banana.typeck.connections as conn -import monasca_analytics.banana.typeck.type_table as typetbl -import monasca_analytics.banana.typeck.type_util as u -import monasca_analytics.exception.banana as exception -import monasca_analytics.exception.monanas as exception_monanas -import monasca_analytics.util.common_util as introspect - - -import six - - -def typeck(banana_file): - """ - Type-check the provided BananaFile instance. - If it type check, it returns the associated TypeTable. - :type banana_file: ast.BananaFile - :param banana_file: The file to typecheck. - :rtype: typetbl.TypeTable - :return: Returns the TypeTable for this BananaFile - """ - type_table = typetbl.TypeTable() - statement_index = 0 - for stmt in banana_file.statements: - lhs, rhs = stmt - type_computed = typeck_rhs(rhs, type_table) - type_table.set_type(lhs, type_computed, statement_index) - statement_index += 1 - conn.typeck_connections(banana_file.connections, type_table) - return type_table - - -def typeck_rhs(ast_value, type_table): - """ - Type-check the provided ast value. And returns its type. - This function does not support assignment, - :type ast_value: ast.ASTNode - :param ast_value: The ast_value to type check. - :type type_table: typetbl.TypeTable - :param type_table: The type table. Used for type lookup. - :rtype: u.Component | u.Object | u.Number | u.String - :return: Returns the computed type. - """ - if isinstance(ast_value, ast.Number): - return u.Number() - if isinstance(ast_value, ast.StringLit): - return u.String() - if isinstance(ast_value, ast.Ident): - return type_table.get_type(ast_value) - if isinstance(ast_value, ast.DotPath): - return type_table.get_type(ast_value) - if isinstance(ast_value, ast.Expr): - return typeck_expr(ast_value, type_table) - if isinstance(ast_value, ast.JsonObj): - return typeck_jsonobj(ast_value, type_table) - if isinstance(ast_value, ast.Component): - return typeck_component(ast_value, type_table) - raise Exception("Unhandled ast value type {}!!".format(ast_value)) - - -def typeck_jsonobj(json_obj, type_table): - """ - Type-check a json-like object. If it succeeds - it return the appropriate type describing this - json like object. Raise an exception otherwise. - :type json_obj: ast.JsonObj - :param json_obj: The JsonObj ast node. - :type type_table: typetbl.TypeTable - :param type_table: The type table. - :rtype: u.Object - :return: Returns an instance of util.Object describing - the full type of this json object. - """ - root_type = u.Object(strict_checking=False) - - for k, v in six.iteritems(json_obj.props): - sub_type = u.create_object_tree(k, typeck_rhs(v, type_table)) - u.attach_to_root(root_type, sub_type, json_obj.span) - - return root_type - - -def typeck_expr(expr, type_table): - """ - Type-check the given expression. If the typecheck - pass, the resulting type will be used for the strategy - to use when evaluating this expression. - :type expr: ast.Expr - :param expr: The expression to typecheck. - :type type_table: typetbl.TypeTable - :param type_table: Type of the table - :rtype: u.Number | u.String - :return: Returns the type of the expression if possible - :raise: Raise an exception - """ - # In the case where we are just wrapping around - # only one expression, the logic below - # needs to be skipped. - if len(expr.expr_tree) == 1: - return typeck_rhs(expr.expr_tree[0], type_table) - - _type = None - must_be_number = False - - def check_type(old_type, new_type): - if new_type == old_type: - return old_type - elif new_type == u.String(): - if must_be_number: - raise exception.BananaTypeError( - expected_type=u.Number, - found_type=new_type - ) - if old_type is None: - return new_type - elif u.can_to_str(old_type): - return new_type - else: - raise exception.BananaTypeError( - expected_type=old_type, - found_type=new_type - ) - elif new_type == u.Number(): - if old_type is None: - return new_type - elif old_type == u.String(): - return old_type - elif not old_type == u.Number(): - raise exception.BananaTypeError( - expected_type=old_type, - found_type=new_type - ) - else: - raise exception.BananaTypeError( - expected_type=old_type, - found_type=new_type - ) - - def allowed_symbol(current_type): - if current_type == u.String(): - return ['+'] - else: - return ['+', '-', '*', '/'] - - for el in expr.expr_tree: - if isinstance(el, ast.StringLit): - _type = check_type(_type, u.String()) - elif isinstance(el, ast.Number): - _type = check_type(_type, u.Number()) - elif isinstance(el, ast.Ident): - ident_type = type_table.get_type(el) - _type = check_type(_type, ident_type) - elif isinstance(el, ast.DotPath): - dotpath_type = type_table.get_type(el) - _type = check_type(_type, dotpath_type) - elif isinstance(el, ast.Expr): - _type = check_type(_type, typeck_expr(el, type_table)) - elif isinstance(el, six.string_types): - if el not in allowed_symbol(_type): - raise exception.BananaUnknownOperator(expr.span, el, _type) - if el in ['-', '*', '/']: - must_be_number = True - else: - raise exception.BananaTypeError( - expected_type=[u.Number.__name__, u.String.__name__, - u.Object.__name__], - ) - - # The final type if we made until here! - return _type - - -def typeck_component(component, type_table): - """ - Type-check the provided component. Returns - the appropriate subclass of util.Component if - successful, or raise an exception if there's - an error. - :type component: ast.Component - :param component: The component ast node. - :type type_table: typetbl.TypeTable - :param type_table: the type table. - :rtype: u.Source | u.Sink | u.Voter | u.Ldp | u.Sml | u.Ingestor - :return: Returns the appropriate type for the component. - """ - # TODO(Joan): This wont't work for type that are defined - # TODO(Joan): at the language level. We need a registration service - # TODO(Joan): to manage the Types of component that we can create - # TODO(Joan): instead of this hacky function call. - try: - component_type = introspect.get_class_by_name(component.type_name.val) - comp_params = component_type.get_params() - except exception_monanas.MonanasNoSuchClassError: - raise exception.BananaUnknown( - component - ) - - # Compute the type of the component - if issubclass(component_type, source.BaseSource): - comp_type = u.Source(component_type.__name__, comp_params) - elif issubclass(component_type, sink.BaseSink): - comp_type = u.Sink(component_type.__name__, comp_params) - elif issubclass(component_type, sml.BaseSML): - comp_type = u.Sml(component_type.__name__, comp_params) - elif issubclass(component_type, voter.BaseVoter): - comp_type = u.Voter(component_type.__name__, comp_params) - elif issubclass(component_type, ldp.BaseLDP): - comp_type = u.Ldp(component_type.__name__, comp_params) - elif issubclass(component_type, ingestor.BaseIngestor): - comp_type = u.Ingestor(component_type.__name__, comp_params) - else: - raise exception.BananaTypeCheckerBug("Couldn't find a type for '{}'" - .format(component.type_name.val)) - - # Type check the parameters - if len(component.args) > len(comp_params): - raise exception.BananaComponentTooManyParams(component.span) - - # Does saying that parameter should either all have a name - # or non at all satisfying? -> Yes - # Are parameter all named? - all_named = -1 - for arg in component.args: - if arg.arg_name is not None: - if all_named == 0: - raise exception.BananaComponentMixingParams(arg.span, False) - all_named = 1 - else: - if all_named == 1: - raise exception.BananaComponentMixingParams(arg.span, True) - all_named = 0 - - if all_named == 1: - for arg in component.args: - param = list(filter(lambda x: - x.param_name == arg.arg_name.inner_val(), - comp_params)) - if len(param) != 1: - raise exception.BananaComponentIncorrectParamName( - component=component.type_name, - found=arg.arg_name - ) - param = param[0] - expr_type = typeck_rhs(arg.value, type_table) - if not u.can_be_cast_to(expr_type, param.param_type): - raise exception.BananaArgumentTypeError( - where=arg, - expected_type=param.param_type, - received_type=expr_type - ) - else: - for arg, param in zip(component.args, comp_params): - arg.arg_name = ast.Ident(arg.span, param.param_name) - expr_type = typeck_rhs(arg.value, type_table) - if not u.can_be_cast_to(expr_type, param.param_type): - raise exception.BananaArgumentTypeError( - where=arg, - expected_type=param.param_type, - received_type=expr_type - ) - - return comp_type diff --git a/monasca_analytics/banana/typeck/connections.py b/monasca_analytics/banana/typeck/connections.py deleted file mode 100644 index ff78fe6..0000000 --- a/monasca_analytics/banana/typeck/connections.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monasca_analytics.banana.typeck.type_util as util -import monasca_analytics.exception.banana as exception - -valid_connections_types = { - util.Source: [util.Ingestor, util.Ldp], - util.Ingestor: [util.Sml, util.Sink], - util.Sml: [util.Voter, util.Sink], - util.Voter: [util.Ldp, util.Sink], - util.Ldp: [util.Sink], - util.Sink: [] -} - - -def typeck_connections(connection, type_table): - """ - Once all variable have been type-checked, we can - try to type-check connections. - :type connection: monasca_analytics.banana.grammar.ast.Connection - :param connection: The connection to type-check - :type type_table: monasca_analytics.banana.typeck.type_table.TypeTable - :param type_table: The table with all variable already type-checked. - :raise Raise an exception if there's a type error in connections. - """ - if connection is not None: - for ident_from, ident_to in connection.connections: - type_from = type_table.get_type(ident_from) - type_to = type_table.get_type(ident_to) - if not util.is_comp(type_from): - raise exception.BananaTypeError( - expected_type=util.Component(), - found_type=type_from - ) - if not util.is_comp(type_to): - raise exception.BananaTypeError( - expected_type=util.Component(), - found_type=type_to - ) - if type(type_to) not in valid_connections_types[type(type_from)]: - possible_types = map(lambda x: x.__name__, - valid_connections_types[type(type_from)]) - raise exception.BananaConnectionError( - connection.span, - ident_from, ident_to, type_from, possible_types - ) diff --git a/monasca_analytics/banana/typeck/type_table.py b/monasca_analytics/banana/typeck/type_table.py deleted file mode 100644 index 2db387f..0000000 --- a/monasca_analytics/banana/typeck/type_table.py +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy - -import monasca_analytics.banana.grammar.ast as ast -import monasca_analytics.banana.typeck.type_util as util -import monasca_analytics.exception.banana as exception -import monasca_analytics.util.string_util as strut - - -import six - - -class TypeTable(object): - """ - Type table. Support lookup for JsonLike object. - Json-like object have properties that needs to be - type-checked. The TypeTable allows to store - that information as well. All type values are - rooted by their variable name. - - Every-time a variable type is erased, we create a new - snapshot of the variables types. This allow to have - variable where the type change as the statement are - being executed. - """ - def __init__(self): - self._variables_snapshots = [(0, {})] - self._variables = self._variables_snapshots[0][1] - - def get_type(self, var, statement_index=None): - variables = self.get_variables(statement_index) - if isinstance(var, ast.Ident): - if var in variables: - return variables[var] - else: - raise exception.BananaUnknown(var) - # If we encounter a dot path: - if isinstance(var, ast.DotPath): - if var.varname in variables: - if len(var.properties) > 0: - return variables[var.varname][var.next_dot_path()] - else: - return variables[var.varname] - else: - raise exception.BananaUnknown(var.varname) - raise exception.BananaTypeCheckerBug("Unkown type for {}".format(var)) - - def set_type(self, var, _type, statement_index): - """ - Set the type for the given var to _type. - - :type var: ast.Ident | ast.DotPath - :param var: The var to set a type. - :type _type: util.Object | util.Component | util.String | util.Number - :param _type: The type for the var. - :type statement_index: int - :param statement_index: The statement at which this assignment was - made. - """ - if _type is None: - raise exception.BananaTypeCheckerBug( - "'None' is not a valid banana type" - ) - - if isinstance(var, ast.Ident): - self._check_needs_for_snapshot(var, _type, statement_index) - self._variables[var] = _type - return - - if isinstance(var, ast.DotPath): - if util.is_comp(_type) and len(var.properties) > 0: - raise exception.BananaAssignCompError(var.span) - - if len(var.properties) == 0: - self._check_needs_for_snapshot( - var.varname, - _type, - statement_index - ) - self._variables[var.varname] = _type - else: - if var.varname in self._variables: - var_type = self._variables[var.varname] - if isinstance(var_type, util.Object): - new_type = util.create_object_tree( - var.next_dot_path(), _type) - util.attach_to_root(var_type, new_type, var.span, - erase_existing=True) - elif isinstance(var_type, util.Component): - var_type[var.next_dot_path()] = _type - else: - raise exception.BananaTypeError( - expected_type=util.Object, - found_type=type(var) - ) - # Var undeclared, declare its own type - else: - new_type = util.create_object_tree(var.next_dot_path(), - _type) - self._variables[var.varname] = new_type - return - raise exception.BananaTypeCheckerBug("Unreachable code reached.") - - def get_variables(self, statement_index=None): - """ - Returns the list of variables with their associated type. - - :type statement_index: int - :param: Statement index. - :rtype: dict[str, util.Object|util.Component|util.String|util.Number] - """ - if statement_index is None: - return self._variables - - variables = {} - for created_at, snap in self._variables_snapshots: - if created_at < statement_index: - variables = snap - else: - break - - return variables - - def get_variables_snapshots(self): - return self._variables_snapshots - - def _check_needs_for_snapshot(self, var, _type, statement_index): - if var in self._variables: - # If we shadow a component, we need to raise an error - if util.is_comp(self._variables[var]): - raise exception.BananaShadowingComponentError( - where=var.span, - comp=self._variables[var].class_name - ) - - # If we change the type of the variable, we create a new snapshot: - # This is very strict but will allow to know exactly how - # the type of a variable (or a property) changed. - if self._variables[var] != _type: - self._create_snapshot(statement_index) - - def _create_snapshot(self, statement_index): - """ - Create a new snapshot of the variables. - :type statement_index: int - :param statement_index: index of the statement - (should be strictly positive) - """ - new_snapshot = copy.deepcopy(self._variables) - self._variables_snapshots.append(( - statement_index, new_snapshot - )) - self._variables = new_snapshot - - def to_json(self): - """ - Convert this type table into a dictionary. - Useful to serialize the type table. - - :rtype: dict - :return: Returns this type table as a dict. - """ - res = {} - for key, val in six.iteritems(self._variables): - res[key.inner_val()] = val.to_json() - return res - - def __contains__(self, key): - """ - Test if the type table contains or not the provided - path. This function is more permissive than the other two. - It will never raise any exception (or should aim not to). - :type key: six.string_types | ast.Ident | ast.DothPath - :param key: The key to test. - :return: Returns True if the TypeTable contains a type for the - given path or identifier. - """ - if isinstance(key, six.string_types): - return key in self._variables - - if isinstance(key, ast.Ident): - return key.val in self._variables - - if isinstance(key, ast.DotPath): - res = key.varname in self._variables - if not res: - return False - val = self._variables[key.varname] - for prop in key.properties: - if isinstance(val, util.Object): - if prop in val.props: - val = val[prop] - else: - return False - return True - - return False - - def __str__(self): - return strut.dict_to_str(self._variables) diff --git a/monasca_analytics/banana/typeck/type_util.py b/monasca_analytics/banana/typeck/type_util.py deleted file mode 100644 index 262e4e6..0000000 --- a/monasca_analytics/banana/typeck/type_util.py +++ /dev/null @@ -1,626 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Util files to manipulates banana types. - -The list of possible types is as follow: - - * `Number` - * `Boolean` - * `String` - * `Object` (Json-like object) - * `Component.Source.` - * `Component.Ingestor.` - * `Component.Sink.` - * `Component.Voter.` - * `Component.Ldp.` - * `Component.Sml.` - -where will be the component class name defined -in the code base. - -For type defined in banana such as Json parsers, -refers the name they are defined with. -""" -import abc -import six - -import monasca_analytics.banana.grammar.ast as ast -import monasca_analytics.exception.banana as exception -import monasca_analytics.util.string_util as strut - - -@six.add_metaclass(abc.ABCMeta) -class IsType(object): - """ - Any class that represents a Banana type should inherit - from this class. - """ - - def __ne__(self, other): - # Dispatch to eq function - return not self.__eq__(other) - - @abc.abstractmethod - def default_value(self): - pass - - @abc.abstractmethod - def to_json(self): - pass - - -class Any(IsType): - """ - Any type. This type should be used by component's writer when - they have a complex handling of parameters. This is not - recommended though as it move the error handling to - the component writer. - """ - - def __str__(self): - return "TypeAny" - - def __eq__(self, _): - # Any type is equal to nothing not even itself. - return False - - def __ne__(self, other): - return not self.__eq__(other) - - def __getitem__(self, _): - return Any() - - def __hash__(self): - raise Exception("Any type should not be used in dictionaries.") - - def default_value(self): - return {} - - def to_json(self): - return {"id": "any"} - - -class String(IsType): - """ - String Type. - """ - - def __str__(self): - return "TypeString" - - def __eq__(self, other): - return isinstance(other, String) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(str(self)) - - def default_value(self): - return "" - - def to_json(self): - return {"id": "string"} - - -class Number(String): - """ - Number type. Banana has only floating point value. - """ - - def __str__(self): - return "TypeNumber" - - def __eq__(self, other): - return isinstance(other, Number) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(str(self)) - - def default_value(self): - return 0 - - def to_json(self): - return {"id": "number"} - - -class Enum(String): - """ - Enum type. This type is a way to constraint a string or number, - to a specific set of values. - """ - - def __init__(self, variants): - self.variants = variants - - def __eq__(self, other): - return isinstance(other, Enum) and self.variants == other.variants - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.variants) - - def __str__(self): - return "TypeEnum < {} >".format(','.join(self.variants)) - - def default_value(self): - return "" - - def to_json(self): - return { - "id": "enum", - "variants": self.variants - } - - -def attach_to_root(root_obj, obj1, span, erase_existing=False): - """ - Attach the object obj1 to the root_obj object type. - - :type root_obj: Object - :param root_obj: The root object - :type obj1: Object - :param obj1: The object to attach. - :type span: Span - :param span: The span for this change. - :type erase_existing: bool - :param erase_existing: Set to true if the root type should - always be erased. - """ - for key, child_type in six.iteritems(obj1.props): - if key in root_obj.props: - root_sub_type = root_obj.props[key] - # Both are object -> recurse - if isinstance(root_sub_type, Object) and\ - isinstance(child_type, Object): - attach_to_root(root_sub_type, child_type, span, erase_existing) - elif erase_existing: - root_obj.props[key] = child_type - else: - raise exception.BananaTypeError( - expected_type=root_sub_type, - found_type=child_type, - span=span - ) - else: - # We can simply attach the new type! - root_obj.props[key] = child_type - - -def create_object_tree(dot_path, value): - """ - Create a linear tree of object type from the dot_path. - Also work when dot_path is an Ident or StringLit. - - :type dot_path: ast.DotPath | ast.Ident | ast.StringLit - :param dot_path: The ast node that forms a linear tree of type. - :type value: Object | String | Number - :param value: the value to set at the end of the linear tree. - :rtype: Object - :return: Returns the created object - """ - if is_comp(value): - raise exception.BananaAssignCompError(dot_path.span) - - # {a.b.c: value} - root_object = Object(strict_checking=False) - if isinstance(dot_path, ast.DotPath): - # {a: value} - if len(dot_path.properties) == 0: - root_object.props[dot_path.varname.inner_val()] = value - else: - # {a: } - root_object.props[dot_path.varname.inner_val()] = \ - Object(strict_checking=False) - # {b.c: value} - current_obj = root_object.props[dot_path.varname.inner_val()] - last_index = len(dot_path.properties) - 1 - for index, sub_prop in enumerate(dot_path.properties): - sub_prop_name = sub_prop.inner_val() - if index != last_index: - current_obj.props[sub_prop_name] = \ - Object(strict_checking=False) - current_obj = current_obj.props[sub_prop_name] - else: - current_obj.props[sub_prop_name] = value - else: - # Ident and StringLit are captured here. - root_object.props[dot_path.inner_val()] = value - return root_object - - -class Object(String): - """ - Object Type. The value that are dictionary-like have this type. - """ - - def __init__(self, props=None, strict_checking=True): - if props is None: - props = {} - self.props = props - # Strict checking is off for all objects defined within the banana - # language. It is on by default for components so that they can - # force the type checker to throw errors when we try to access - # or to modify unknown properties - self.strict_checking = strict_checking - - def __getitem__(self, key): - # a.b or a."b" - if isinstance(key, ast.Ident) or isinstance(key, ast.StringLit): - if key.inner_val() not in self.props: - raise exception.BananaPropertyDoesNotExists(key, - on_type=self) - return self.props[key.inner_val()] - - # a.b.c - if isinstance(key, ast.DotPath): - if key.varname.inner_val() not in self.props: - raise exception.BananaPropertyDoesNotExists(key.varname, - on_type=self) - sub_object = self.props[key.varname.inner_val()] - if len(key.properties) == 0: - return sub_object - # Recurse - if isinstance(sub_object, Object): - return sub_object[key.next_dot_path()] - if isinstance(sub_object, Any): - return sub_object - - raise exception.BananaPropertyDoesNotExists(key.next_dot_path(), - on_type=sub_object) - - raise exception.BananaTypeCheckerBug( - "Unreachable code in Object.__getitem__ reached." - ) - - def __str__(self): - if self.strict_checking: - return "TypeStruct < {} >".format(strut.dict_to_str(self.props)) - else: - return "TypeObject < {} >".format(strut.dict_to_str(self.props)) - - def __eq__(self, other): - return self.props == other - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.props) - - def default_value(self): - default_value = {} - for key, val in six.iteritems(self.props): - default_value[key] = val.default_value() - return default_value - - def to_json(self): - res = {"id": "object", "props": {}} - for key, val in six.iteritems(self.props): - res["props"][key] = val.to_json() - return res - - -class Component(IsType): - """ - Type of all components. While not strictly used directly, it - is very useful to performs checks on variable that are supposed - to be any of the available components. - """ - - def __init__(self, ctor_properties=None, class_name=None): - """ - Component type - - :type ctor_properties: - list[monasca_analytics.component.params.ParamDescriptor] - :param ctor_properties: - :type class_name: str - :param class_name: Name of the class if there's any. - """ - self.ctor_properties = ctor_properties - self.class_name = class_name - - def __str__(self): - if self.class_name is None: - return "TypeComponent" - else: - return self.class_name + "(" +\ - ",".join(map(lambda x: x.param_name + "=" + str(x.param_type), - self.ctor_properties))\ - + ")" - - def __setitem__(self, dot_path, value): - """ - Attempt to set the value at 'dot_path' to 'value'. - - :type dot_path: ast.DotPath - :param dot_path: The path of the property - :type value: String | Enum | Object | Number - :param value: The new type to set. - """ - if self.ctor_properties is None: - raise exception.BananaTypeCheckerBug( - "Component type can't have properties" - ) - - if len(dot_path.properties) == 0: - for arg in self.ctor_properties: - if arg.param_name == dot_path.varname.inner_val(): - if not can_be_cast_to(value, arg.param_type): - raise exception.BananaArgumentTypeError( - expected_type=arg.param_type, - received_type=value, - where=dot_path.span - ) - else: - return - else: - for arg in self.ctor_properties: - if arg.param_name == dot_path.varname.inner_val(): - if isinstance(arg.param_type, Any): - return - elif isinstance(arg.param_type, Object): - next_dot_path = dot_path.next_dot_path() - sub_arg_type = arg.param_type[next_dot_path] - if not can_be_cast_to(value, sub_arg_type): - raise exception.BananaArgumentTypeError( - expected_type=sub_arg_type, - received_type=value, - where=next_dot_path.span - ) - else: - return - else: - raise exception.BananaPropertyDoesNotExists( - dot_path.next_dot_path(), - arg.param_type - ) - - raise exception.BananaPropertyDoesNotExists(dot_path, on_type=self) - - def __getitem__(self, dot_path): - """ - Return the type of the given item. - - :type dot_path: ast.DotPath - :param dot_path: The path to follow - :return: - """ - if self.ctor_properties is None: - raise exception.BananaTypeCheckerBug( - "Component type can't have properties" - ) - - if len(dot_path.properties) == 0: - for arg in self.ctor_properties: - if arg.param_name == dot_path.varname.inner_val(): - return arg.param_type - else: - for arg in self.ctor_properties: - if arg.param_name == dot_path.varname.inner_val(): - if isinstance(arg.param_type, Object): - return arg.param_type[dot_path.next_dot_path()] - else: - raise exception.BananaPropertyDoesNotExists( - dot_path.next_dot_path(), - arg.param_type - ) - - raise exception.BananaPropertyDoesNotExists(dot_path, on_type=self) - - def __eq__(self, other): - return isinstance(other, Component) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(str(self)) - - def default_value(self): - return None - - def to_json(self): - res = {"id": "component", "name": self.class_name, "args": []} - for arg in self.ctor_properties: - res["args"].append(arg.to_json()) - return res - - -class Source(Component): - """ - Source type. All component that inherits from BaseSource have - this type in Banana. - """ - def __init__(self, class_name, ctor_properties): - super(Source, self).__init__(ctor_properties, class_name) - - def __eq__(self, other): - return self.class_name == other.class_name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.class_name) - - -class Ingestor(Component): - """ - Ingestor type. All component that inherits from BaseIngestor have - this type in Banana. - """ - def __init__(self, class_name, ctor_properties): - super(Ingestor, self).__init__(ctor_properties, class_name) - - def __eq__(self, other): - return self.class_name == other.class_name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.class_name) - - -class Sink(Component): - """ - Sink type. All component that inherits from BaseSink have - this type in Banana. - """ - def __init__(self, class_name, ctor_properties): - super(Sink, self).__init__(ctor_properties, class_name) - - def __eq__(self, other): - return self.class_name == other.class_name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.class_name) - - -class Voter(Component): - """ - Voter type. All component that inherits from BaseVoter have - this type in Banana. - """ - def __init__(self, class_name, ctor_properties): - super(Voter, self).__init__(ctor_properties, class_name) - - def __eq__(self, other): - return self.class_name == other.class_name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.class_name) - - -class Ldp(Component): - """ - Ldp type. All component that inherits from BaseLdp have - this type in Banana. - """ - def __init__(self, class_name, ctor_properties): - super(Ldp, self).__init__(ctor_properties, class_name) - - def __eq__(self, other): - return self.class_name == other.class_name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.class_name) - - -class Sml(Component): - """ - Sml type. All component that inherits from BaseSml have - this type in Banana. - """ - def __init__(self, class_name, ctor_properties): - super(Sml, self).__init__(ctor_properties, class_name) - - def __eq__(self, other): - return self.class_name == other.class_name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.class_name) - - -def get_type(ast_node): - """ - Returns the type for the given ast node. - This function only works for literal node such as - Number, StringLit and JsonObj. - :type ast_node: ast.Number | ast.StringLit | ast.JsonObj | ast.Component - :param ast_node: the node. - :return: Returns the appropriate type. - """ - if isinstance(ast_node, ast.Number): - return Number() - if isinstance(ast_node, ast.StringLit): - return String() - if isinstance(ast_node, ast.JsonObj): - return Object(strict_checking=False) - if isinstance(ast_node, ast.Component): - return Component() - return None - - -def can_to_str(_type): - """ - Check if we the type can be cast to str. - :param _type: Type to check - :return: Returns True if it can be casted - """ - return isinstance(_type, String) - - -def is_comp(_type): - """ - :type _type: String | Number | Object | Component - :param _type: Type to check. - :rtype: bool - :return: Returns True if the provided _type is a component - """ - return isinstance(_type, Component) - - -def can_be_cast_to(_type1, _type2): - """ - Check if the given type `_type1` can be cast into `_type2`. - :type _type1: String | Number | Enum | Object - :param _type1: Type to try to change into _type2 - :type _type2: String | Number | Enum | Object - :param _type2: Type reference. - :return: Returns true if the conversion can be done. - """ - if isinstance(_type2, Any): - return True - elif _type1 == _type2: - return True - elif _type2 == String(): - return can_to_str(_type1) - elif isinstance(_type2, Enum): - return isinstance(_type1, String) or isinstance(_type2, Enum) - elif isinstance(_type1, Object) and isinstance(_type2, Object): - if not _type2.strict_checking: - return True - else: - for prop_name, prop_type in six.iteritems(_type2.props): - if prop_name not in _type1.props: - return False - if not can_be_cast_to(_type1.props[prop_name], prop_type): - return False - return True - return False diff --git a/monasca_analytics/component/__init__.py b/monasca_analytics/component/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/component/base.py b/monasca_analytics/component/base.py deleted file mode 100644 index 2a4ad91..0000000 --- a/monasca_analytics/component/base.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import six - - -class abstractstatic(staticmethod): - """ - See http://stackoverflow.com/a/4474495 for - more details on this class purpose - """ - __slots__ = () - - def __init__(self, function): - super(abstractstatic, self).__init__(function) - function.__isabstractmethod__ = True - __isabstractmethod__ = True - - -@six.add_metaclass(abc.ABCMeta) -class BaseComponent(object): - """Base class for all component types. - - Should be as lightweight as possible, becuase any data here will be sent - to all workers every time a component is added. - """ - - def __init__(self, _id, _config): - """BaseComponent constructor. - - :type _id: str - :param _id: identifier of this Source - :type _config: dict - :param _config: configuration of this Source - """ - self.validate_config(_config) - self._id = _id - - @abstractstatic - def validate_config(_config): # @NoSelf - """Abstract static method for configuration validation. - - To be implemented by BaseComponent children. - It is called at creation time, and it should validate the - schema of the configuration passed as parameter - (i.e. check for expected keys and value format). - This function should raise exceptions if the validation fails, - otherwise it is assumed the validation was successful. - - :type _config: dict - :param _config: configuration of this module to be validated. - """ - pass - - @abstractstatic - def get_default_config(): # @NoSelf - """Abstract static method that returns the default configuration. - - To be implemented by BaseComponent children. It has to return a default - valid configuration for the module. - - :rtype: dict - :returns: default configuration - """ - pass - - @abstractstatic - def get_params(): # @NoSelf - """Abstract static method that returns the description of the params. - - To be implemented by BaseComponent children. It has to return - a list of the params description such as: - - return [ - ParamDescriptor('param1', type_util.String(), 'default value'), - ParamDescriptor('param2', type_util.Object({ - 'a': type_util.Number() - ), {'a': 123}), - ... - ] - - This function must be kept in sync with `get_default_config` and - `validate_config`, otherwise banana scripts using this component - will get runtime errors when being evaluated. - - The order in the list maps to the order the parameter must be - passed when the component would be created, in banana: - - a = MyComponent(param1, param2) - - `param1` and `param2` would be type-checked respectively against - the first and the second element of the returned list. - - :rtype: list[monasca_analytics.component.params.ParamDescriptor] - :return: Returns the list of parameters accepted by this component. - """ - pass - - def id(self): - return self._id - - def __hash__(self): - return hash(self._id) - - def __eq__(self, other): - return self._id == other.id() - - def __ne__(self, other): - return not(self == other) - - def __str__(self): - return self._id diff --git a/monasca_analytics/component/params.py b/monasca_analytics/component/params.py deleted file mode 100644 index 3c7a820..0000000 --- a/monasca_analytics/component/params.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monasca_analytics.banana.typeck.type_util as u - - -class ParamDescriptor(object): - """ - Description of a component parameter. This object contains - information such as the name of the parameter, the type, - the default value and a validator that will be evaluated - when the component is instantiated. - """ - def __init__(self, name, _type, default=None, validator=None): - """ - Construct a parameter descriptor. - :type name: str - :param name: The name of the parameter - :type _type: u.String | u.Number | u.Object | u.Enum | u.Any - :param _type: The type of the parameter - :type default: str | float | int | dict - :param default: The default value for the parameter. - :param validator: Additional validator for the parameter. - """ - if not isinstance(_type, u.String) and\ - not isinstance(_type, u.Number) and\ - not isinstance(_type, u.Object) and\ - not isinstance(_type, u.Enum) and\ - not isinstance(_type, u.Any): - raise Exception("ParamDescriptor incorrectly defined") - self.param_name = name - self.default_value = default - self.param_type = _type - self.validator = validator - - def to_json(self): - return { - "name": self.param_name, - "default_value": self.default_value, - "type": self.param_type.to_json(), - } diff --git a/monasca_analytics/config/__init__.py b/monasca_analytics/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/config/config.py b/monasca_analytics/config/config.py deleted file mode 100644 index 4d3d144..0000000 --- a/monasca_analytics/config/config.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import sys - -import monasca_analytics.config.connection as connection -import monasca_analytics.config.creation as creation -import monasca_analytics.config.validation as validation -import monasca_analytics.source.base as msource - -logger = logging.getLogger(__name__) - - -def instantiate_components(_config): - """Instantiate the components from the configuration. - - :type _config: dict - :param _config: configuration of the module to instantiate - :rtype: dict - :returns: the list of components - """ - try: - validation.validate_config(_config) - logger.debug("Creating components according to configuration...") - components = creation.create_components(_config) - logger.debug("Done creating components. Creating link data...") - links = connection.connect_components(components, _config) - logger.debug("Done connecting components. Validating links...") - validation.validate_links(links) - logger.debug("Done validating links. Successful instantiation") - return links - except Exception as ex: - logger.error("Failed to instantiate components") - logger.error("Reason : " + str(ex)) - sys.exit(-1) - - -def collect_sources(links): - """Collect the sources from the links and return them in a list - - :type links: dict - :returns: List of sources - :rtype: list[msource.BaseSource] - """ - sources = [] - for key in links.keys(): - if isinstance(key, msource.BaseSource): - sources.append(key) - return sources diff --git a/monasca_analytics/config/connection.py b/monasca_analytics/config/connection.py deleted file mode 100644 index 9f8ed17..0000000 --- a/monasca_analytics/config/connection.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import monasca_analytics.config.const as const -import monasca_analytics.exception.monanas as err - - -logger = logging.getLogger(__name__) - - -def connect_components(components, _config): - """Connects the components using the information from the configuration - - :type components: dict - :param components: components to connect - :type _config: dict - :param _config: configuration containing the - connections definitions - :rtype: dict - :returns: Dictionary where keys are IDs of sources of connections, - and values are their destinations IDs. - """ - return _perform_all_connections(const.CONNECTIONS, _config, components) - - -def _perform_all_connections(connection_kind, _config, components): - """Connect all component that are of the given kind. - - :type connection_kind: str - :param connection_kind: Kind of connection (feedback or flow) - :type _config: dict - :param _config: configuration containing the connections - :type components: dict - :param components: components to connect. - :rtype: dict - :returns: Dictionay where keys are IDs of sources of connections, - and values are their destinations IDs. - """ - links = {} - for origin_id in _config[connection_kind].keys(): - for comp_type in const.components_types: - if origin_id in components[comp_type]: - component = components[comp_type][origin_id] - connections_list = _config[connection_kind][origin_id] - _perform_connections( - component, connections_list, - components, links) - return links - - -def _perform_connections(origin, connections_list, components, links): - """Connect 'origin' with all destinations in connections_list - - :type origin: str - :param origin: origin component - :type connections_list: list[str] - :param connections_list: destinations IDs to be connected - :type components: dict - :param components: all components - :type links: dict - :param links: links that we are going to represent - """ - if origin not in links: - links[origin] = [] - - if not connections_list: - logger.debug("No connections for {}".format(origin)) - return - - for dest_id in connections_list: - if dest_id in components[const.INGESTORS].keys(): - ingestor = components[const.INGESTORS][dest_id] - links[origin].append(ingestor) - continue - if dest_id in components[const.SMLS].keys(): - sml = components[const.SMLS][dest_id] - links[origin].append(sml) - continue - if dest_id in components[const.VOTERS].keys(): - voter = components[const.VOTERS][dest_id] - links[origin].append(voter) - continue - if dest_id in components[const.SINKS].keys(): - sink = components[const.SINKS][dest_id] - links[origin].append(sink) - continue - if dest_id in components[const.LDPS].keys(): - ldp = components[const.LDPS][dest_id] - links[origin].append(ldp) - continue - raise err.MonanasWrongConnectoinError( - "wrong component defined in connection : " + dest_id) diff --git a/monasca_analytics/config/const.py b/monasca_analytics/config/const.py deleted file mode 100644 index 62a1533..0000000 --- a/monasca_analytics/config/const.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy - -SOURCES = "sources" -INGESTORS = "ingestors" -SMLS = "smls" -VOTERS = "voters" -SINKS = "sinks" -LDPS = "ldps" -CONNECTIONS = "connections" -FEEDBACK = "feedback" - -components_types = [SOURCES, INGESTORS, SMLS, VOTERS, SINKS, LDPS] - - -_default_base_config = { - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": False - }, - "sources": {}, - "ingestors": {}, - "smls": {}, - "voters": {}, - "sinks": {}, - "ldps": {}, - "connections": {}, - "feedback": {} -} - - -def get_default_base_config(): - return copy.deepcopy(_default_base_config) diff --git a/monasca_analytics/config/creation.py b/monasca_analytics/config/creation.py deleted file mode 100644 index 3325f57..0000000 --- a/monasca_analytics/config/creation.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -from monasca_analytics.config import const -from monasca_analytics.util import common_util - -import six - - -logger = logging.getLogger(__name__) - - -def create_components(_config): - """Creates the components defined by the configuration - - :type _config: dict - :param _config: configuration containing components - :rtype: dict - :returns: Created components indexed by type and ID - """ - components = {} - for component_type in const.components_types: - components[component_type] = \ - _create_comps_by_module(component_type, _config) - return components - - -def _create_comps_by_module(comp_type, _config): - """Instantiates all the components of a type defined in the configuration - - :type comp_type: str - :param comp_type: type of the components to be deployed - (e.g. source, ingestor, ...) - :type _config: dict - :param _config: Configuration containing components - :rtype: dict - :returns: deployed components, keyed by ID - :raises: MonanasNoSuchSourceError -- if no source class found. - """ - logger.debug("Creating components of type : " + comp_type) - ret = {} - for comp_id, comp_config in six.iteritems(_config[comp_type]): - comp = _create_component_by_module( - comp_id, comp_config, comp_type) - ret[comp_id] = comp - return ret - - -def _create_component_by_module(comp_id, comp_config, comp_type): - """Create a single component matching the past configuration. - - The id assigned to that component will be comp_id. - - :type comp_id: str - :param comp_id: ID of the component to create - :type comp_config: dict - :param comp_config: Configuration of the component to create - :type comp_type: str - :param comp_type: type of component to create - :rtype: monasca_analytics.component.base.BaseComponent - :returns: Instantiated component object - """ - logger.debug("deploying " + comp_config["module"] + " object") - clazz = common_util.get_class_by_name(comp_config["module"], - comp_type) - _comp = clazz(comp_id, comp_config) - return _comp diff --git a/monasca_analytics/config/validation.py b/monasca_analytics/config/validation.py deleted file mode 100644 index a97c136..0000000 --- a/monasca_analytics/config/validation.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A list of functions to validate config models.""" - -import logging - -import six -import voluptuous - -from monasca_analytics.config import const - - -logger = logging.getLogger(__name__) - - -valid_connection_types = { - const.SOURCES: [const.INGESTORS, const.LDPS], - const.INGESTORS: [const.SINKS], - const.SMLS: [const.VOTERS, const.SINKS], - const.VOTERS: [const.LDPS, const.SINKS], - const.LDPS: [const.SINKS], - const.SINKS: [] -} - - -valid_feedback_connection_types = { - const.SOURCES: [], - const.INGESTORS: [], - const.SMLS: [], - const.VOTERS: [], - const.LDPS: [], - const.SINKS: [const.VOTERS, const.SMLS] -} - - -def validate_config(config): - """Perform the whole validation: schema, uniqueness and connections - - :type config: dict - :param config: configuration to validate - :raises: SchemaError -- if the configuration is not valid for any reason - """ - _validate_schema(config) - _validate_only_one_voter(config) - _validate_ids_uniqueness(config) - _validate_connections(config) - - -def validate_links(links): - """Validate links to make sure, nothing is missing - - :type links: dict - :param links: connection links to validate - :raises: SchemaError -- if any link is missing - """ - missing = set([]) - all_keys = set(links.keys()) - for connections in links.values(): - for component in connections: - if component not in all_keys: - missing.add(component.id()) - if len(missing) > 0: - raise voluptuous.Invalid([ - "In connections section, the following components are not " - "connected\n\t{}\n" - "please modify the configuration so that their list of " - "connections is at least '[]'".format(", ".join(missing))], []) - - -def _validate_schema(config): - """Validate the configuration, with spark, up to the orchestration level - - Checks that hte spark configuration is valid, as well as the modules - structure in the configuration up to the orchestration level. - Each module will be responsible to validate its own sub-configuration. - - :type config: dict - :param config: configuration model for the whole system - :raises: SchemaError -- if the configuration, up to the - orchestration level, is not valid - """ - config_schema = voluptuous.Schema({ - "spark_config": { - "appName": six.string_types[0], - "streaming": { - "batch_interval": voluptuous.And(int, voluptuous.Range(min=1)) - } - }, - "server": { - "port": int, - "debug": bool - }, - "sources": { - voluptuous.Optional(six.string_types[0]): {six.string_types[0]: - object} - }, - "ingestors": { - voluptuous.Optional(six.string_types[0]): {six.string_types[0]: - object} - }, - "smls": { - voluptuous.Optional(six.string_types[0]): {six.string_types[0]: - object} - }, - "voters": { - voluptuous.Optional(six.string_types[0]): {six.string_types[0]: - object} - }, - "sinks": { - voluptuous.Optional(six.string_types[0]): {six.string_types[0]: - object} - }, - "ldps": { - voluptuous.Optional(six.string_types[0]): {six.string_types[0]: - object} - }, - "connections": { - voluptuous.Optional(six.string_types[0]): [six.string_types[0]] - }, - "feedback": { - voluptuous.Optional(six.string_types[0]): [six.string_types[0]] - } - }, required=True) - return config_schema(config) - - -def _validate_only_one_voter(config): - """Check that the configuration defines only a single voter - - :type config: dict - :param config: configuration model for the whole system - :raises: SchemaError -- if there is more than one voter defined in config - """ - def _raise(comp): - raise voluptuous.Invalid([ - "More than one {} found in the config, please modify " + - "it specifying only one {}".format(comp, comp)], []) - - if len(config["voters"]) > 1: - _raise("voter") - - -def _validate_ids_uniqueness(config): - """Validate that the IDs of the components are unique - - :type config: dict - :param config: configuration model for the whole system - :raises: SchemaError -- if there is any duplicated ID in the configuration - """ - all_ids = set() - for comp_type in valid_connection_types.keys(): - for com_id in config[comp_type].keys(): - if com_id in all_ids: - raise voluptuous.Invalid( - ["Duplicated component ID : " + com_id], []) - all_ids.add(com_id) - - -def _validate_expected_dest_type(config, from_id, to_ids, expected_types): - """Check that the connection is valid according to expected_types. - - :type config: dict - :param config: configuration model for the whole system - :type from_id: str - :param from_id: ID of the component which is the - origin point of the connection - :type to_ids: list - :param to_ids: IDs of the components which are the - destination points of the connections - :type expected_types: list - :param expected_types: types of components that are allowed - as destination points - """ - for to_id in to_ids: - logger.debug("validating connection " + from_id + " --> " + to_id) - valid_connection = False - for expected_type in expected_types: - if to_id in config[expected_type].keys(): - valid_connection = True - break - if not valid_connection: - raise voluptuous.Invalid([ - from_id + " connected to a wrong component: " + to_id + - ". It should be connected only to any of : " + - str(expected_types)], []) - - -def _validate_existing_id(config, component_id): - """Check that the id passed as parameter is defined in the configuration - - :type config: dict - :param config: configuration model for the whole system - :type component_id: str - :param component_id: component ID to be found in configuration - """ - found_id = False - for comp_type in valid_connection_types.keys(): - if component_id in config[comp_type].keys(): - found_id = True - if not found_id: - raise voluptuous.Invalid([ - 'In "connections", component `{}` hasn\'t been defined' - .format(component_id) - ], []) - - -def _validate_from_dictionary(config, conf_key, validation_dict): - """Validate connections in config[conf_key] according to validation_dict - - :type config: dict - :param config: configuration model for the whole system - :type conf_key: str - :param conf_key: key of the configuration dictionary where - the connections to be checked are defined - :type validation_dict: dict - :param validation_dict: keys are source types, and values - are lists of allowed destination types - for that particular source type - """ - for from_id in config[conf_key].keys(): - _validate_existing_id(config, from_id) - to_ids = config[conf_key][from_id] - for comp_type in validation_dict.keys(): - if from_id in config[comp_type].keys(): - _validate_expected_dest_type( - config, from_id, to_ids, validation_dict[comp_type]) - - -def _validate_connections(config): - """Validate that the connections defined in config are allowed - - :type config: dict - :param config: configuration model for the whole system - """ - _validate_from_dictionary(config, "connections", valid_connection_types) - _validate_from_dictionary(config, "feedback", - valid_feedback_connection_types) diff --git a/monasca_analytics/exception/__init__.py b/monasca_analytics/exception/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/exception/banana.py b/monasca_analytics/exception/banana.py deleted file mode 100644 index 96a54c9..0000000 --- a/monasca_analytics/exception/banana.py +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Banana Error classes.""" - -import abc -import pyparsing as p -import six - -import monasca_analytics.banana.grammar.base_ast as ast - - -@six.add_metaclass(abc.ABCMeta) -class BananaException(Exception): - - @abc.abstractmethod - def __str__(self): - pass - - @abc.abstractmethod - def get_span(self): - """ - :rtype: ast.Span - :return: Returns the span where the error occured if appropriate - """ - pass - - -class BananaInvalidExpression(BananaException): - def __init__(self, value): - self._value = value - - def __str__(self): - return repr(self._value) - - def get_span(self): - return ast.DUMMY_SPAN - - -class BananaEnvironmentError(BananaException): - def __init__(self, value): - self._value = value - - def __str__(self): - return repr(self._value) - - def get_span(self): - return ast.DUMMY_SPAN - - -class BananaNoFullPath(BananaException): - def __init__(self, missing): - self._value = "None of the paths can be executed. Missing at least" \ - " one {}.".format(missing) - - def __str__(self): - return self._value - - def get_span(self): - return ast.DUMMY_SPAN - - -class BananaArgumentTypeError(BananaException): - def __init__(self, where, expected_type, received_type): - if isinstance(where, ast.ASTNode): - self._span = where.span - else: - self._span = where - self._value = "Wrong type of argument. Expected '{}' got '{}'."\ - .format(expected_type, received_type) - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaComponentTooManyParams(BananaException): - def __init__(self, span): - self._span = span - self._value = "Too many params provided to '{}'.".format( - span, span.get_lineno() - ) - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaComponentMixingParams(BananaException): - def __init__(self, span, named_is_wrong): - self._span = span - if named_is_wrong: - self._value = "'{}' should be named as " \ - "previous parameters are.".format(span) - else: - self._value = "'{}' should not be named as " \ - "previous parameters are.".format(span) - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaComponentIncorrectParamName(BananaException): - def __init__(self, found, component): - if isinstance(component, ast.ASTNode): - component = component.span - if isinstance(found, ast.ASTNode): - self._span = found.span - found = found.span - else: - self._span = found - self._value = "Incorrect parameter name. Parameter '{}' " \ - "does not exists on component {}."\ - .format(found, component) - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaComponentAlreadyDefined(BananaException): - def __init__(self, first_def, second_def): - self._value = "Component already defined!\n" \ - " First definition: '{}'\n" \ - " Second definition: '{}'."\ - .format(first_def, second_def) - - def __str__(self): - return self._value - - def get_span(self): - # TODO(Joan): This could be a real span instead of this one. - return ast.DUMMY_SPAN - - -class BananaShadowingComponentError(BananaException): - def __init__(self, where, comp): - self._span = where - self._value = "Shadowing component '{}'. " \ - "Please use another variable name.".format(comp) - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaAssignmentError(BananaException): - def __init__(self, lhs, rhs): - self._value = "You can't assign '{}' to '{}'.".format(lhs, rhs) - - def __str__(self): - return self._value - - def get_span(self): - return ast.DUMMY_SPAN - - -class BananaGrammarBug(BananaException, p.ParseFatalException): - def __init__(self, error): - super(BananaGrammarBug, self).__init__(pstr=error) - self._value = "Bug found in the grammar!" \ - " Please report this error: {}".format(error) - - def __str__(self): - return self._value - - def get_span(self): - return ast.DUMMY_SPAN - - -class BananaJsonObjShadowingError(BananaException, p.ParseFatalException): - def __init__(self, span, error): - self._span = span - error = "Can't shadow property already defined in {}.".format(error) - super(BananaJsonObjShadowingError, self).__init__(pstr=error) - - def __str__(self): - return self.msg - - def get_span(self): - return self._span - - -class BananaTypeCheckerBug(BananaException): - def __init__(self, error): - self._value = "Bug found in the TypeChecker!" \ - " Please report this error: {}".format(error) - - def __str__(self): - return self._value - - def get_span(self): - return ast.DUMMY_SPAN - - -class BananaEvalBug(BananaException): - def __init__(self, error): - self._value = "Bug found in the evaluator!" \ - " Please report this error: {}".format(error) - - def __str__(self): - return self._value - - def get_span(self): - return ast.DUMMY_SPAN - - -class BananaUnknown(BananaException): - def __init__(self, ident): - self._span = ident.span - self._value = "Unknown '{}'.".format( - ident.into_unmodified_str() - ) - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaUnknownOperator(BananaException): - def __init__(self, span, operator, for_type): - self._span = span - self._value = "Unknown operator '{}' for type '{}'.".format( - operator, - for_type - ) - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaPropertyDoesNotExists(BananaException): - def __init__(self, dotpath, on_type=None): - self._span = dotpath.span - if on_type is None: - self._value = "Property '{}' " \ - "does not exists."\ - .format( - dotpath.into_unmodified_str() - ) - else: - self._value = "Property '{}' " \ - "does not exists on type '{}'."\ - .format( - dotpath.into_unmodified_str(), - str(on_type) - ) - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaTypeError(BananaException): - def __init__(self, expected_type, found_type=None, span=None): - self._span = span - if expected_type is None: - class DummyType(object): - def __str__(self): - return "_" - expected_type = DummyType - if found_type is None: - if isinstance(expected_type, list): - self._value = "Type error found. Expected" \ - " one among '{}'."\ - .format(', '.join(map(lambda x: str(x), expected_type))) - else: - self._value = "Type error found. Expected '{}'.".format( - str(expected_type) - ) - else: - if isinstance(expected_type, list): - self._value = "Type error found. Expected" \ - " one among '{}', found '{}'."\ - .format(', '.join(map(lambda x: str(x), expected_type)), - str(found_type)) - else: - self._value = "Type error found. Expected" \ - " '{}', found '{}'."\ - .format(str(expected_type), str(found_type)) - - def __str__(self): - return self._value - - def get_span(self): - if self._span is None: - return ast.DUMMY_SPAN - return self._span - - -class BananaAssignCompError(BananaException): - def __init__(self, span): - self._span = span - self._value = "Component objects " \ - "can't be assigned to " \ - "properties of other objects." - - def __str__(self): - return self._value - - def get_span(self): - return self._span - - -class BananaConnectionError(BananaException): - - def __init__(self, span, ident_from, ident_to, type_from, - possible_connections): - self._value = "Can't connect '{}'" \ - " to '{}'," \ - " '{}' can only be connected to a {}."\ - .format( - ident_from.val, - ident_to.val, - type_from.class_name, ' or a '.join(possible_connections)) - self._span = span - - def __str__(self): - return self._value - - def get_span(self): - return self._span diff --git a/monasca_analytics/exception/dsl.py b/monasca_analytics/exception/dsl.py deleted file mode 100644 index e2efc01..0000000 --- a/monasca_analytics/exception/dsl.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""DSL Error classes.""" - -import abc -import six - - -@six.add_metaclass(abc.ABCMeta) -class DSLException(Exception): - - @abc.abstractmethod - def __str__(self): - pass - - -class DSLExistingConnection(DSLException): - def __init__(self, value): - self._value = value - - def __str__(self): - DSLException.__str__(self) - return repr(self._value) - - -class DSLInexistentComponent(DSLException): - def __init__(self, value): - self._value = value - - def __str__(self): - DSLException.__str__(self) - return repr(self._value) - - -class DSLInvalidConnection(DSLException): - def __init__(self, value): - self._value = value - - def __str__(self): - DSLException.__str__(self) - return repr(self._value) - - -class DSLInterpreterException(DSLException): - def __init__(self, value): - self._value = value - - def __str__(self): - DSLException.__str__(self) - return repr(self._value) diff --git a/monasca_analytics/exception/ingestor.py b/monasca_analytics/exception/ingestor.py deleted file mode 100644 index ad79a6a..0000000 --- a/monasca_analytics/exception/ingestor.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Data Ingestor Error classes.""" - -import abc -import six - - -@six.add_metaclass(abc.ABCMeta) -class IngestorException(Exception): - - @abc.abstractmethod - def __str__(self): - pass - - -class IngestorNoSuchSourceModel(IngestorException): - def __init__(self, value): - self._value = value - - def __str__(self): - IngestorException.__str__(self) - return repr(self._value) diff --git a/monasca_analytics/exception/monanas.py b/monasca_analytics/exception/monanas.py deleted file mode 100644 index 55b088b..0000000 --- a/monasca_analytics/exception/monanas.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Monanas Error classes.""" - -import abc -import six - - -@six.add_metaclass(abc.ABCMeta) -class MonanasException(Exception): - - @abc.abstractmethod - def __str__(self): - pass - - -class MonanasMainError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasInitError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasBindSourcesError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasAlreadyStartedStreaming(MonanasException): - def __init__(self): - self._value = "Monanas has already started streaming." - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasAlreadyStoppedStreaming(MonanasException): - def __init__(self): - self._value = "Monanas has already stopped streaming." - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasStreamingError(MonanasException): - def __init__(self): - self._value = "Unable to start streaming data. Make sure there is at \ - least one operation registered." - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasNoSuchClassError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasDuplicateClassError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasNoSuchSourceError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasDuplicateSourceError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasNoSuchIngestorError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasDuplicateIngestorError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasNoSuchAggregatorError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasDuplicateAggregatorError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasNoSuchSMLError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasDuplicateSMLError(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasDStreamNotReady(MonanasException): - def __init__(self): - self._value = "The dstream requested is not ready." - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasAlreadyIngested(MonanasException): - def __init__(self): - self._value = "Monanas has already ingested data." - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasNoSuchSourceModel(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasInvalidConfig(MonanasException): - def __init__(self, value): - self._value = value - - def __str__(self): - MonanasException.__str__(self) - return repr(self._value) - - -class MonanasCyclicRandomSourceError(Exception): - - def __init__(self): - self._value = "Monanas couldn't start the linearly dependent \ - random source: Cycle found." - - def __str__(self): - return repr(self._value) - - -class MonanasWrongConnectoinError(Exception): - - def __init__(self, value): - self._value = value - - def __str__(self): - return repr(self._value) diff --git a/monasca_analytics/ingestor/__init__.py b/monasca_analytics/ingestor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/ingestor/base.py b/monasca_analytics/ingestor/base.py deleted file mode 100644 index 94086fa..0000000 --- a/monasca_analytics/ingestor/base.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import logging - -from monasca_analytics.component import base - -logger = logging.getLogger(__name__) - - -class BaseIngestor(base.BaseComponent): - """Base class for all the Ingestor modules""" - - def __init__(self, _id, _config): - """Constructor with ID and configuration - - :type _id: str - :param _id: ID assigned to this component - :type _config: dict - :param _config: configuration of this component - """ - self._features = None - super(BaseIngestor, self).__init__(_id, _config) - - @abc.abstractmethod - def map_dstream(self, dstream): - """Transforms the data provided by a dstream to another dstream - - Abstract method to be implemented by BaseIngestor children. - The processed dstream should be returned. - - :type dstream: pyspark.streaming.DStream - :param dstream: stream of data before being processed - :rtype: pyspark.streaming.DStream - :returns: stream of data after being processed - """ - pass - - def set_feature_list(self, features): - """Set the list of features - - :type features: list[str] - :param features: List of features names (when extracted from the data) - """ - self._features = features diff --git a/monasca_analytics/ingestor/cloud.py b/monasca_analytics/ingestor/cloud.py deleted file mode 100644 index 8df6b1a..0000000 --- a/monasca_analytics/ingestor/cloud.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -import voluptuous - -from monasca_analytics.ingestor import base -import monasca_analytics.util.spark_func as fn -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - - -class CloudIngestor(base.BaseIngestor): - """Data ingestor for Cloud""" - - def __init__(self, _id, _config): - super(CloudIngestor, self).__init__(_id=_id, _config=_config) - - @staticmethod - def validate_config(_config): - cloud_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()) - }, required=True) - return cloud_schema(_config) - - @staticmethod - def get_params(): - return [] - - def map_dstream(self, dstream): - features_list = list(self._features) - return dstream.map(fn.from_json)\ - .map(lambda x: (x['ctime'], x['event']))\ - .groupByKey()\ - .map(lambda rdd_entry: CloudIngestor._parse_and_vectorize( - rdd_entry[1], - features_list)) - - @staticmethod - def get_default_config(): - return {"module": CloudIngestor.__name__} - - @staticmethod - def _parse_and_vectorize(iterable, feature_list): - values = { - "support_1": 0.0 - } - for feature in feature_list: - values[feature] = 0.0 - for e in iterable: - if e["id"] in values: - values[e["id"]] += 1.0 - res = [values[f] for f in feature_list] - return np.array(res) diff --git a/monasca_analytics/ingestor/iptables.py b/monasca_analytics/ingestor/iptables.py deleted file mode 100644 index a2eefca..0000000 --- a/monasca_analytics/ingestor/iptables.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -import voluptuous - -from monasca_analytics.ingestor import base -from monasca_analytics.source import iptables_markov_chain as src -import monasca_analytics.util.spark_func as fn -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - -RDD_EVENT = "event" -RDD_CTIME = "ctime" -EVENT_MSG = "msg" - - -class IptablesIngestor(base.BaseIngestor): - """This ingestor class implements an IPTable parsing and vectorization. - - Assuming the dstream contains iptables that have been triggered, it creates - an np array for each sample wit the number of times that each IPTable has - been triggered. - """ - - def __init__(self, _id, _config): - super(IptablesIngestor, self).__init__(_id=_id, _config=_config) - - @staticmethod - def validate_config(_config): - iptables_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()) - }, required=True) - return iptables_schema(_config) - - @staticmethod - def get_default_config(): - return {"module": IptablesIngestor.__name__} - - @staticmethod - def get_params(): - return [] - - def map_dstream(self, dstream): - """ - Map the provided dstream into another dstream. - - :type dstream: pyspark.streaming.DStream - :param dstream: DStream to transform. - :rtype: pyspark.streaming.DStream - :return: Returns a new dstream. - """ - features_list = list(self._features) - return dstream.map(fn.from_json)\ - .map(lambda x: (x['ctime'], x))\ - .groupByKey()\ - .map(lambda rdd_entry: IptablesIngestor._process_data( - rdd_entry[1], - features_list)) - - @staticmethod - def _process_data(rdd_entry, feature_list): - """Parse and vectorize the rdd_entry - - Assuming the rdd_entry is encoded in JSON format, this method - gets the events and vectorizes them according to the features. - - :type rdd_entry: list[dict] - :param rdd_entry: event - :type feature_list: list[str] - :param feature_list: features to extract, in order - """ - events = [] - for event in rdd_entry: - events.append(event[RDD_EVENT]) - return IptablesIngestor.vectorize_events(events, feature_list) - - @staticmethod - def vectorize_events(events, feature_list): - """Event vectorizing logic. - - For each event, we get the message, - which is the IPTable that has been triggered. - Then we get the corresponding feature for the IPtable. - Finally, we increase the index of the vector corresponding to - that feature. - - :type events: list[dict] - :param events: List of collected events. - :type feature_list: list[str] - :param feature_list: features to extract, in order - """ - ret = np.zeros(len(feature_list)) - for event in events: - iptable_id = src.iptables[event[EVENT_MSG]] - feature = src.get_iptable_type(iptable_id) - index = feature_list.index(feature) - ret[index] += 1 - return ret diff --git a/monasca_analytics/ldp/__init__.py b/monasca_analytics/ldp/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/ldp/base.py b/monasca_analytics/ldp/base.py deleted file mode 100644 index f4b5e27..0000000 --- a/monasca_analytics/ldp/base.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import six - -from monasca_analytics.component import base - - -@six.add_metaclass(abc.ABCMeta) -class BaseLDP(base.BaseComponent): - """Base class for Live Data Processor (LDP), to be extended""" - - def __init__(self, _id, _config): - """Constructor with ID and configuration - - :type _id: str - :param _id: ID assigned to this component - :type _config: dict - :param _config: configuration of this component - """ - super(BaseLDP, self).__init__(_id, _config) - self._features = None - self._data = { - "features": None, - "matrix": None - } - - @abc.abstractmethod - def map_dstream(self, dstream): - """Map the dstream input to new values - - :type dstream: pyspark.streaming.DStream - :param dstream: DStream created by the source. - """ - pass - - def set_voter_output(self, matrix): - """Assign the features and matrix to the _data object that - - :type matrix: numpy.ndarray - :param matrix: the causality matrix learned by the voter. - """ - self._data["features"] = self._features - self._data["matrix"] = matrix - - def set_feature_list(self, features): - """Set the list of features - - :type features: list[str] - :param features: list of the features names. - """ - self._features = features diff --git a/monasca_analytics/ldp/cloud_causality.py b/monasca_analytics/ldp/cloud_causality.py deleted file mode 100644 index 4d928f5..0000000 --- a/monasca_analytics/ldp/cloud_causality.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import six -import voluptuous - -import monasca_analytics.ldp.base as bt -import monasca_analytics.util.spark_func as fn -import monasca_analytics.util.validation_utils as vu - - -logger = logging.getLogger(__name__) - - -class CloudCausalityLDP(bt.BaseLDP): - """A causality live data processor""" - - @staticmethod - def validate_config(_config): - cloud_causality_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()) - }, required=True) - return cloud_causality_schema(_config) - - @staticmethod - def get_default_config(): - return {"module": CloudCausalityLDP.__name__} - - @staticmethod - def get_params(): - return [] - - def map_dstream(self, dstream): - """Executes _aggregate for each RDD in the dstream - - :type dstream: pyspark.streaming.DStream - :param dstream: DStream created by the source. - """ - data = self._data - return dstream.map(fn.from_json)\ - .map(lambda x: (x['ctime'], x))\ - .groupByKey()\ - .flatMap(lambda r: CloudCausalityLDP._aggregate(r[1], data)) - - @staticmethod - def _aggregate(rdd_entry, data): - new_entries = [] - features = data["features"] - matrix = data["matrix"] - - if features is None or matrix is None: - return rdd_entry - - for event in rdd_entry: - causes = [] - - try: - cause = features.index(event["event"]["id"]) - for other_event in rdd_entry: - if other_event["event"]["id"] != event["event"]["id"]: - try: - caused = features.index(other_event["event"]["id"]) - - if matrix[caused][cause]: - causes.append(other_event["event"]["id"]) - except ValueError: - pass - except ValueError: - pass - - event["__monanas__"] = dict(causes=causes) - new_entries.append(event) - - return new_entries diff --git a/monasca_analytics/ldp/iptables_ldp.py b/monasca_analytics/ldp/iptables_ldp.py deleted file mode 100644 index a3e00da..0000000 --- a/monasca_analytics/ldp/iptables_ldp.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import six -import voluptuous - -import monasca_analytics.ingestor.iptables as ip_ing -import monasca_analytics.ldp.base as bt -from monasca_analytics.sml import svm_one_class -import monasca_analytics.util.spark_func as fn -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - -FEATURES = "features" -MATRIX = "matrix" - - -class IptablesLDP(bt.BaseLDP): - """An anomaly detection life module""" - - @staticmethod - def validate_config(_config): - iptables_ldp_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()) - }, required=True) - return iptables_ldp_schema(_config) - - @staticmethod - def get_default_config(): - return {"module": IptablesLDP.__name__} - - @staticmethod - def get_params(): - return [] - - def map_dstream(self, dstream): - """Detect anomalies in a dstream using the learned classifier - - :type dstream: pyspark.streaming.DStream - :param dstream: Spark's DStream - """ - data = self._data - return dstream.map(fn.from_json)\ - .map(lambda x: (x['ctime'], x))\ - .groupByKey()\ - .flatMap(lambda r: IptablesLDP._detect_anomalies(r[1], data)) - - @staticmethod - def _detect_anomalies(rdd_entry, data): - """Classifies and marks the RDD entry as anomalous or non-anomalous - - :type rdd_entry: list[dict] - :param rdd_entry: entry to be classified - :type data: dict - :param data: contains the features and the classifier - """ - new_entries = [] - events = [] - ctimes = [] - for event in rdd_entry: - events.append(event[ip_ing.RDD_EVENT]) - ctimes.append(event["ctime"]) - features = data[FEATURES] - classifier = data[MATRIX] - - if features is None or classifier is None: - return events - - X = ip_ing.IptablesIngestor.vectorize_events(events, features) - Y = classifier.predict(X) - for i in range(len(events)): - event = events[i] - event["ctime"] = ctimes[i] - if Y[0] == svm_one_class.ANOMALY: - event["anomalous"] = True - else: - event["anomalous"] = False - new_entries.append(event) - return new_entries diff --git a/monasca_analytics/ldp/monasca/__init__.py b/monasca_analytics/ldp/monasca/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/ldp/monasca/base.py b/monasca_analytics/ldp/monasca/base.py deleted file mode 100644 index 92b7a62..0000000 --- a/monasca_analytics/ldp/monasca/base.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.ldp import base - - -class MonascaBaseLDP(base.BaseLDP): - """Base class for monasca ldp, To be extended by concrete monasca LDPs.""" - pass diff --git a/monasca_analytics/ldp/monasca/helpers.py b/monasca_analytics/ldp/monasca/helpers.py deleted file mode 100644 index a5e6346..0000000 --- a/monasca_analytics/ldp/monasca/helpers.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import bisect -import time - - -def interpolate(timestamp, metric_values, metric_timestamps): - """ - :type timestamp: int - :param timestamp: Timestamp at witch we want to interpolate or extrapolate - the metric value - :type metric_values: list[dict] - :param metric_values: List of metrics - :type metric_timestamps: list[int] - :param metric_timestamps: List of timestamp for the given metric. - :rtype: float - :return: Returns the interpolated value - """ - insertion_pos = bisect.bisect_left(metric_timestamps, timestamp) - # Edge cases: - if insertion_pos == 0: - return metric_values[0]["metric"]["value"] - if insertion_pos == len(metric_timestamps) - 1: - return metric_values[len(metric_timestamps) - 1]["metric"]["value"] - if metric_timestamps[insertion_pos] == timestamp: - return metric_values[insertion_pos]["metric"]["value"] - # General case: - lo = metric_timestamps[insertion_pos - 1] - hi = metric_timestamps[insertion_pos] - dt = hi - lo - return metric_values[insertion_pos - 1]["metric"]["value"] *\ - (timestamp - lo) / dt + \ - metric_values[insertion_pos]["metric"]["value"] *\ - (hi - timestamp) / dt - - -def create_agg_metric(metric_name, meta, dimensions, timestamp, value): - return { - "metric": { - "name": metric_name, - "dimensions": dimensions, - "timestamp": timestamp, - "value": value - }, - "meta": meta, - "creation_time": int(time.time()) - } diff --git a/monasca_analytics/ldp/monasca_aggregate.py b/monasca_analytics/ldp/monasca_aggregate.py deleted file mode 100644 index fa08fd6..0000000 --- a/monasca_analytics/ldp/monasca_aggregate.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -import six - -import monasca_analytics.ldp.base as bt -import monasca_analytics.ldp.monasca.helpers as helpers -import monasca_analytics.util.spark_func as fn -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - - -class MonascaAggregateLDP(bt.BaseLDP): - """Monasca aggregator live data processor""" - - def __init__(self, _id, _config): - super(MonascaAggregateLDP, self).__init__(_id, _config) - self._aggregation_period = _config["period"] - self._reducer_func = MonascaAggregateLDP.select_reducer(_config) - self._suffix = "_" + _config["func"] - - @staticmethod - def validate_config(_config): - monasca_ag_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "period": voluptuous.Or(float, int), - "func": voluptuous.Or( - "avg", - "max", - "sum", - "min", - "cnt" - ) - }, required=True) - return monasca_ag_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": MonascaAggregateLDP.__name__, - "period": 60.0 * 60.0, - "func": "avg" - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('period', type_util.Number(), 60 * 60), - params.ParamDescriptor( - 'func', - type_util.Enum(['avg', 'max', 'sum', 'min', 'cnt']), - 'avg' - ) - ] - - def map_dstream(self, dstream): - """ - Map the given DStream into a new DStream where metrics - have been aggregated by name. - - :type dstream: pyspark.streaming.DStream - :param dstream: DStream - :return: Returns the stream of aggregated metrics - """ - red = self._reducer_func - suf = self._suffix - agg_period = self._aggregation_period - # TODO(Joan): Add a filter to only aggregate some metrics - # TODO(Joan): or particular dimensions - return dstream.map(fn.from_json) \ - .window(agg_period, agg_period) \ - .map(lambda metric: (metric["metric"]["name"], metric)) \ - .groupByKey() \ - .flatMapValues(lambda metrics: MonascaAggregateLDP.aggregate( - metrics, - red, - suf - ))\ - .map(lambda metric_and_name: metric_and_name[1]) - - @staticmethod - def aggregate(all_metrics, reducer, suffix): - """ - Aggregate values produced by different providers together. - The metric name is assumed to be the same for all providers. - - :type all_metrics: list[dict] - :param all_metrics: Values to aggregate mapping to a specific - metric name. - :type reducer: ((float, float) -> float, - (float, float, float) -> float) - :param reducer: Combine the metrics values together - :type suffix: str - :param suffix: Suffix to append to the metric name in its combined form - """ - # Collect metric separately - separated_metrics = {} # type: dict[frozenset, list[dict]] - for el in all_metrics: - key = frozenset(el["metric"]["dimensions"].items()) - if key not in separated_metrics: - separated_metrics[key] = [el] - else: - separated_metrics[key].append(el) - - # Collect all dimensions - dims = {} - for metric_dims in separated_metrics.keys(): - for prop, val in six.iteritems(dict(metric_dims)): - if prop in dims: - dims[prop].add(val) - else: - dims[prop] = set(val) - - # Sort each metric - for _, metric in six.iteritems(separated_metrics): - metric.sort(key=lambda v: v["metric"]["timestamp"]) - - separated_metrics = sorted(list(separated_metrics.values()), key=len) - separated_metrics.reverse() - - # Compute the new values - new_values = [] - all_timestamps = [[x["metric"]["timestamp"] for x in l] - for l in separated_metrics] - metric_count = len(separated_metrics) - for index in range(0, len(separated_metrics[0])): - new_value = reducer[0]( - separated_metrics[0][index]["metric"]["value"], - metric_count) - new_timestamp = separated_metrics[0][index]["metric"]["timestamp"] - for metric_index in range(1, metric_count): - new_value = reducer[1](new_value, helpers.interpolate( - new_timestamp, - separated_metrics[metric_index], - all_timestamps[metric_index] - ), metric_count) - new_values.append((new_timestamp, new_value)) - - # Aggregate the other details: - metric_name = separated_metrics[0][0]["metric"]["name"] + suffix - meta = separated_metrics[0][0]["meta"] - new_metrics = [ - helpers.create_agg_metric( - metric_name, - meta, - dims, - val[0], - val[1] - ) for val in new_values - ] - return new_metrics - - @staticmethod - def select_reducer(_config): - return { - "avg": ( - lambda m, cnt: m / cnt, - lambda acc, m, cnt: m / cnt + acc, - ), - "max": ( - lambda m, cnt: m, - lambda acc, m, cnt: max(m, acc), - ), - "sum": ( - lambda m, cnt: m, - lambda acc, m, cnt: m + acc, - ), - "min": ( - lambda m, cnt: m, - lambda acc, m, cnt: min(m, acc), - ), - "cnt": ( - lambda m, cnt: m, - lambda acc, m, cnt: cnt, - ), - }[_config["func"]] diff --git a/monasca_analytics/ldp/monasca_combine.py b/monasca_analytics/ldp/monasca_combine.py deleted file mode 100644 index bf8463d..0000000 --- a/monasca_analytics/ldp/monasca_combine.py +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import math -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -import six - -import monasca_analytics.ldp.base as bt -import monasca_analytics.ldp.monasca.helpers as helpers -import monasca_analytics.parsing.api as parsing -import monasca_analytics.util.spark_func as fn -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - - -class MonascaCombineLDP(bt.BaseLDP): - """Monasca combiner live data processor""" - - def __init__(self, _id, _config): - super(MonascaCombineLDP, self).__init__(_id, _config) - logger.debug(_config["bindings"]) - logger.debug(_config["lambda"]) - self._combine_function = parsing.create_fn_with_config( - env=_config["bindings"], - expr_string=_config["lambda"] - ) - self._combine_period = _config["period"] - self._combine_metric_name = _config["metric"] - self._metrics_of_interest = _config["bindings"].values() - - def map_dstream(self, dstream): - """ - Map the given DStream into a new DStream where the specified metrics - have been combined together. - - :type dstream: pyspark.streaming.DStream - :param dstream: DStream - :return: Returns the stream of combined metrics - """ - combine_fn = self._combine_function - metric_names = self._metrics_of_interest - nb_metrics = len(metric_names) - metric_name = self._combine_metric_name - return dstream.map(fn.from_json)\ - .window(self._combine_period, self._combine_period)\ - .filter(lambda x: x["metric"]["name"] in metric_names and - x["metric"]["value"] is not None) \ - .map(lambda x: (frozenset(x["metric"]["dimensions"]), x))\ - .groupByKey()\ - .flatMapValues(lambda metrics: MonascaCombineLDP.combine( - metrics, - combine_fn, - metric_name, - nb_metrics - ))\ - .map(lambda x: x[1]) - - @staticmethod - def combine(all_metrics, combine_fn, combine_metric_name, nb_of_metrics): - """ - Combine the given metrics of this RDD into one. - - :type all_metrics: pyspark.resultiterable.ResultIterable - :param all_metrics: List containing the metrics. - :param combine_fn: Combiner. - :type combine_metric_name: str - :param combine_metric_name: Name of the new metric - :type nb_of_metrics: int - :param nb_of_metrics: The number of metrics expected - """ - # Separate metrics based on name - separated_metrics = {} # type: dict[str, list[dict]] - dims = None - for el in all_metrics: - key = el["metric"]["name"] - if dims is None: - dims = el["metric"]["dimensions"] - if key not in separated_metrics: - separated_metrics[key] = [el] - else: - separated_metrics[key].append(el) - - if len(separated_metrics.keys()) != nb_of_metrics: - return [] - - separated_metrics = sorted(list(six.iteritems(separated_metrics)), - key=lambda x: len(x[1])) - separated_metrics = separated_metrics # type: list[(str, list[dict])] - - # Sort each metric - for metric in separated_metrics: - metric[1].sort(key=lambda v: v["metric"]["timestamp"]) - - temp_values = [] - all_timestamp = [[x["metric"]["timestamp"] for x in l[1]] - for l in separated_metrics] - for index in range(0, len(separated_metrics[0][1])): - current_env = { - separated_metrics[0][0]: - separated_metrics[0][1][index]["metric"]["value"] - } - timestamp = all_timestamp[0][index] - for metric_index in range(1, len(separated_metrics)): - metric_prop = separated_metrics[metric_index] - metric_name = metric_prop[0] - current_env[metric_name] = helpers.interpolate( - timestamp, - metric_prop[1], - all_timestamp[metric_index] - ) - temp_values.append(current_env) - - new_values = map(combine_fn, temp_values) - - new_metrics = [ - helpers.create_agg_metric( - combine_metric_name, - {}, - dims, - tsmp, - val - ) for val, tsmp in zip(new_values, all_timestamp[0]) - ] - return new_metrics - - @staticmethod - def validate_config(_config): - monasca_comb_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "metric": six.string_types[0], - "period": voluptuous.And( - voluptuous.Or(float, int), - lambda i: i >= 0 and math.floor(i) == math.ceil(i)), - "lambda": six.string_types[0], - "bindings": { - six.string_types[0]: voluptuous.Or( - "apache.net.kbytes_sec", - "apache.net.requests_sec", - "apache.performance.cpu_load_perc", - "cpu.idle_perc", - "cpu.stolen_perc", - "cpu.system_perc", - "cpu.total_logical_cores", - "cpu.user_perc", - "cpu.wait_perc", - "disk.allocation", - "disk.inode_used_perc", - "disk.space_used_perc", - "disk.total_space_mb", - "disk.total_used_space_mb", - "host_alive_status", - "io.read_kbytes_sec", - "io.read_req_sec", - "io.write_time_sec", - "kafka.consumer_lag", - "load.avg_1_min", - "load.avg_5_min", - "mem.free_mb", - "mem.swap_free_mb", - "mem.swap_total_mb", - "mem.total_mb", - "mem.usable_mb", - "mem.used_cache", - "metrics-added-to-batch-counter[0]", - "mysql.innodb.buffer_pool_free", - "mysql.innodb.buffer_pool_used", - "mysql.innodb.data_reads", - "mysql.innodb.mutex_spin_rounds", - "mysql.performance.com_delete_multi", - "mysql.performance.com_insert", - "mysql.performance.com_insert_select", - "mysql.performance.com_select", - "mysql.performance.com_update", - "mysql.performance.created_tmp_disk_tables", - "mysql.performance.created_tmp_files", - "mysql.performance.open_files", - "mysql.performance.questions", - "mysql.performance.user_time", - "net.in_bytes_sec", - "net.in_errors_sec", - "net.in_packets_dropped_sec", - "net.in_packets_sec", - "net.out_bytes_sec", - "net.out_errors_sec", - "net.out_packets_dropped_sec", - "net.out_packets_sec", - "nova.vm.disk.total_allocated_gb", - "process.pid_count", - "raw-sql.time.max", - "vcpus", - "vm.cpu.utilization_perc", - "vm.host_alive_status", - "vm.mem.total_mb", - "zookeeper.out_bytes", - "zookeeper.outstanding_bytes" - ) - } - }, required=True) - monasca_comb_schema(_config) - # Checks the expression and the environment - handle = parsing.validate_expression(_config["lambda"]) - parsing.validate_name_binding(handle, - _config["bindings"]) - - @staticmethod - def get_default_config(): - return { - "module": MonascaCombineLDP.__name__, - "metric": "cpu.logical_cores_actives", - "period": 1, - "lambda": "a * b", - "bindings": { - "a": "cpu.idle_perc", - "b": "cpu.total_logical_cores" - } - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor( - 'metric', - type_util.String(), - 'cpu.logcal_cores_actives' - ), - params.ParamDescriptor( - 'period', - type_util.Number(), - 1 - ), - params.ParamDescriptor( - 'lambda', - type_util.String(), - 'a * b' - ), - params.ParamDescriptor( - 'bindings', - type_util.Any(), - {'a': 'cpu.ilde_perc', 'b': 'cpu.total_logical_cores'} - ) - ] diff --git a/monasca_analytics/ldp/monasca_derivative.py b/monasca_analytics/ldp/monasca_derivative.py deleted file mode 100644 index c82cb76..0000000 --- a/monasca_analytics/ldp/monasca_derivative.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import math -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -import six - -import monasca_analytics.ldp.base as bt -import monasca_analytics.ldp.monasca.helpers as helpers -import monasca_analytics.util.spark_func as fn -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - - -# FIXME: This code is inaccurate because values on "edge" of the RDD -# FIXME: have a computed derivative with less precision than others. -# FIXME: The base idea would be to use the sliding window capability -# FIXME: to compute the derivative with the unbiased variant for all -# FIXME: values. However, we need a way to "know" how many derivative -# FIXME: calculation values needs to be skipped from one window to the -# FIXME: other. -# FIXME: -class MonascaDerivativeLDP(bt.BaseLDP): - """Monasca derivative live data processor""" - - def __init__(self, _id, _config): - super(MonascaDerivativeLDP, self).__init__(_id, _config) - self._period = _config["period"] - - @staticmethod - def validate_config(_config): - monasca_der_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - # Derivative period in multiple of batch interval - "period": voluptuous.And( - voluptuous.Or(float, int), - lambda i: i >= 0 and math.floor(i) == math.ceil(i)) - }, required=True) - return monasca_der_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": MonascaDerivativeLDP.__name__, - "period": 1 - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('period', type_util.Number(), 1) - ] - - def map_dstream(self, dstream): - """ - Map the given DStream into a new DStream where metrics - are replaced with their derivative. - - :type dstream: pyspark.streaming.DStream - :param dstream: DStream - :return: Returns the stream of derivative. - """ - period = self._period - return dstream.map(fn.from_json) \ - .window(period, period) \ - .map(lambda m: ((frozenset( - list(m["metric"]["dimensions"].items())), - m["metric"]["name"]), - m)) \ - .groupByKey() \ - .flatMapValues(lambda metric: MonascaDerivativeLDP.derivative( - metric, - )) \ - .map(lambda x: x[1]) - - @staticmethod - def derivative(metric_values): - """ - Compute the derivative of the given function. - - :type metric_values: pyspark.resultiterable.ResultIterable[dict] - :param metric_values: The list of metric_values - :return: Returns the derivative of the provided metric. - """ - if len(metric_values.data) < 2: - return [] - - metric_name = metric_values.data[0]["metric"]["name"] + "_derivative" - meta = metric_values.data[0]["meta"] - dims = metric_values.data[0]["metric"]["dimensions"] - # All values - timestamps = [m["metric"]["timestamp"] for m in metric_values] - all_values = [m["metric"]["value"] for m in metric_values] - # Sort values - all_values = [y for (_, y) in - sorted(zip(timestamps, all_values), key=lambda x: x[0])] - timestamps = sorted(timestamps) - # Remove duplicates - last_timestamp = timestamps[0] - tmp_all_values = [all_values[0]] - tmp_timestamps = [last_timestamp] - for index in range(1, len(timestamps)): - if timestamps[index] == last_timestamp: - continue - else: - last_timestamp = timestamps[index] - tmp_all_values.append(all_values[index]) - tmp_timestamps.append(last_timestamp) - all_values = tmp_all_values - timestamps = tmp_timestamps - - if len(all_values) < 2: - return [] - - # Filter all values that have the same timestamp - n = len(all_values) - 1 - new_values = [ - float(all_values[1] - all_values[0]) / - float(timestamps[1] - timestamps[0]) - ] - for index in range(1, n): - new_values.append( - float(all_values[index + 1] - all_values[index - 1]) / - float(timestamps[index + 1] - timestamps[index - 1]) - ) - new_values.append( - float(all_values[n] - all_values[n - 1]) / - float(timestamps[n] - timestamps[n - 1]) - ) - new_metrics = [ - helpers.create_agg_metric( - metric_name, - meta, - dims, - tmst, - val - ) for val, tmst in zip(new_values, timestamps) - ] - return new_metrics diff --git a/monasca_analytics/monanas.py b/monasca_analytics/monanas.py deleted file mode 100644 index 2e6b1e4..0000000 --- a/monasca_analytics/monanas.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os -import sys - -from tornado import ioloop -import voluptuous - -import monasca_analytics.banana.emitter as emit -import monasca_analytics.banana.pass_manager as executor -import monasca_analytics.exception.monanas as err -import monasca_analytics.spark.driver as driver -import monasca_analytics.util.common_util as cu -import monasca_analytics.web_service.web_service as ws - -logger = logging.getLogger(__name__) - -debugging = False - - -class Monanas(object): - """Monanas. - - Monanas entry point. This class implements the high level functions to - manage the streaming. - - Currently: start and stop. - """ - - def __init__(self, _config): - """Monanas constructor. - - It validates the configuration passed as parameter and, - if it is fine, it creates the components defined by it. - - :type _config: dict - :param _config: configuration of the system. - """ - self._is_streaming = False - self._driver = driver.DriverExecutor(_config) - logger.info("Monanas initialized.") - - def is_streaming(self): - """Gets the status of streaming. - - :rtype: bool - :returns: `True` if Monanas is streaming,`False` otherwise. - """ - return self._is_streaming - - def try_change_configuration(self, banana_str, emitter): - """Try to change the configuration to the provided one. - - :type banana_str: str - :param banana_str: New configuration. - :type emitter: emit.JsonEmitter - :param emitter: a Json emitter instance - """ - if not isinstance(emitter, emit.JsonEmitter): - raise err.MonanasException() - # Try to change the configuration. - executor.execute_banana_string(banana_str, self._driver, emitter) - - def typeck_configuration(self, banana_str, emitter): - """Only type check the provided configuration. - - :type banana_str: str - :param banana_str: New configuration. - :type emitter: emit.JsonEmitter - :param emitter: a Json emitter instance - """ - executor.execute_banana_string(banana_str, None, emitter) - - def compute_type_table(self, banana_str): - """Compute the type table for the provided configuration. - - :type banana_str: str - :param banana_str: Configuration to test. - :rtype: dict - :return: Returns the type table - """ - type_table = executor.try_compute_type_table(banana_str) - if type_table is not None: - return type_table.to_json() - return {} - - def start_streaming(self): - """Starts streaming data. - - :raises: MonanasStreamingError -- if one or more sources fail to bind - or StreamingContext fails to start. - :raises: MonanasAlreadyStartedStreaming -- if Monanas is already - streaming. - """ - if not self._is_streaming: - try: - self._driver.start_pipeline() - self._is_streaming = True - except Exception: - raise err.MonanasStreamingError() - else: - raise err.MonanasAlreadyStartedStreaming() - - def stop_streaming(self): - """Stops streaming data. - - :raises: MonanasAlreadyStoppedStreaming -- if Monanas is not streaming - """ - if self._is_streaming: - self._driver.stop_pipeline() - self._is_streaming = False - else: - raise err.MonanasAlreadyStoppedStreaming() - - def stop_streaming_and_terminate(self): - """Stops streaming data and terminate Monanas.""" - try: - self.stop_streaming() - logger.info("All streaming stopped.") - except (err.MonanasAlreadyStoppedStreaming, NameError): - logger.info("All streaming stopped.") - logger.info("Monanas stopped.") - os.kill(os.getpid(), 1) - - -if __name__ == "__main__": - monanas = None - if debugging: - import pydevd - pydevd.settrace(suspend=False) - try: - try: - cu.setup_logging(sys.argv[2]) - except IOError: - raise err.MonanasMainError("File not found: \ - `{0}`.".format(sys.argv[2])) - except ValueError: - raise err.MonanasMainError("`{0}` is not a valid logging \ - config file.".format(sys.argv[2])) - - logger = logging.getLogger(__name__) - - try: - config = cu.parse_json_file(sys.argv[1]) - except IOError: - raise err.MonanasMainError("File not found: \ - `{0}`.".format(sys.argv[1])) - except ValueError as e: - raise err.MonanasMainError("`{0}` is not a \ - valid json file.".format(sys.argv[1])) - except voluptuous.Invalid: - raise err.MonanasMainError("`{0}` has an \ - invalid schema.".format(sys.argv[1])) - - monanas = Monanas(config) - web_service = ws.WebService(monanas, config["server"]) - web_service.listen(config["server"]["port"]) - ioloop.IOLoop.instance().start() - except IOError as e: - logger.error("Address already in use.") - except (err.MonanasInitError, err.MonanasMainError) as e: - logger.error(e.__str__()) - except KeyboardInterrupt: - try: - if monanas is not None: - monanas.stop_streaming_and_terminate() - except NameError: - logger.info("Premature termination.") - except Exception as e: - logger.error("Unexpected error: {0}. {1}.". - format(sys.exc_info()[0], e)) - - try: - if monanas is not None: - monanas.stop_streaming_and_terminate() - except NameError: - logger.info("Premature termination.") - - logger.info("Monanas stopped.") diff --git a/monasca_analytics/parsing/__init__.py b/monasca_analytics/parsing/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/parsing/api.py b/monasca_analytics/parsing/api.py deleted file mode 100644 index a0caee3..0000000 --- a/monasca_analytics/parsing/api.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.parsing.expression import create_fn_with_config -from monasca_analytics.parsing.expression import validate_environment -from monasca_analytics.parsing.expression import validate_expression -from monasca_analytics.parsing.expression import validate_name_binding - -create_fn_with_config = create_fn_with_config -validate_expression = validate_expression -validate_environment = validate_environment -validate_name_binding = validate_name_binding diff --git a/monasca_analytics/parsing/expression.py b/monasca_analytics/parsing/expression.py deleted file mode 100644 index 8b96e6e..0000000 --- a/monasca_analytics/parsing/expression.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import pyparsing as p -import types - -import monasca_analytics.banana.bytecode.assembler as asbl -import monasca_analytics.exception.banana as exception -import monasca_analytics.parsing.private as priv - -import six - - -logger = logging.getLogger(__name__) - - -class ExpressionParser(object): - - def __init__(self): - """ - Create a parser that parse arithmetic expressions. They can - contains variable identifiers or raw numbers. The meaning - for the identifiers is left to the - """ - number = p.Regex(r'\d+(\.\d*)?([eE]\d+)?') - identifier = p.Word(p.alphas) - terminal = identifier | number - self._expr = p.infixNotation(terminal, [ - (p.oneOf('* /'), 2, p.opAssoc.LEFT), - (p.oneOf('+ -'), 2, p.opAssoc.LEFT) - ]) + p.stringEnd() - - def parse(self, string, code=asbl.Code()): - """ - Parse a given string and construct an Evaluator - :type string: six.string_types - :param string: String to parse. - :type code: ass.Code - :param code: Generated code will be written here. - :return: Returns an evaluator that will returns a value - given the appropriate environment to resolve - variables. - """ - tree = self._expr.parseString(string)[0] - self._build_tree(tree, code) - - def parse_tree(self, expr): - """ - Parse the given string and return the generated tree - by pyparsing. - :type expr: str - :param expr: Expression to parse. - :return: Returns the generated tree. - """ - return self._expr.parseString(expr) - - @staticmethod - def _build_tree(subtree, code): - """ - :type subtree: list - :param subtree: Sub tree to parse - :type code: ass.Code - :param code: Generated code is written here. - """ - current_operator = None - pushed_one_stack_value = False - for child in subtree: - if isinstance(child, six.string_types): - if priv.is_op(child): - current_operator = child - else: - code(asbl.Local(child)) - if not pushed_one_stack_value: - pushed_one_stack_value = True - else: - ExpressionParser._push_op(current_operator, code) - else: - ExpressionParser._build_tree(child, code) - if not pushed_one_stack_value: - pushed_one_stack_value = True - else: - ExpressionParser._push_op(current_operator, code) - - @staticmethod - def _push_op(operator, code): - if operator is None: - raise exception.BananaInvalidExpression( - "Bug found! please fill a bug report on ??" - ) - if operator == '+': - code.BINARY_ADD() - elif operator == '-': - code.BINARY_SUBTRACT() - elif operator == '/': - code.BINARY_DIVIDE() - elif operator == '*': - code.BINARY_MULTIPLY() - - -def create_fn_with_config(env, expr_string): - """ - Create an evaluator given the expected - environment renaming and expression. - :type env: dict[str, str] - :param env: Environment to use. - :type expr_string: str - :param expr_string: String containing the expression - to be evaluated - :returns: Returns a function that accept one argument - expected to be the environment. - """ - code = asbl.Code() - # Argument - code(asbl.Local('__monanas__env')) - code.co_argcount = 1 - # Create local variables - for key, value in six.iteritems(env): - code(asbl.Call( - asbl.Getattr( - asbl.Local('__monanas__env'), 'get'), - [asbl.Const(value)]), - asbl.LocalAssign(str(key))) - parser = ExpressionParser() - try: - parser.parse(expr_string, code) - except p.ParseException as e: - raise exception.BananaInvalidExpression(e.message) - code.RETURN_VALUE() - final_fn = types.FunctionType(code.code(), globals()) - return final_fn - - -def validate_environment(env): - """ - Validate the given arguments that create_fn_with_config - is expecting. - :param env: Environment spec - """ - for key, val in six.iteritems(env): - if not isinstance(key, six.string_types): - raise exception.BananaEnvironmentError( - "{} is not a valid key (only string are)".format(key) - ) - if not isinstance(val, six.string_types): - raise exception.BananaEnvironmentError( - "{} is not a valid value (only string are)".format(val) - ) - - -def validate_expression(expr_string): - """ - Validate the provided expression string. - :type expr_string: str - :param expr_string: Expression string to validate. - :returns: Returns a handle that can be use to validate - name usage against an environment. - :raises: exception.BananaInvalidExpression - """ - if not isinstance(expr_string, six.string_types): - raise exception.BananaArgumentTypeError( - expected_type=six.string_types[0], - received_type=type(expr_string) - ) - parser = ExpressionParser() - try: - res = parser.parse_tree(expr_string) - return ExpressionHandle(res, expr_string) - except p.ParseException as e: - raise exception.BananaInvalidExpression(str(e)) - - -def validate_name_binding(expr_handle, environment): - """ - Validate the name binding in the expr_handle for - the provided environment. - :type expr_handle: ExpressionHandle - :param expr_handle: The expression handle - :type environment: dict - :param environment: The environment - """ - if not isinstance(expr_handle, ExpressionHandle): - raise exception.BananaArgumentTypeError( - expected_type=ExpressionHandle, - received_type=type(expr_handle) - ) - - def collect_names(subtree): - """ - Collect names used in this subtree - :type subtree: list - :param subtree: subtree - """ - for child in subtree: - if isinstance(child, six.string_types): - if priv.is_not_op(child): - names.add(child) - else: - collect_names(child) - names = set() - collect_names(expr_handle.tree) - for name in names: - if name not in list(environment.keys()): - raise exception.BananaInvalidExpression( - "The expression '{}' can't be used with the provided " - "environment: '{}'. Reason: '{}' is not defined.".format( - expr_handle.original_str, - environment, - name - ) - ) - - -class ExpressionHandle(object): - def __init__(self, tree, original_string): - self.tree = tree - self.original_str = original_string diff --git a/monasca_analytics/parsing/private.py b/monasca_analytics/parsing/private.py deleted file mode 100644 index c77f822..0000000 --- a/monasca_analytics/parsing/private.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -EXPRESSION_OPERATOR_LIST = ['+', '-', '*', '/'] - - -def is_op(x): - return x in EXPRESSION_OPERATOR_LIST - - -def is_not_op(x): - return x not in EXPRESSION_OPERATOR_LIST diff --git a/monasca_analytics/sink/__init__.py b/monasca_analytics/sink/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/sink/base.py b/monasca_analytics/sink/base.py deleted file mode 100644 index 20fc4a5..0000000 --- a/monasca_analytics/sink/base.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import six - -from monasca_analytics.component import base - - -@six.add_metaclass(abc.ABCMeta) -class BaseSink(base.BaseComponent): - """Base class for dstream sink to be extended by concrete dstream sinks.""" - - @abc.abstractmethod - def sink_dstream(self, dstream): - pass - - @abc.abstractmethod - def sink_ml(self, voter_id, matrix): - pass diff --git a/monasca_analytics/sink/base_sqlite.py b/monasca_analytics/sink/base_sqlite.py deleted file mode 100644 index aeeca8f..0000000 --- a/monasca_analytics/sink/base_sqlite.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import logging -import six -from six.moves import cPickle -import sqlite3 -import time - - -from monasca_analytics.sink import base - - -DB_NAME_DEFAULT = "sqlite_sink.db" -logger = logging.getLogger(__name__) - - -@six.add_metaclass(abc.ABCMeta) -class BaseSQLiteSink(base.BaseSink): - """Base class for SQLite sink to be extended by concrete implementations""" - - def __init__(self, _id, _config): - super(BaseSQLiteSink, self).__init__(_id, _config) - self.db_name = self._get_db_name(_config) - with sqlite3.connect(self.db_name) as conn: - c = conn.cursor() - c.execute(self._rdds_table_create_query()) - c.execute('''CREATE TABLE IF NOT EXISTS smls - (timestamp INTEGER, voter_id TEXT, sml BLOB)''') - conn.commit() - - def _get_db_name(self, _config): - if "db_name" in _config: - return _config["db_name"] - return DB_NAME_DEFAULT - - def sink_dstream(self, dstream): - dstream.foreachRDD(self._persist) - - def _persist(self, _, rdd): - rdd_entries = rdd.collect() - with sqlite3.connect(self.db_name) as conn: - c = conn.cursor() - for rdd_entry in rdd_entries: - query = self._rdd_insert_query(rdd_entry) - c.execute(query) - conn.commit() - - def sink_ml(self, voter_id, matrix): - with sqlite3.connect(self.db_name) as conn: - c = conn.cursor() - timestamp = time.time() - blob_matrix = cPickle.dumps(matrix, cPickle.HIGHEST_PROTOCOL) - c.execute( - 'INSERT INTO smls (timestamp, voter_id, sml) VALUES(?, ?, ?);', - [timestamp, voter_id, sqlite3.Binary(blob_matrix)]) - conn.commit() - - @abc.abstractmethod - def _rdds_table_create_query(self): - pass - - @abc.abstractmethod - def _rdd_insert_query(self, rdd_json): - pass diff --git a/monasca_analytics/sink/file.py b/monasca_analytics/sink/file.py deleted file mode 100644 index 3554fb1..0000000 --- a/monasca_analytics/sink/file.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import os.path as path -import tempfile -import time -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -import monasca_analytics.sink.base as base -from monasca_analytics.util import validation_utils as vu - - -import six - - -class FileSink(base.BaseSink): - """Sink that prints the dstream to a file in the driver - - This sink is for development **only**. - """ - - def __init__(self, _id, _config): - super(FileSink, self).__init__(_id, _config) - if _config["path"] is not None: - _path = path.expanduser(_config["path"]) - if path.isdir(_path): - _path = path.join(_path, time.time() + '.log') - self._file_path = _path - else: - self._file_path = tempfile.NamedTemporaryFile().name - - def sink_dstream(self, dstream): - """ - Sink the provided DStream into a file. - - :type dstream: pyspark.streaming.DStream - :param dstream: DStream to sink - """ - _file_name = self._file_path - - def write_output(rdd): - _file = open(_file_name, 'a+') - for rdd_entry in rdd.collect(): - _file.write(json.dumps(rdd_entry, indent=4)) - - dstream.foreachRDD(lambda _, rdd: write_output(rdd)) - - def sink_ml(self, voter_id, matrix): - pass - - @staticmethod - def get_default_config(): - return { - "module": FileSink.__name__, - "path": None - } - - @staticmethod - def get_params(): - return [params.ParamDescriptor('path', type_util.String())] - - @staticmethod - def validate_config(_config): - file_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "path": voluptuous.Or( - voluptuous.And(six.string_types[0], vu.ExistingPath()), - None) - }, required=True) - return file_schema(_config) diff --git a/monasca_analytics/sink/iptables_sqlite.py b/monasca_analytics/sink/iptables_sqlite.py deleted file mode 100644 index 31e7390..0000000 --- a/monasca_analytics/sink/iptables_sqlite.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -import monasca_analytics.sink.base_sqlite as base -from monasca_analytics.util import validation_utils as vu - - -import six - - -class IptablesSQLiteSink(base.BaseSQLiteSink): - """IPTables SQLite Sink implementation.""" - - def _rdds_table_create_query(self): - return """CREATE TABLE IF NOT EXISTS rdds - (msg TEXT, anomalous TEXT, msg_id TEXT, ctime TEXT)""" - - def _rdd_insert_query(self, rdd_json): - return ('INSERT INTO rdds VALUES("' + str(rdd_json["msg"]) + '", "' + - str(rdd_json["anomalous"]) + '", "' + str(rdd_json["id"]) + - '", "' + str(rdd_json["ctime"]) + '")') - - @staticmethod - def get_default_config(): - return { - "module": IptablesSQLiteSink.__name__, - "db_name": "sqlite_sink.db" - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('db_name', - type_util.String(), - 'sqlite_sink.db') - ] - - @staticmethod - def validate_config(_config): - iptables_sql_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - voluptuous.Optional("db_name"): voluptuous.And( - six.string_types[0], vu.NoSpaceCharacter()), - }, required=True) - return iptables_sql_schema(_config) diff --git a/monasca_analytics/sink/kafkas.py b/monasca_analytics/sink/kafkas.py deleted file mode 100644 index 4d31f95..0000000 --- a/monasca_analytics/sink/kafkas.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import time - -import kafka - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -from monasca_analytics.sink import base -import monasca_analytics.sink.sink_config_validator as validator - - -class KafkaSink(base.BaseSink): - """A Kafka sink for dstream""" - - def __init__(self, _id, _config): - self._topic = None - self._producer = None - super(KafkaSink, self).__init__(_id, _config) - self._host = _config["host"] - self._port = int(_config["port"]) - self._topic = _config["topic"] - - def sink_dstream(self, dstream): - if self._producer is None: - self._producer = kafka.KafkaProducer( - bootstrap_servers="{0}:{1}".format(self._host, self._port)) - dstream.foreachRDD(self._persist) - - def _persist(self, _, rdd): - rdd_entries = rdd.collect() - - for rdd_entry in rdd_entries: - self._producer.send(self._topic, - json.dumps(rdd_entry)) - self._producer.flush() - - def sink_ml(self, voter_id, matrix): - output = dict( - id=voter_id, - matrix=matrix, - timestamp=time.time() - ) - self._producer.send(self._topic, - json.dumps(output)) - - @staticmethod - def validate_config(_config): - validator.validate_kafka_sink_config(_config) - - @staticmethod - def get_default_config(): - return { - "module": KafkaSink.__name__, - "host": "localhost", - "port": 9092, - "topic": "transformed_alerts" - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('host', type_util.String(), 'localhost'), - params.ParamDescriptor('port', type_util.Number(), 9092), - params.ParamDescriptor('topic', type_util.String(), - 'transformed_alerts') - ] diff --git a/monasca_analytics/sink/sink_config_validator.py b/monasca_analytics/sink/sink_config_validator.py deleted file mode 100644 index 08a45b1..0000000 --- a/monasca_analytics/sink/sink_config_validator.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A list of functions for validating sink configs.""" - -import math -import voluptuous - -from monasca_analytics.util import validation_utils as vu - - -import six - - -def validate_kafka_sink_config(config): - """Validates the KafkaSink configuration""" - - config_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], vu.AvailableSink()), - "host": voluptuous.And( - six.string_types[0], vu.NoSpaceCharacter()), - "port": voluptuous.And( - voluptuous.Or(float, int), - lambda i: i >= 0 and math.floor(i) == math.ceil(i)), - "topic": voluptuous.And( - six.string_types[0], vu.NoSpaceCharacter()) - }, required=True) - return config_schema(config) diff --git a/monasca_analytics/sink/stdout_sink.py b/monasca_analytics/sink/stdout_sink.py deleted file mode 100644 index a66a8c4..0000000 --- a/monasca_analytics/sink/stdout_sink.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import voluptuous - -from monasca_analytics.sink import base -from monasca_analytics.util import validation_utils as vu - - -import six - - -class StdoutSink(base.BaseSink): - """Sink that prints the dstream to stdout, using pprint command""" - - def sink_dstream(self, dstream): - dstream.pprint(1000) - - def sink_ml(self, voter_id, matrix): - pass - - @staticmethod - def get_default_config(): - return {"module": StdoutSink.__name__} - - @staticmethod - def validate_config(_config): - stdout_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()) - }, required=True) - stdout_schema(_config) - - @staticmethod - def get_params(): - return [] diff --git a/monasca_analytics/sml/__init__.py b/monasca_analytics/sml/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/sml/base.py b/monasca_analytics/sml/base.py deleted file mode 100644 index d7f23ca..0000000 --- a/monasca_analytics/sml/base.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc - -from monasca_analytics.component import base - - -class BaseSML(base.BaseComponent): - """Base SML, to be extended by statistical or machine learning classes""" - - def __init__(self, _id, _config): - super(BaseSML, self).__init__(_id, _config) - self._voter = None - - @abc.abstractmethod - def learn_structure(self, samples): - """Learn structure based on the provided data - - Abstract method to be implemented by subclasses. - It should learn over those samples a structure such as - a causality matrix or a decision tree that is then - going to be suggested to a voter which then - forward it to a LDP. - - :type samples: numpy.ndarray - :param samples: the list of samples to be processed - :rtype: numpy.ndarray - :returns: The information learned with the training set. - """ - pass - - @abc.abstractmethod - def number_of_samples_required(self): - """This function returns the number of samples required - - Algorithm typically requires a number of samples before - being able to produce a result. You can specify this constraint - with this function. - - :rtype: int - :returns: the number of samples required. - """ - pass - - def learn(self, samples): - """Learning phase - - This method is called by the aggregator owning this sml. - - :type samples: numpy.ndarray - :param samples: the list of samples that can be - processed by the data. - """ - self._voter.suggest_structure(self, self.learn_structure(samples)) - - def set_voter(self, voter): - if self._voter is not None: - self._voter.remove_sml(self) - self._voter = voter - self._voter.append_sml(self) diff --git a/monasca_analytics/sml/decision_tree.py b/monasca_analytics/sml/decision_tree.py deleted file mode 100644 index b7b98d8..0000000 --- a/monasca_analytics/sml/decision_tree.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -from sklearn.metrics import classification_report -from sklearn import tree -import voluptuous - -from monasca_analytics.sml.base import BaseSML -from monasca_analytics.util.validation_utils import NoSpaceCharacter - -logger = logging.getLogger(__name__) - -ANOMALY = 1 -NON_ANOMALY = 0 -N_SAMPLES = 1000 - - -class DecisionTreeClassifier(BaseSML): - """Anomaly detection based on the DecisionTreeClassifier algorithm""" - - def __init__(self, _id, _config): - super(DecisionTreeClassifier, self).__init__(_id, _config) - self._nb_samples = int(_config['nb_samples']) - - @staticmethod - def validate_config(_config): - decisiontree_schema = voluptuous.Schema({ - 'module': voluptuous.And(six.string_types[0], - NoSpaceCharacter()), - 'nb_samples': voluptuous.Or(float, int) - }, required=True) - return decisiontree_schema(_config) - - @staticmethod - def get_default_config(): - return { - 'module': DecisionTreeClassifier.__name__, - 'nb_samples': N_SAMPLES - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('nb_samples', type_util.Number(), N_SAMPLES) - ] - - def number_of_samples_required(self): - return self._nb_samples - - def _generate_train_test_sets(self, samples, ratio_train): - num_samples_train = int(len(samples) * ratio_train) - - data, labels = np.hsplit(samples, [-1]) - X_train = np.array(data[:num_samples_train]) - _labels = np.array(labels[:num_samples_train]) - X_train_label = _labels.ravel() - X_test = np.array(data[num_samples_train:]) - _labels = np.array(labels[num_samples_train:]) - X_test_label = _labels.ravel() - return X_train, X_train_label, X_test, X_test_label - - def _get_best_detector(self, train, label): - detector = tree.DecisionTreeClassifier() - detector.fit(train, label) - return detector - - def learn_structure(self, samples): - X_train, X_train_label, X_test, X_test_label = \ - self._generate_train_test_sets(samples, 0.75) - logger.info('Training with ' + str(len(X_train)) + - 'samples; testing with ' + str(len(X_test)) + ' samples.') - - dt_detector = self._get_best_detector(X_train, X_train_label) - Y_test = dt_detector.predict(X_test) - - num_anomalies = Y_test[Y_test == ANOMALY].size - logger.info('Found ' + str(num_anomalies) + - ' anomalies in testing set') - - logger.info('Confusion Matrix: \n{}'. - format(classification_report( - X_test_label, - Y_test, - target_names=['no', 'yes']))) - return dt_detector diff --git a/monasca_analytics/sml/elliptic_envelope.py b/monasca_analytics/sml/elliptic_envelope.py deleted file mode 100644 index 69cc2cb..0000000 --- a/monasca_analytics/sml/elliptic_envelope.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -from sklearn import covariance -import voluptuous - -from monasca_analytics.sml.base import BaseSML -from monasca_analytics.util.validation_utils import NoSpaceCharacter - -logger = logging.getLogger(__name__) - -ANOMALY = -1 -NON_ANOMALY = 1 -N_SAMPLES = 1000 - - -class EllipticEnvelope(BaseSML): - """Anomaly detection based on the EllipticEnvelope algorithm""" - - def __init__(self, _id, _config): - super(EllipticEnvelope, self).__init__(_id, _config) - self._nb_samples = int(_config['nb_samples']) - - @staticmethod - def validate_config(_config): - elliptic_schema = voluptuous.Schema({ - 'module': voluptuous.And(six.string_types[0], - NoSpaceCharacter()), - 'nb_samples': voluptuous.Or(float, int) - }, required=True) - return elliptic_schema(_config) - - @staticmethod - def get_default_config(): - return { - 'module': EllipticEnvelope.__name__, - 'nb_samples': N_SAMPLES - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('nb_samples', type_util.Number(), N_SAMPLES) - ] - - def number_of_samples_required(self): - return self._nb_samples - - def _generate_train_test_sets(self, samples, ratio_train): - num_samples_train = int(len(samples) * ratio_train) - X_train = np.array(samples[:num_samples_train]) - X_test = np.array(samples[num_samples_train:]) - return X_train, X_test - - def _get_best_detector(self, train): - detector = covariance.EllipticEnvelope() - detector.fit(train) - return detector - - def learn_structure(self, samples): - X_train, X_test = self._generate_train_test_sets(samples, 0.75) - logger.info('Training with ' + str(len(X_train)) + - 'samples; testing with ' + str(len(X_test)) + ' samples.') - - ee_detector = self._get_best_detector(X_train) - Y_test = ee_detector.predict(X_test) - - num_anomalies = Y_test[Y_test == ANOMALY].size - logger.info('Found ' + str(num_anomalies) + - ' anomalies in testing set') - return ee_detector diff --git a/monasca_analytics/sml/isolation_forest.py b/monasca_analytics/sml/isolation_forest.py deleted file mode 100644 index 4510f3b..0000000 --- a/monasca_analytics/sml/isolation_forest.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -from sklearn import ensemble -import voluptuous - -from monasca_analytics.sml.base import BaseSML -from monasca_analytics.util.validation_utils import NoSpaceCharacter - -logger = logging.getLogger(__name__) - -ANOMALY = -1 -NON_ANOMALY = 1 -N_SAMPLES = 1000 - - -class IsolationForest(BaseSML): - """Anomaly detection based on the IsolationForest algorithm""" - - def __init__(self, _id, _config): - super(IsolationForest, self).__init__(_id, _config) - self._nb_samples = int(_config['nb_samples']) - - @staticmethod - def validate_config(_config): - isolation_schema = voluptuous.Schema({ - 'module': voluptuous.And(six.string_types[0], - NoSpaceCharacter()), - 'nb_samples': voluptuous.Or(float, int) - }, required=True) - return isolation_schema(_config) - - @staticmethod - def get_default_config(): - return { - 'module': IsolationForest.__name__, - 'nb_samples': N_SAMPLES - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('nb_samples', type_util.Number(), N_SAMPLES) - ] - - def number_of_samples_required(self): - return self._nb_samples - - def _generate_train_test_sets(self, samples, ratio_train): - num_samples_train = int(len(samples) * ratio_train) - X_train = np.array(samples[:num_samples_train]) - X_test = np.array(samples[num_samples_train:]) - return X_train, X_test - - def _get_best_detector(self, train): - detector = ensemble.IsolationForest() - detector.fit(train) - return detector - - def learn_structure(self, samples): - X_train, X_test = self._generate_train_test_sets(samples, 0.75) - logger.info('Training with ' + str(len(X_train)) + - 'samples; testing with ' + str(len(X_test)) + ' samples.') - - if_detector = self._get_best_detector(X_train) - Y_test = if_detector.predict(X_test) - - num_anomalies = Y_test[Y_test == ANOMALY].size - logger.info('Found ' + str(num_anomalies) + - ' anomalies in testing set') - return if_detector diff --git a/monasca_analytics/sml/lingam.py b/monasca_analytics/sml/lingam.py deleted file mode 100644 index 09eef7e..0000000 --- a/monasca_analytics/sml/lingam.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import math - -import numpy as np -import six -from sklearn import decomposition -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params - -from monasca_analytics.sml import base -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - - -class LiNGAM(base.BaseSML): - """Causality discovery assuming a linear non gaussian acyclic data model""" - - def __init__(self, _id, _config): - super(LiNGAM, self).__init__(_id, _config) - self._threshold = _config["threshold"] - self._threshold = 0.1 - - @staticmethod - def validate_config(_config): - lingam_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "threshold": float - }, required=True) - return lingam_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": LiNGAM.__name__, - "threshold": 0.1 - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('threshold', type_util.Number(), 0.1) - ] - - def number_of_samples_required(self): - return 5000 - - def learn_structure(self, samples): - logger.debug(samples.shape) - logger.debug(samples) - threshold = self._threshold - causality_matrix, _ = LiNGAM._discover_structure(samples) - structure = causality_matrix > threshold - - logger.info("Causality Matrix: {0}".format(causality_matrix)) - logger.info("Learned causality: {0}".format(structure)) - return structure - - @staticmethod - def _discover_structure(data): - - # Add a random noise uniformly distributed to avoid singularity - # when performing the ICA - data += np.random.random_sample(data.shape) - - # Create the ICA node to get the inverse of the mixing matrix - k, w, _ = decomposition.fastica(data) - - w = np.dot(w, k) - n = w.shape[0] - best_nzd = float("inf") - best_slt = float("inf") - best_w_permuted = w - causality_matrix = None - causal_perm = None - - if n < 9: - perm = LiNGAM._perms(n) - - for i in range(perm.shape[1]): - perm_matrix = np.eye(n) - perm_matrix = perm_matrix[:, perm[:, i]] - w_permuted = perm_matrix.dot(w) - cost = LiNGAM._cost_non_zero_diag(w_permuted) - if cost < best_nzd: - best_nzd = cost - best_w_permuted = w_permuted - - w_opt = best_w_permuted - - w_opt = w_opt / np.diag(w_opt).reshape((n, 1)) - b_matrix = np.eye(n) - w_opt - best_b_permuted = b_matrix - best_i = 0 - - for i in range(perm.shape[1]): - b_permuted = b_matrix[:, perm[:, i]][perm[:, i], :] - cost = LiNGAM._cost_strictly_lower_triangular( - b_permuted) - if cost < best_slt: - best_slt = cost - best_i = i - best_b_permuted = b_permuted - - causal_perm = perm[:, best_i] - causality_matrix = b_matrix - - percent_upper = best_slt / np.sum(best_b_permuted ** 2) - - if percent_upper > 0.2: - # TODO(David): Change that code to raise an exception instead - logger.error("LiNGAM failed to run on the data set") - logger.error( - "--> B permuted matrix is at best {}% lower triangular" - .format(percent_upper)) - - return causality_matrix, causal_perm - - @staticmethod - def _cost_strictly_lower_triangular(b): - return np.sum((np.tril(b, -1) - b) ** 2) - - @staticmethod - def _cost_non_zero_diag(w): - return np.sum(1 / np.abs(np.diag(w))) - - @staticmethod - def _perms(n): - k = 1 - p = np.empty((2 * n - 1, math.factorial(n)), np.uint8) - for i in range(n): - p[i, :k] = i - p[i + 1:2 * i + 1, :k] = p[:i, :k] - for j in range(i): - p[:i + 1, k * (j + 1):k * (j + 2)] = p[j + 1:j + i + 2, :k] - k *= i + 1 - return p[:n, :] diff --git a/monasca_analytics/sml/logistic_regression.py b/monasca_analytics/sml/logistic_regression.py deleted file mode 100644 index 30e5d1c..0000000 --- a/monasca_analytics/sml/logistic_regression.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -from sklearn import linear_model -from sklearn.metrics import classification_report -import voluptuous - -from monasca_analytics.sml.base import BaseSML -from monasca_analytics.util.validation_utils import NoSpaceCharacter - -logger = logging.getLogger(__name__) - -ANOMALY = 1 -NON_ANOMALY = 0 -N_SAMPLES = 1000 - - -class LogisticRegression(BaseSML): - """Anomaly detection based on the LogisticRegression algorithm""" - - def __init__(self, _id, _config): - super(LogisticRegression, self).__init__(_id, _config) - self._nb_samples = int(_config['nb_samples']) - - @staticmethod - def validate_config(_config): - log_reg_schema = voluptuous.Schema({ - 'module': voluptuous.And(six.string_types[0], - NoSpaceCharacter()), - 'nb_samples': voluptuous.Or(float, int) - }, required=True) - return log_reg_schema(_config) - - @staticmethod - def get_default_config(): - return { - 'module': LogisticRegression.__name__, - 'nb_samples': N_SAMPLES - } - - def get_params(): - return [ - params.ParamDescriptor('nb_samples', type_util.Number(), N_SAMPLES) - ] - - def number_of_samples_required(self): - return self._nb_samples - - def _generate_train_test_sets(self, samples, ratio_train): - num_samples_train = int(len(samples) * ratio_train) - - data, labels = np.hsplit(samples, [-1]) - X_train = np.array(data[:num_samples_train]) - _labels = np.array(labels[:num_samples_train]) - X_train_label = _labels.ravel() - X_test = np.array(data[num_samples_train:]) - _labels = np.array(labels[num_samples_train:]) - X_test_label = _labels.ravel() - return X_train, X_train_label, X_test, X_test_label - - def _get_best_detector(self, train, label): - detector = linear_model.LogisticRegression() - detector.fit(train, label) - return detector - - def learn_structure(self, samples): - X_train, X_train_label, X_test, X_test_label = \ - self._generate_train_test_sets(samples, 0.75) - logger.info('Training with ' + str(len(X_train)) + - 'samples; testing with ' + str(len(X_test)) + ' samples.') - - lr_detector = self._get_best_detector(X_train, X_train_label) - Y_test = lr_detector.predict(X_test) - - num_anomalies = Y_test[Y_test == ANOMALY].size - logger.info('Found ' + str(num_anomalies) + - ' anomalies in testing set') - - logger.info('Confusion Matrix: \n{}'. - format(classification_report( - X_test_label, - Y_test, - target_names=['no', 'yes']))) - return lr_detector diff --git a/monasca_analytics/sml/random_forest_classifier.py b/monasca_analytics/sml/random_forest_classifier.py deleted file mode 100644 index 66dffdf..0000000 --- a/monasca_analytics/sml/random_forest_classifier.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -from sklearn import ensemble -from sklearn.metrics import classification_report -import voluptuous - -from monasca_analytics.sml.base import BaseSML -from monasca_analytics.util.validation_utils import NoSpaceCharacter - -logger = logging.getLogger(__name__) - -ANOMALY = 1 -NON_ANOMALY = 0 -N_SAMPLES = 1000 - - -class RandomForestClassifier(BaseSML): - """Anomaly detection based on the RandomForestClassifier algorithm""" - - def __init__(self, _id, _config): - super(RandomForestClassifier, self).__init__(_id, _config) - self._nb_samples = int(_config['nb_samples']) - - @staticmethod - def validate_config(_config): - randomforest_schema = voluptuous.Schema({ - 'module': voluptuous.And(six.string_types[0], - NoSpaceCharacter()), - 'nb_samples': voluptuous.Or(float, int) - }, required=True) - return randomforest_schema(_config) - - @staticmethod - def get_default_config(): - return { - 'module': RandomForest.__name__, - 'nb_samples': N_SAMPLES - } - - def get_params(): - return [ - params.ParamDescriptor('nb_samples', type_util.Number(), N_SAMPLES) - ] - - def number_of_samples_required(self): - return self._nb_samples - - def _generate_train_test_sets(self, samples, ratio_train): - num_samples_train = int(len(samples) * ratio_train) - - data, labels = np.hsplit(samples, [-1]) - X_train = np.array(data[:num_samples_train]) - _labels = np.array(labels[:num_samples_train]) - X_train_label = _labels.ravel() - X_test = np.array(data[num_samples_train:]) - _labels = np.array(labels[num_samples_train:]) - X_test_label = _labels.ravel() - return X_train, X_train_label, X_test, X_test_label - - def _get_best_detector(self, train, label): - detector = ensemble.RandomForestClassifier() - detector.fit(train, label) - return detector - - def learn_structure(self, samples): - X_train, X_train_label, X_test, X_test_label = \ - self._generate_train_test_sets(samples, 0.75) - logger.info('Training with ' + str(len(X_train)) + - 'samples; testing with ' + str(len(X_test)) + ' samples.') - - rf_detector = self._get_best_detector(X_train, X_train_label) - Y_test = rf_detector.predict(X_test) - - num_anomalies = Y_test[Y_test == ANOMALY].size - logger.info('Found ' + str(num_anomalies) + - ' anomalies in testing set') - - logger.info('Confusion Matrix: \n{}'. - format(classification_report( - X_test_label, - Y_test, - target_names=['no', 'yes']))) - return rf_detector diff --git a/monasca_analytics/sml/svc.py b/monasca_analytics/sml/svc.py deleted file mode 100644 index 22729a3..0000000 --- a/monasca_analytics/sml/svc.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -from sklearn.metrics import classification_report -from sklearn import svm -import voluptuous - -from monasca_analytics.sml.base import BaseSML -from monasca_analytics.util.validation_utils import NoSpaceCharacter - -logger = logging.getLogger(__name__) - -ANOMALY = 1 -NON_ANOMALY = 0 -N_SAMPLES = 1000 - - -class Svc(BaseSML): - """Anomaly detection based on the SVC algorithm""" - - def __init__(self, _id, _config): - super(Svc, self).__init__(_id, _config) - self._nb_samples = int(_config['nb_samples']) - - @staticmethod - def validate_config(_config): - svc_schema = voluptuous.Schema({ - 'module': voluptuous.And(six.string_types[0], - NoSpaceCharacter()), - 'nb_samples': voluptuous.Or(float, int) - }, required=True) - return svc_schema(_config) - - @staticmethod - def get_default_config(): - return { - 'module': Svc.__name__, - 'nb_samples': N_SAMPLES - } - - def get_params(): - return[ - params.ParamDescriptor('nb_samples', type_util.Number(), N_SAMPLES) - ] - - def number_of_samples_required(self): - return self._nb_samples - - def _generate_train_test_sets(self, samples, ratio_train): - num_samples_train = int(len(samples) * ratio_train) - - data, labels = np.hsplit(samples, [-1]) - X_train = np.array(data[:num_samples_train]) - _labels = np.array(labels[:num_samples_train]) - X_train_label = _labels.ravel() - X_test = np.array(data[num_samples_train:]) - _labels = np.array(labels[num_samples_train:]) - X_test_label = _labels.ravel() - return X_train, X_train_label, X_test, X_test_label - - def _get_best_detector(self, train, label): - detector = svm.SVC(kernel='rbf') - detector.fit(train, label) - return detector - - def learn_structure(self, samples): - X_train, X_train_label, X_test, X_test_label = \ - self._generate_train_test_sets(samples, 0.75) - logger.info('Training with ' + str(len(X_train)) + - 'samples; testing with ' + str(len(X_test)) + ' samples.') - - svc_detector = self._get_best_detector(X_train, X_train_label) - Y_test = svc_detector.predict(X_test) - - num_anomalies = Y_test[Y_test == ANOMALY].size - logger.info('Found ' + str(num_anomalies) + - ' anomalies in testing set') - - logger.info('Confusion Matrix: \n{}'. - format(classification_report( - X_test_label, - Y_test, - target_names=['no', 'yes']))) - return svc_detector diff --git a/monasca_analytics/sml/svm_one_class.py b/monasca_analytics/sml/svm_one_class.py deleted file mode 100644 index e143b53..0000000 --- a/monasca_analytics/sml/svm_one_class.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -import six -from sklearn import svm -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -from monasca_analytics.sml import base -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - -ANOMALY = -1 -NON_ANOMALY = 1 -N_SAMPLES = 1000 -OUTLIERS_FRACTION = 0.10 - - -class SvmOneClass(base.BaseSML): - """Anomaly detection based on the SVM one class algorithm""" - - def __init__(self, _id, _config): - super(SvmOneClass, self).__init__(_id, _config) - self._nb_samples = int(_config["nb_samples"]) - - @staticmethod - def validate_config(_config): - svm_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "nb_samples": voluptuous.Or(float, int) - }, required=True) - return svm_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": SvmOneClass.__name__, - "nb_samples": N_SAMPLES - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor("nb_samples", type_util.Number(), N_SAMPLES) - ] - - def number_of_samples_required(self): - return self._nb_samples - - def _generate_train_test_sets(self, samples, ratio_train): - num_samples_train = int(len(samples) * ratio_train) - X_train = np.array(samples[:num_samples_train]) - X_test = np.array(samples[num_samples_train:]) - return X_train, X_test - - def learn_structure(self, samples): - X_train, X_test = self._generate_train_test_sets(samples, 0.75) - logger.info("Training with " + str(len(X_train)) + - "samples; testing with " + str(len(X_test)) + " samples.") - svm_detector = svm.OneClassSVM(nu=0.95 * OUTLIERS_FRACTION + 0.05, - kernel="rbf", gamma=0.1) - svm_detector.fit(X_train) - Y_test = svm_detector.predict(X_test) - num_anomalies = Y_test[Y_test == -1].size - logger.info("Found " + str(num_anomalies) + - " anomalies in testing set") - return svm_detector diff --git a/monasca_analytics/source/__init__.py b/monasca_analytics/source/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/source/base.py b/monasca_analytics/source/base.py deleted file mode 100644 index 0e20ccb..0000000 --- a/monasca_analytics/source/base.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import logging - -from monasca_analytics.component import base - -logger = logging.getLogger(__name__) - - -class BaseSource(base.BaseComponent): - """Base class for source provider - - To be extended by concrete sources - """ - - def __init__(self, _id, _config): - super(BaseSource, self).__init__(_id, _config) - # This is fine to store the config within the class for sources - # as they're not supposed to do any processing with the dstream - # Thus won't be transmitted to workers. - self._config = _config - - @abc.abstractmethod - def get_feature_list(self): - """Returns the list of features names. - - :rtype: list[str] - """ - pass - - @abc.abstractmethod - def create_dstream(self, ssc): - """Create a dstream to be consumed by Monanas. - - :type ssc: pyspark.streaming.StreamingContext - :param ssc: Spark streaming context. It shouldn't be stored by self. - :rtype: pyspark.streaming.DStream - :returns: a Spark dstream to be consumed by Monanas. - """ - pass - - @abc.abstractmethod - def terminate_source(self): - """Terminate the source. - - Note to implementers: - - Please, implement the termination logic here, like - disconnecting from a server, unsubscribing from a queue, - closing a file, etc. - """ - pass diff --git a/monasca_analytics/source/cloud_markov_chain.py b/monasca_analytics/source/cloud_markov_chain.py deleted file mode 100644 index d39c02c..0000000 --- a/monasca_analytics/source/cloud_markov_chain.py +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params - -from monasca_analytics.source.markov_chain import base -from monasca_analytics.source.markov_chain import events as ev -import monasca_analytics.source.markov_chain.prob_checks as pck -import monasca_analytics.source.markov_chain.state_check as dck -import monasca_analytics.source.markov_chain.transition as tr -from monasca_analytics.util import validation_utils as vu - -import six - - -logger = logging.getLogger(__name__) - - -class CloudMarkovChainSource(base.MarkovChainSource): - - @staticmethod - def validate_config(_config): - source_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "min_event_per_burst": voluptuous.Or(float, int), - "sleep": voluptuous.And( - float, voluptuous.Range( - min=0, max=1, min_included=False, max_included=False)), - "transitions": { - "web_service": { - "run=>slow": { - voluptuous.And(vu.NumericString()): voluptuous.And( - voluptuous.Or(int, float), - voluptuous.Range(min=0, max=1)), - }, - "slow=>run": { - voluptuous.And(vu.NumericString()): voluptuous.And( - voluptuous.Or(int, float), - voluptuous.Range(min=0, max=1)), - }, - "stop=>run": voluptuous.And( - voluptuous.Or(int, float), - voluptuous.Range(min=0, max=1)), - }, - "switch": { - "on=>off": voluptuous.And(voluptuous.Or(int, float), - voluptuous.Range(min=0, max=1)), - "off=>on": voluptuous.And(voluptuous.Or(int, float), - voluptuous.Range(min=0, max=1)), - }, - "host": { - "on=>off": voluptuous.And(voluptuous.Or(int, float), - voluptuous.Range(min=0, max=1)), - "off=>on": voluptuous.And(voluptuous.Or(int, float), - voluptuous.Range(min=0, max=1)), - }, - }, - "triggers": { - "support": { - "get_called": { - voluptuous.And(vu.NumericString()): voluptuous.And( - voluptuous.Or(int, float), - voluptuous.Range(min=0, max=1)), - }, - }, - }, - "graph": { - voluptuous.And(six.string_types[0], - vu.ValidMarkovGraph()): [six.string_types[0]] - } - }, required=True) - return source_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": CloudMarkovChainSource.__name__, - "sleep": 0.01, - "min_event_per_burst": 500, - "transitions": { - "web_service": { - "run=>slow": { - "0": 0.001, - "8": 0.02, - "12": 0.07, - "14": 0.07, - "22": 0.03, - "24": 0.001 - }, - "slow=>run": { - "0": 0.99, - "8": 0.7, - "12": 0.1, - "14": 0.1, - "22": 0.8, - "24": 0.99 - }, - "stop=>run": 0.7 - }, - "host": { - "on=>off": 0.005, - "off=>on": 0.5 - }, - "switch": { - "on=>off": 0.01, - "off=>on": 0.7 - } - }, - "triggers": { - "support": { - "get_called": { - "0": 0.1, - "8": 0.2, - "12": 0.8, - "14": 0.8, - "22": 0.5, - "24": 0.0 - } - } - }, - "graph": { - "h1:host": ["s1"], - "h2:host": ["s1"], - "s1:switch": [], - "w1:web_service": ["h1"], - "w2:web_service": ["h2"] - } - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('sleep', type_util.Number(), 0.01), - params.ParamDescriptor('min_event_per_burst', type_util.Number(), - 500), - params.ParamDescriptor('transitions', type_util.Object({ - 'web_service': type_util.Object({ - 'run=>slow': type_util.Any(), - 'slow=>run': type_util.Any(), - 'stop=>run': type_util.Any(), - }), - 'switch': type_util.Object({ - 'on=>off': type_util.Number(), - 'off=>on': type_util.Number(), - }), - 'host': type_util.Object({ - 'on=>off': type_util.Number(), - 'off=>on': type_util.Number(), - }) - })), - params.ParamDescriptor('triggers', type_util.Object({ - 'support': type_util.Object({ - 'get_called': type_util.Any() - }) - })), - params.ParamDescriptor('graph', type_util.Any()) - ] - - def get_feature_list(self): - node_names = [k.split(":")[0] - for k in dict(self._config["graph"]).keys()] - node_names.append("support_1") - return node_names - - def _create_system(self): - triggers = self._create_event_triggers() - markov_chains = self._create_markov_chain_models() - graph = self._config["graph"] - nodes = {} - support_node = base.StateNode(None, - markov_chains["support"], - triggers["support"], - _id="support_1") - - for k in graph.keys(): - node_name, node_type = k.split(":") - if node_type == "host": - nodes[node_name] = base.StateNode("on", markov_chains["host"], - triggers["host"], - _id=node_name) - elif node_type == "switch": - nodes[node_name] = base.StateNode("on", - markov_chains["switch"], - triggers["switch"], - _id=node_name) - elif node_type == "web_service": - webs = base.StateNode("run", markov_chains["web_service"], - triggers["web_service"], - _id=node_name) - support_node.dependencies.append(webs) - nodes[node_name] = webs - - for k, v in six.iteritems(graph): - node_name, _ = k.split(":") - - for depend_on in v: - if depend_on not in nodes: - logger.warn( - "Configuration error: '{}'" - " is not a proper dependency" - " of '{}'".format(depend_on, node_name)) - else: - n = nodes[node_name] - o = nodes[depend_on] - n.dependencies.append(o) - - return [support_node] - - def _create_event_triggers(self): - triggers = self._config["triggers"] - sup_tr = triggers["support"] - user_support = ev.Trigger( - prob_check=pck.ProbCheck(sup_tr["get_called"]), - node_check=dck.AnyDepCheck(dck.NeqCheck("run")), - event_builder=ev.EventBuilder( - "User complained for poor web service")) - webs1 = ev.Trigger( - node_check=dck.OrCheck( - dck.EqCheck("stop"), - dck.DepCheck(dck.EqCheck("off")), - dck.DepCheck(dck.DepCheck(dck.EqCheck("off"))) - ), - prob_check=pck.NoProbCheck(), - event_builder=ev.EventBuilder("Application is down") - ) - webs2 = ev.Trigger( - node_check=dck.EqCheck("slow"), - prob_check=pck.NoProbCheck(), - event_builder=ev.EventBuilder("Application is slow") - ) - host = ev.Trigger( - node_check=dck.OrCheck(dck.EqCheck("off"), - dck.DepCheck(dck.EqCheck("off"))), - prob_check=pck.NoProbCheck(), - event_builder=ev.EventBuilder("Host is unreachable or down") - ) - switch = ev.Trigger( - node_check=dck.EqCheck("off"), - prob_check=pck.NoProbCheck(), - event_builder=ev.EventBuilder("Switch is unreachable or down") - ) - - return { - "switch": switch, - "host": host, - "web_service": [webs1, webs2], - "support": user_support - } - - def _create_markov_chain_models(self): - transitions = self._config["transitions"] - - # Switch Transitions - sw_tr = transitions["switch"] - tr1 = tr.Transition( - from_state="on", to_state="off", - prob_check=pck.ProbCheck(sw_tr["on=>off"])) - tr2 = tr.Transition( - from_state="off", to_state="on", - prob_check=pck.ProbCheck(sw_tr["off=>on"])) - switch_mc = tr.MarkovChain([tr1, tr2]) - - # Host Transitions - hs_tr = transitions["host"] - tr1 = tr.Transition( - from_state="on", to_state="off", - prob_check=pck.ProbCheck(hs_tr["on=>off"])) - tr2 = tr.Transition( - from_state="off", to_state="on", - prob_check=pck.ProbCheck(hs_tr["off=>on"])) - host_mc = tr.MarkovChain([tr1, tr2]) - - # Web service Transitions - ws_tr = transitions["web_service"] - tr1 = tr.Transition( - from_state="run", to_state="stop", - prob_check=pck.NoProbCheck(), - deps_check=dck.EqCheck("off")) - tr2 = tr.Transition( - from_state="slow", to_state="stop", - prob_check=pck.NoProbCheck(), - deps_check=dck.EqCheck("off")) - tr3 = tr.Transition( - from_state="slow", to_state="run", - prob_check=pck.NoProbCheck(), - deps_check=dck.AndCheck( - dck.EqCheck("on"), - dck.DepCheck(dck.EqCheck("off")))) - tr4 = tr.Transition( - from_state="run", to_state="slow", - prob_check=pck.ProbCheck(ws_tr["run=>slow"]), - deps_check=dck.AndCheck( - dck.EqCheck("on"), - dck.DepCheck(dck.EqCheck("on")))) - tr5 = tr.Transition( - from_state="slow", to_state="run", - prob_check=pck.ProbCheck(ws_tr["slow=>run"]), - deps_check=dck.AndCheck( - dck.EqCheck("on"), - dck.DepCheck(dck.EqCheck("on")))) - tr6 = tr.Transition( - from_state="stop", to_state="run", - prob_check=pck.ProbCheck(ws_tr["stop=>run"]), - deps_check=dck.EqCheck("on")) - webs_mc = tr.MarkovChain([tr1, tr2, tr3, tr4, tr5, tr6]) - - # User support markov chain - sup_mc = tr.MarkovChain([]) - - return { - "switch": switch_mc, - "host": host_mc, - "web_service": webs_mc, - "support": sup_mc - } diff --git a/monasca_analytics/source/iptables_markov_chain.py b/monasca_analytics/source/iptables_markov_chain.py deleted file mode 100644 index 633e60a..0000000 --- a/monasca_analytics/source/iptables_markov_chain.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params - -from monasca_analytics.source.markov_chain import base -from monasca_analytics.source.markov_chain import events -from monasca_analytics.source.markov_chain import prob_checks as pck -import monasca_analytics.source.markov_chain.state_check as dck -import monasca_analytics.source.markov_chain.transition as tr -from monasca_analytics.util import validation_utils as vu - -import six - - -logger = logging.getLogger(__name__) - -STATE_STOP = "stop" -STATE_NORMAL = "normal" -STATE_ATTACK = "ping_flood_attack" - -iptables = {"INPUT -i eth0 -p tcp --dport 22 -j ACCEPT": "ssh0", - "OUTPUT -o eth0 -p tcp --sport 22 -j ACCEPT": "ssh1", - "INPUT -s 1.2.3.4 -j DROP": "ip0", - "INPUT -s 5.6.7.8 -j DROP": "ip1", - "INPUT -s 1.2.1.2 -j DROP": "ip2", - "INPUT -s 6.5.4.3 -j DROP": "ip3", - "INPUT -i eth0 -p tcp --dport 80 -j ACCEPT": "http0", - "OUTPUT -o eth0 -p tcp --sport 80 -j ACCEPT": "http1", - "INPUT -p icmp --icmp-type echo-request -j ACCEPT": "ping0", - "OUTPUT -p icmp --icmp-type echo-reply -j ACCEPT": "ping1", - "OUTPUT -p icmp --icmp-type echo-request -j ACCEPT": "ping2", - "INPUT -p icmp --icmp-type echo-reply -j ACCEPT": "ping3"} - - -iptable_types = ["ssh", "ip", "http", "ping"] - - -def get_iptable_type(identifier): - for ip_type in iptable_types: - if identifier.startswith(ip_type): - return ip_type - - -class IPTablesSource(base.MarkovChainSource): - """This source class implements an IPTable triggering emulator. - - It models a network system where iptables are triggered following - a specific probability depending on the state of the Markov chain. - """ - - @staticmethod - def validate_config(_config): - source_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "sleep": voluptuous.And( - float, - voluptuous.Range( - min=0, max=1, min_included=False, max_included=False)), - }, required=True) - return source_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": IPTablesSource.__name__, - "sleep": 0.01, - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('sleep', type_util.Number(), 0.01) - ] - - def get_feature_list(self): - return iptable_types - - def _create_system(self): - """Markov Chain of IPTables being triggered in each state. - - Implements the Markov Chain model corresponding to the IPTables that - are triggered depending on the state. - In order to create the model, we have to create the triggers - (events with some probability for each state), - and the Markov chain model (probabilities of state transitions). - """ - logger.debug("Creating IPTables System") - triggers = self._create_event_triggers() - logger.debug("Generated " + str(len(triggers)) + - " IPTables event triggers") - markov_chain = self._create_markov_chain_model() - logger.debug("Created Markov chain model for IPTables") - support_node = base.StateNode(STATE_STOP, - markov_chain, - triggers) - return [support_node] - - def _create_markov_chain_model(self): - """Defines the Markov chain transitions. - - The transition will happen with the probability defined by ProbCheck, - which will be evaluated each server_sleep_in_seconds time - (defined by config). The values are quite low because the time when - these are evaluated is the same as when the traffic is evaluated. - Over time, the probability accumulation is much higher, though. - """ - tr_stop_normal = tr.Transition( - from_state=STATE_STOP, to_state=STATE_NORMAL, - prob_check=pck.ProbCheck(0.8)) - tr_normal_stop = tr.Transition( - from_state=STATE_NORMAL, to_state=STATE_STOP, - prob_check=pck.ProbCheck(0.001)) - tr_normal_attack = tr.Transition( - from_state=STATE_NORMAL, to_state=STATE_ATTACK, - prob_check=pck.ProbCheck(0.0001)) - tr_attack_normal = tr.Transition( - from_state=STATE_ATTACK, to_state=STATE_NORMAL, - prob_check=pck.ProbCheck(0.2)) - return tr.MarkovChain([tr_stop_normal, - tr_normal_stop, - tr_normal_attack, - tr_attack_normal]) - - def _create_event_triggers(self): - """Defines the events that will be triggered in each state.""" - return self._create_normal_traffic_behaviour() +\ - self._create_ping_flood_traffic_behaviour() - - def _create_normal_traffic_behaviour(self): - """These are the triggers that will happen in the NORMAL state. - - The http traffic is predominant, but there is also some ping traffic, - and a little bit of ssh traffic - """ - tr = [] - for iptable, feature in six.iteritems(iptables): - if feature.startswith("ssh"): - tr.append(self._create_trigger(0.1, STATE_NORMAL, iptable)) - elif feature.startswith("http"): - tr.append(self._create_trigger(0.6, STATE_NORMAL, iptable)) - elif feature.startswith("ping"): - tr.append(self._create_trigger(0.2, STATE_NORMAL, iptable)) - return tr - - def _create_ping_flood_traffic_behaviour(self): - """These are the triggers that will happen in the ATTACK state. - - The ssh and http traffic is the same as in the normal state, - but the ping traffic is dramatically increased - """ - tr = [] - for iptable, feature in six.iteritems(iptables): - if feature.startswith("ssh"): - tr.append(self._create_trigger(0.1, STATE_ATTACK, iptable)) - elif feature.startswith("http"): - tr.append(self._create_trigger(0.6, STATE_ATTACK, iptable)) - elif feature.startswith("ping"): - tr.append(self._create_trigger(0.95, STATE_ATTACK, iptable)) - return tr - - def _create_trigger(self, prob, state, event_msg): - """Aux function to create an event trigger - - :param prob: float between 0 and 1 -- probability of the event - being triggered - :param state: str -- State where this event can be triggered - :param event_msg: str -- message that will be sent for this event - """ - return events.Trigger( - prob_check=pck.ProbCheck(prob), - node_check=dck.EqCheck(state), - event_builder=events.EventBuilder(event_msg)) diff --git a/monasca_analytics/source/kafka.py b/monasca_analytics/source/kafka.py deleted file mode 100644 index 61753fa..0000000 --- a/monasca_analytics/source/kafka.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -from pyspark.streaming import kafka -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params - -from monasca_analytics.source import base -from monasca_analytics.util import validation_utils as vu - -import six - - -logger = logging.getLogger(__name__) - - -class KafkaSource(base.BaseSource): - """A Kafka source implementation that consumes data from a Kafka queue.""" - - @staticmethod - def validate_config(_config): - source_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "params": { - "zk_host": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "zk_port": int, - "group_id": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "topics": { - voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()): - voluptuous.And(int, voluptuous.Range(min=1)) - } - } - }, required=True) - return source_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": KafkaSource.__name__, - "params": { - "zk_host": "localhost", - "zk_port": 2181, - "group_id": "my_group_id", - "topics": { - "my_topic": 1 - } - } - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('zk_host', type_util.String(), - 'localhost'), - params.ParamDescriptor('zk_port', type_util.Number(), - 2181), - params.ParamDescriptor('group_id', type_util.String(), - 'my_group_id'), - params.ParamDescriptor('topics', - type_util.Object(strict_checking=False)) - ] - - def create_dstream(self, ssc): - """Dstream creation - - The _dstream object is created before this source is bound - to the consumers. It uses a KafkaUtils.createStream, to read data from - the Kafka queue that was defined in the configuration. - - :type ssc: pyspark.streaming.StreamingContext - :param ssc: Spark Streaming Context - """ - return kafka.KafkaUtils.createStream( - ssc, - "{0}:{1}".format( - self._config["params"]["zk_host"], - self._config["params"]["zk_port"]), - self._config["params"]["group_id"], - self._config["params"]["topics"]) - - def terminate_source(self): - pass - - def get_feature_list(self): - raise NotImplementedError("This method needs to be implemented") diff --git a/monasca_analytics/source/markov_chain/__init__.py b/monasca_analytics/source/markov_chain/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/source/markov_chain/base.py b/monasca_analytics/source/markov_chain/base.py deleted file mode 100644 index b47e80b..0000000 --- a/monasca_analytics/source/markov_chain/base.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import datetime -import itertools -import json -import logging -import six -from six.moves import socketserver -import threading -import time - -from monasca_analytics.source import base - -logger = logging.getLogger(__name__) - - -class MarkovChainSource(base.BaseSource): - """Base Source of data generated according a Markov chain model. - - A Markov chain model baked by a finite state machine - to model more realistic scenario with optional confounding variables. - This source is useful to see how causality is discovered in the presence - of confounding variable as well as scenario where some alert appears only - in a deterministic fashion. - """ - - @abc.abstractmethod - def _create_system(self): - """ - Abstract method that should be implemented by subclasses - - :rtype: list[StateNode] - :returns: List of StateNode that do not have any dependencies - """ - pass - - def create_dstream(self, ssc): - """Initiates the system in a TCP server, and creates the dstream object - - The _dstream object is created before this source is bound - to the consumers. It uses a socketTextStream, to read data from - the ThreadingTCPServer. - - :type ssc: pyspark.streaming.StreamingContext - :param ssc: Spark Streaming Context that provides the data input - """ - system = LeafNodes(self._create_system()) - port = self._start_thread(system) - return ssc.socketTextStream( - "localhost", - port) - - def terminate_source(self): - # This is to allow all the messages - # being sent by the handler to clear up. - self._server.terminate = True - self._server.shutdown() - self._server.server_close() - self._server_thread.join() - - def _start_thread(self, system): - self._server = socketserver.ThreadingTCPServer( - ("", 0), # Let the OS pick a port for us - FMSTCPHandler, # Handler of the requests - False) - self._server.allow_reuse_address = True - self._server.server_bind() - self._server.server_activate() - self._server.terminate = False - self._server.system = system - self._server.sleep_in_seconds = self._config["sleep"] - if "min_event_per_burst" in self._config: - self._server.min_event_per_burst = \ - int(self._config["min_event_per_burst"]) - else: - self._server.min_event_per_burst = 500 - - self._server_thread = threading.Thread(target=self._serve_forever) - self._server_thread.start() - port_used = self._server.socket.getsockname()[1] - return port_used - - def _serve_forever(self): - try: - self._server.serve_forever() - except IOError: - logger.debug("Markov chain source's server stopped.") - - -class MetaId(abc.ABCMeta): - - def __new__(mcs, name, bases, namespace): # @NoSelf - cls = super(abc.ABCMeta, mcs).__new__(mcs, name, bases, namespace) - cls.ids = itertools.count(1) - return cls - - -class LeafNodes(object): - - def __init__(self, state_nodes): - """Constructor with list of state nodes - - :type state_nodes: list[StateNode] - :param state_nodes: The node of the directed acyclic graph - that has no dependencies. - """ - self._state_nodes = state_nodes - - def next_state(self, hour_of_day): - """Move to next state - - :type hour_of_day: int - :param hour_of_day: An hour of the day that is used by - StateNode.next_state - """ - ignored_states = set([]) - for s in self._state_nodes: - s.next_state(hour_of_day, ignored_states) - - def collect_events(self, hour_of_day, fake_date, request): - """Get list of events - - :type hour_of_day: int - :param hour_of_day: An hour of the day that is used by - StateNode.collect_event - :type fake_date: datetime.datetime - :param fake_date: A date that you can use to generate a ctime. - :type request: RequestBuilder - :param request: Request object to send data. - :rtype: list - :returns: List of event. Specific to the event builder. - """ - for node in self._state_nodes: - node.collect_events(hour_of_day, fake_date, request) - - -@six.add_metaclass(MetaId) -class StateNode(object): - """This class describes a particular node in the dependency graph. - - It holds the state information relative to that node and the - dependencies with the other nodes. The Markov - It is managed by one instance of a StateDescriptor. - """ - - def __init__(self, initial_state, markov_chain, trigger, _id=None): - """Constructor - - :type _id: int | str - :param _id: Id of the node - :type initial_state: str | int | None - :param initial_state: Initial state for this node. - :type markov_chain: - monasca_analytics.source.markov_chain.transition.MarkovChain - :param markov_chain: Markov Chain managing this node state. - :type trigger: - list | monasca_analytics.source.markov_chain.events.Trigger - :param trigger: List of triggers or single trigger that you want - to attached to this node. - """ - if _id is None: - self._id = next(self.__class__.ids) - else: - self._id = _id - self.state = initial_state - self._markov_chain = markov_chain - if type(trigger) == list: - self._triggers = trigger - else: - self._triggers = [trigger] - self.dependencies = [] - - def id(self): - """ - :rtype: int | str - :returns: Returns this object id. - """ - return self._id - - def next_state(self, hour_of_day, ignored_states): - """Move this element to the next state if it is not in the ignored set - - This will affect this element's children that conditionally depends - on this element's state. - - :type hour_of_day: int - :param hour_of_day: An integer in the range of 0 to 24 to - express the hour of the day. - :type ignored_states: set - :param ignored_states: set of states that should not change. - """ - if self._id not in ignored_states: - ignored_states.add(self._id) - for dep in self.dependencies: - dep.next_state(hour_of_day, ignored_states) - self._markov_chain.apply_on(self, hour_of_day) - - def collect_events(self, hour_of_day, fake_date, request): - """Collect event triggered for the next burst. - - :type hour_of_day: int - :param hour_of_day: an integer in the range of 0 to 24 to express - he hour of the day. - :type fake_date: datetime.datetime - :param fake_date: A date that you can use to generate a ctime. - :type request: RequestBuilder - :param request: Request builder to send specific events - :rtype: list - :returns: events for this step or None - """ - for trigger in self._triggers: - trigger.apply_on(self, hour_of_day, fake_date, request) - for dep in self.dependencies: - dep.collect_events(hour_of_day, fake_date, request) - - -class FMSTCPHandler(socketserver.BaseRequestHandler): - """A TCP server handler for the alert generation.""" - - def handle(self): - """Handles incoming connection and pushing data into them.""" - - fake_date = datetime.datetime.today() - hour_of_day = fake_date.hour - while not self.server.terminate: - request = RequestBuilder(self.request) - - # Collect some events - while request.nb_events() < self.server.min_event_per_burst: - self.server.system.next_state(hour_of_day) - self.server.system.collect_events(hour_of_day, fake_date, - request) - hour_of_day += 1 - fake_date += datetime.timedelta(hours=1) - if hour_of_day > 24: - hour_of_day = 0 - - try: - request.finalize() - except IOError: - logger.debug("Source is now off") - self.server.terminate = True - - time.sleep(self.server.sleep_in_seconds) - - -class RequestBuilder(object): - - def __init__(self, request): - self._request = request - self._collected_data = [] - - def send(self, data): - """ - Send an object over the network. - :param data: Object to send. - """ - self._collected_data.append(data) - - def nb_events(self): - return len(self._collected_data) - - def finalize(self): - for data in self._collected_data: - self._request.send("{0}\n".format(json.dumps(data, - cls=DictEncoder))) - self._request = None - self._collected_data = None - - -class DictEncoder(json.JSONEncoder): - - def default(self, o): - return o.__dict__ diff --git a/monasca_analytics/source/markov_chain/events.py b/monasca_analytics/source/markov_chain/events.py deleted file mode 100644 index ba05400..0000000 --- a/monasca_analytics/source/markov_chain/events.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class Trigger(object): - """A trigger generate events when a particular graph state is reached.""" - - def __init__(self, node_check, prob_check, event_builder): - """Create a new trigger. - - :type node_check: - (monasca_analytics.source.markov_chain.base.StateNode) -> bool - :param node_check: - Checker function that will return true if the node has an - appropriate type - :type prob_check: (int) -> bool - :param prob_check: - A bernoulli trial that randomly return true or false that - can use the parameter (hour of the day) to modify the - probability of success - - :type event_builder: - (monasca_analytics.source.markov_chain.base.StateNode, - datetime.datetime, - monasca_analytics.source.markov_chain.base.RequestBuilder) -> None - :param event_builder: Event builder that receive the node and use - the state to return an event. - """ - self._prob_check = prob_check - self._node_check = node_check - self._event_builder = event_builder - - def apply_on(self, node, hour_of_day, fake_date, request): - """Apply this trigger on the given node. - - :type node: monasca_analytics.source.markov_chain.base.StateNode - :param node: Node to test the trigger with. - :type hour_of_day: int - :param hour_of_day: An integer between [0, 24) representing - the hour of the day. - :type fake_date: datetime.datetime - :param fake_date: A date that you can use to generate a ctime. - :type request: - monasca_analytics.source.markov_chain.base.RequestBuilder - :param request: Request builder to send events - """ - if self._prob_check(hour_of_day) and self._node_check(node): - self._event_builder(node, fake_date, request) - - -class Event(object): - - def __init__(self, msg, ident): - """ - :type msg: str - :param msg: The event message. - :type ident: str - :param ident: The id of the node causing this event. - """ - self.id = ident - self.msg = msg - - -class EventBuilder(object): - - def __init__(self, msg): - """ - :type msg: str - :param msg: The event message. - """ - self._msg = msg - - def __call__(self, node, fake_date, request): - """ - :type node: monasca_analytics.source.markov_chain.base.StateNode - :param node: The node associated with the event. - :type fake_date: datetime.datetime - :param fake_date: A date that you can use to generate a ctime. - :type request: - monasca_analytics.source.markov_chain.base.RequestBuilder - """ - request.send({ - 'ctime': fake_date.ctime(), - 'event': Event(self._msg, str(node.id())) - }) diff --git a/monasca_analytics/source/markov_chain/prob_checks.py b/monasca_analytics/source/markov_chain/prob_checks.py deleted file mode 100644 index 33df9d7..0000000 --- a/monasca_analytics/source/markov_chain/prob_checks.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import random - -from monasca_analytics.util import math - - -class ProbCheck(object): - """Function object to randomly return True or False with probability - - This function object randomly returns true or false - with a chance of success that can optionally vary - given the hour of the day. - """ - - def __init__(self, prob): - """Create a ProbCheck, or change stat if the prob check is passed. - - :type prob: dict[int, float] | float - :param prob: a probability value or a dictionary where keys correspond - to the hour of the day and the value is the probability of - success associated with it. - """ - if isinstance(prob, dict): - self._prob = list(prob.items()) - else: - self._prob = prob - - def __call__(self, hour_of_day): - if isinstance(self._prob, list): - p = math.interpolate_1d(self._prob, hour_of_day) - else: - p = self._prob - return random.random() < p - - -class NoProbCheck(object): - """When you don't want to have any prob check performed.""" - - def __call__(self, *_): - return True diff --git a/monasca_analytics/source/markov_chain/state_check.py b/monasca_analytics/source/markov_chain/state_check.py deleted file mode 100644 index 3a25fbd..0000000 --- a/monasca_analytics/source/markov_chain/state_check.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class DepCheck(object): - """Dependency checks.""" - - def __init__(self, dep_check): - self._dep_check = dep_check - - def __call__(self, node): - res = True - for dep in node.dependencies: - res = res and self._dep_check(dep) - return res and len(node.dependencies) > 0 - - -class AnyDepCheck(object): - """Dependency checks.""" - - def __init__(self, dep_check): - self._dep_check = dep_check - - def __call__(self, node): - for dep in node.dependencies: - if self._dep_check(dep): - return True - return False - - -class TrueCheck(object): - """This dep check always pass.""" - - def __call__(self, *_): - return True - - -class EqCheck(object): - """Check that the node has the appropriate state.""" - - def __init__(self, state): - self._state = state - - def __call__(self, node): - return node.state == self._state - - -class NeqCheck(object): - """Check that the node has the appropriate state.""" - - def __init__(self, state): - self._state = state - - def __call__(self, node): - return node.state != self._state - - -class OrCheck(object): - """Combine two check in an or expression.""" - - def __init__(self, c1, c2, c3=None): - self._c1 = c1 - self._c2 = c2 - self._c3 = c3 - - def __call__(self, node): - if self._c3 is None: - return self._c1(node) or self._c2(node) - else: - return self._c1(node) or self._c2(node) or self._c3(node) - - -class AndCheck(object): - """Combine two check in an or expression.""" - - def __init__(self, c1, c2): - self._c1 = c1 - self._c2 = c2 - - def __call__(self, node): - return self._c1(node) and self._c2(node) diff --git a/monasca_analytics/source/markov_chain/transition.py b/monasca_analytics/source/markov_chain/transition.py deleted file mode 100644 index 901f3ef..0000000 --- a/monasca_analytics/source/markov_chain/transition.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.source.markov_chain import state_check as sc - - -class MarkovChain(object): - """Possible transitions for each state. - - This class describes the possible transitions for - a particular type of state. - It can use the dependencies of the node as it want. - """ - - def __init__(self, transitions): - self._transitions = dict() - for tr in transitions: - from_state = tr.from_state() - if from_state in self._transitions: - self._transitions[from_state].append(tr) - else: - self._transitions[from_state] = [tr] - - def apply_on(self, node, hour_of_day): - """Performs a state transition for the given node. - - :type node: monasca_analytics.source.markov_chain.base.StateNode - :param node: the state of this node will be changed - :type hour_of_day: int - :param hour_of_day: the hour of the day (used by probability checks) - :rtype: bool - :returns: True if there exists in the model at least one transition - from the node current state. - """ - if node.state not in self._transitions: - return False - for tr in self._transitions[node.state]: - if tr(node, hour_of_day): - break - return True - - -class Transition(object): - """Holds requirement for a transition - - This function object holds the requirement for effectively - performing a transition on a node. It assumes to be used - only by the MarkovChain class who make sure that the node - has the proper state before applying the transition. - """ - - def __init__(self, from_state, to_state, prob_check, deps_check=None): - """Create a new Transition instance. - - :param from_state: precondition for the node. It must be in this - state in order to have a chance to be promoted into `to_state` - :param to_state: next state if the condition is met. - :param prob_check: the markov condition. - :param deps_check: deterministic check performed on dependencies. - """ - self._from_state = from_state - self._to_state = to_state - if deps_check is not None: - self._deps_check = sc.DepCheck(deps_check) - else: - self._deps_check = sc.TrueCheck() - self._prob_check = prob_check - - def __call__(self, node, hour_of_day): - if self._prob_check(hour_of_day) and\ - self._deps_check(node) and\ - self._from_state == node.state: - node.state = self._to_state - return True - return False - - def from_state(self): - return self._from_state diff --git a/monasca_analytics/source/monasca_markov_chain.py b/monasca_analytics/source/monasca_markov_chain.py deleted file mode 100644 index e88302b..0000000 --- a/monasca_analytics/source/monasca_markov_chain.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import random -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -import six - -import monasca_analytics.source.markov_chain.base as base -import monasca_analytics.source.markov_chain.events as ev -import monasca_analytics.source.markov_chain.prob_checks as pck -import monasca_analytics.source.markov_chain.state_check as dck -import monasca_analytics.source.markov_chain.transition as tr -import monasca_analytics.util.timestamp as tp -from monasca_analytics.util import validation_utils as vu - - -logger = logging.getLogger(__name__) - - -class MonascaMarkovChainSource(base.MarkovChainSource): - - @staticmethod - def validate_config(_config): - markov_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "sleep": voluptuous.And( - float, voluptuous.Range( - min=0, max=1, min_included=False, max_included=False)), - }, required=True) - return markov_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": MonascaMarkovChainSource.__name__, - "sleep": 0.01, - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('sleep', type_util.Number(), 0.01) - ] - - def get_feature_list(self): - return ["vm1", "vm2", "host1", "host2"] - - def _create_system(self): - mc = tr.MarkovChain([]) - vm_triggers = [ - ev.Trigger( - event_builder=MonascaFakeMetricBuilder("vm.mem.used_mb"), - node_check=dck.TrueCheck(), - prob_check=pck.NoProbCheck() - ), - ev.Trigger( - event_builder=MonascaFakeMetricBuilder("cpu.idle_perc"), - node_check=dck.TrueCheck(), - prob_check=pck.NoProbCheck() - ), - ev.Trigger( - event_builder=MonascaFakeMetricBuilder( - "cpu.total_logical_cores"), - node_check=dck.TrueCheck(), - prob_check=pck.NoProbCheck() - ) - ] - host_trigger = ev.Trigger( - event_builder=MonascaFakeMetricBuilder("mem.total_mb"), - node_check=dck.TrueCheck(), - prob_check=pck.NoProbCheck() - ) - - return [ - # vm.mem.used_mb - base.StateNode(3, mc, vm_triggers[0], _id="vm1"), - base.StateNode(1, mc, vm_triggers[0], _id="vm2"), - # cpu.idle_perc - base.StateNode(0.75, mc, vm_triggers[1], _id="vm1"), - base.StateNode(0.75, mc, vm_triggers[1], _id="vm2"), - # cpu.total_logical_cores - base.StateNode(3, mc, vm_triggers[2], _id="vm1"), - base.StateNode(2, mc, vm_triggers[2], _id="vm2"), - # mem.total_mb - base.StateNode(5, mc, host_trigger, _id="host1"), - base.StateNode(6, mc, host_trigger, _id="host2"), - ] - - -class MonascaFakeMetricBuilder(object): - - def __init__(self, metric_name): - """ - :type metric_name: str - :param metric_name: The name of the metric - """ - self.metric_name = metric_name - - def __call__(self, node, fake_date, request): - """ - :type node: monasca_analytics.source.markov_chain.base.StateNode - :param node: The node associated with the event. - :type fake_date: datetime.datetime - :param fake_date: A date that you can use to generate a ctime. - :type request: - monasca_analytics.source.markov_chain.base.RequestBuilder - """ - half_hour = 60 * 60 / 2 - request.send({ - "metric": { - "name": self.metric_name, - "dimensions": { - "service": "monitoring", - "hostname": node.id() - }, - "timestamp": tp.timestamp(fake_date) + - random.randint(- half_hour, half_hour), - "value": node.state - }, - "meta": { - "tenantId": 0, - "region": "earth" - }, - "creation_time": 0 - }) diff --git a/monasca_analytics/source/randoms.py b/monasca_analytics/source/randoms.py deleted file mode 100644 index 882cc49..0000000 --- a/monasca_analytics/source/randoms.py +++ /dev/null @@ -1,403 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import json -import logging -import numpy as np -import random -import six -from six.moves import socketserver -import threading as th -import time -import uuid -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params - -import monasca_analytics.exception.monanas as err -from monasca_analytics.source import base -from monasca_analytics.util import validation_utils as vu - -logger = logging.getLogger(__name__) - - -class RandomSource(base.BaseSource): - """A randomly generated data source implementation.""" - - def __init__(self, _id, _config): - super(RandomSource, self).__init__(_id, _config) - try: - self._configure_server() - except IOError: - raise err.MonanasInitError("Address already in use.") - except AttributeError: - raise err.MonanasInitError("Invalid generate or validate method.") - - def _configure_server(self): - """Creates and configures the Server object - - The server object is configured according to - the configuration of this source module - """ - self._server = socketserver.ThreadingTCPServer( - (self._config["params"]["host"], - self._config["params"]["port"]), - MonanasTCPHandler, False) - self._server.generate = getattr( - self, "_generate_" + - self._config["params"]["model"]["name"]) - # self._server.validate = getattr( - # source_model, self._config["validate"]) - self._server.allow_reuse_address = True - self._server.server_bind() - self._server.server_activate() - self._server.terminate = False - self._server.generate_alerts_per_second =\ - self._config["params"]["alerts_per_burst"] - self._server.generate_idle_time_between_bursts =\ - self._config["params"]["idle_time_between_bursts"] - self._server_thread = th.Thread(target=self._server.serve_forever) - self._is_server_running = False - - @staticmethod - def validate_config(_config): - source_schema = voluptuous.Schema({ - "module": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "params": { - "host": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "port": int, - "model": { - "name": voluptuous.And(six.string_types[0], - vu.NoSpaceCharacter()), - "params": { - "origin_types": voluptuous.And([ - { - "origin_type": voluptuous.And( - six.string_types[0], - vu.NoSpaceCharacter()), - "weight": voluptuous.And( - voluptuous.Or(int, float), - voluptuous.Range( - min=0, min_included=False)), - } - ], vu.NotEmptyArray()), - voluptuous.Optional("key_causes"): dict - } - }, - "alerts_per_burst": voluptuous.And( - int, voluptuous.Range(min=1)), - "idle_time_between_bursts": voluptuous.And( - voluptuous.Or(int, float), - voluptuous.Range(min=0, min_included=False)) - } - }, required=True) - return source_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": RandomSource.__name__, - "params": { - "host": "localhost", - "port": 1010, - "model": { - "name": "my_model_name", - "params": { - "origin_types": [ - { - "origin_type": "my_origin_type", - "weight": 1.0 - } - ], - } - }, - "alerts_per_burst": 1, - "idle_time_between_bursts": 1.0 - } - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('host', type_util.String(), 'localhost'), - params.ParamDescriptor('port', type_util.Number(), 1010), - params.ParamDescriptor('model', type_util.Object({ - 'name': type_util.String(), - 'params': type_util.Object({ - 'origin_types': type_util.Object(strict_checking=False) - }) - })), - params.ParamDescriptor('alert_per_burst', type_util.Number(), 1), - params.ParamDescriptor('idle_time_between_bursts', - type_util.Number(), 1.0), - ] - - def _start_server(self): - if not self._is_server_running: - self._server_thread.start() - self._is_server_running = True - - def create_dstream(self, ssc): - """Dstream object creation - - The _dstream object is created before this source is bound - to the consumers. It uses a socketTextStream, to read data from - the ThreadingTCPServer. - - :type ssc: pyspark.streaming.StreamingContext - :param ssc: Spark Streaming Context - """ - self._start_server() - self._dstream = ssc.socketTextStream( - self._config["params"]["host"], - self._config["params"]["port"]) - - def get_feature_list(self): - raise NotImplementedError("This method needs to be implemented") - - def terminate_source(self): - """Terminates the source with a delay - - Terminates the source with a delay to allow the messages - being sent by the handler to clear up. - """ - self._server.terminate = True - time.sleep(1) - self._server.server_close() - self._server_thread = None - - def _generate_simple_model(self): - """Generates an alert based on simple_model.""" - current_time = int(round(time.time() * 1000)) - return { - "created": current_time, - "id": str(uuid.uuid4()), - "origin": str(uuid.uuid4()), - "origin_type": self._random_origin_type(), - "data": {}, - "state": "", - "updated": current_time - } - - def _random_origin_type(self): - """Randomizes the origin_type""" - origin_types = self._config[ - "params"]["model"]["params"]["origin_types"] - return origin_types[self._weighted_choice( - [o["weight"] for o in origin_types])]["origin_type"] - - def _weighted_choice(self, weights): - """Gets an index chosen randomly but weighted from a list of weights""" - totals = [] - running_total = 0 - - for w in weights: - running_total += w - totals.append(running_total) - - rnd = random.random() * running_total - - for i, total in enumerate(totals): - if rnd < total: - return i - - -@six.add_metaclass(abc.ABCMeta) -class BaseDataSourceGenerator(object): - """An interface for random data source generators.""" - - @abc.abstractmethod - def __init__(self, _config): - """BaseDataSourceGenerator constructor. - - :type _config: dict - :param _config: Configuration of this source - """ - self._config = _config - self.generate = getattr(self, "generate_" + - self._config["params"]["model"]["name"]) - - @abc.abstractmethod - def is_burst_over(self): - """Should return true when all the burst alerts have been generated""" - pass - - def generate_simple_model(self): - """Generate alert event that are shaped according to the simple model - """ - current_time = time.time() - return { - "created": current_time, - "id": str(uuid.uuid4()), - "origin": str(uuid.uuid4()), - "origin_type": self._pick_next_type(), - "data": {}, - "state": "", - "updated": current_time - } - - @abc.abstractmethod - def _pick_next_type(self): - """Should return the next type for the simple model generation""" - pass - - -class LinearlyDependentDataSourceGenerator(BaseDataSourceGenerator): - """A data source generator where alerts are linearly dependent - - :raises: exception -- if the causal matrix is cyclic - """ - - def __init__(self, config): - BaseDataSourceGenerator.__init__(self, config) - - # Acyclic causality model - config_key_causes = self._config[ - "params"]["model"]["params"]["key_causes"] - - # Create the causal matrix (/graph) - self._features_names = config_key_causes.keys() - n = len(self._features_names) - self._causal_matrix = np.zeros((n, n), dtype=np.float32) - for i in range(n): - for j in range(n): - row = self._features_names[i] - col = self._features_names[j] - if col in config_key_causes[row]: - self._causal_matrix[i, j] = 1 - - # Triangulate the causal matrix - tmp_matrix = np.copy(self._causal_matrix) - n_t = tmp_matrix.shape[0] - while n_t != 1: - for i in range(n_t): - if np.all(tmp_matrix[i, :] == np.zeros(n_t)): - tmp_matrix[[i, 0], :] = tmp_matrix[[0, i], :] - tmp_matrix[:, [i, 0]] = tmp_matrix[:, [0, i]] - k = n - n_t - r = i + k - self._causal_matrix[ - [r, k], :] = self._causal_matrix[[k, r], :] - self._causal_matrix[ - :, [r, k]] = self._causal_matrix[:, [k, r]] - self._features_names[r], self._features_names[ - k] = self._features_names[k], self._features_names[r] - tmp_matrix = tmp_matrix[1:, 1:] - break - if i == n_t - 1: - raise err.MonanasCyclicRandomSourceError - n_t = tmp_matrix.shape[0] - - # Prepare a zero buffer that store the random values generated - # following the causal model - self._features_random_value = np.zeros(len(self._features_names)) - - # This stack will contains the generated values for one burst (if that - # make some sense) - self._features_stack_emitted = [] - logger.debug( - "Causality Matrix (RandomSource): {0}".format( - self._causal_matrix)) - - def is_burst_over(self): - return len(self._features_stack_emitted) == 0 - - def _pick_next_type(self): - while len(self._features_stack_emitted) == 0: - # Generate more features that follows the dag defined by the causal - # matrix - n = len(self._features_names) - self._features_random_value = np.random.laplace(size=n) - for i in range(n): - self._features_random_value[ - i] += np.dot(self._causal_matrix, - self._features_random_value)[i] - - self._features_random_value = np.floor(self._features_random_value) - for i in range(n): - nb = np.abs(int(self._features_random_value[i])) - if nb > 0: - feature = self._features_names[i] - self._features_stack_emitted.extend( - [feature for _ in range(nb)]) - return self._features_stack_emitted.pop() - - -class UncorrelatedDataSourceGenerator(BaseDataSourceGenerator): - """A data source generator where alert item are not correlated. - - Each item has a unique probability to be generated. - """ - - def __init__(self, config): - BaseDataSourceGenerator.__init__(self, config) - self.accumulated_alerts = 0 - self._config = config - - def is_burst_over(self): - is_over = self.accumulated_alerts == self._config[ - "params"]["alerts_per_burst"] - if is_over: - self.accumulated_alerts = 0 - return is_over - - def _pick_next_type(self): - self.accumulated_alerts += 1 - origin_types = self._config[ - "params"]["model"]["params"]["origin_types"] - origin_type = UncorrelatedDataSourceGenerator._weighted_choice( - [o["weight"] for o in origin_types]) - return origin_types[origin_type]["origin_type"] - - @staticmethod - def _weighted_choice(weights): - """Gets an index chosen randomly but weighted from a list of weights""" - totals = [] - running_total = 0 - - for w in weights: - running_total += w - totals.append(running_total) - - rnd = random.random() * running_total - - for i, total in enumerate(totals): - if rnd < total: - return i - - -class MonanasTCPHandler(socketserver.BaseRequestHandler): - """A TCP server handler for the alert generation.""" - - def handle(self): - """Handles the incoming messages.""" - accumulated_alerts = 0 - - while True and not self.server.terminate: - alert = self.server.generate() - - try: - validated_alert = self.server.validate(alert) - self.request.send(json.dumps(validated_alert) + "\n") - accumulated_alerts += 1 - except voluptuous.Invalid: - logger.warn("Invalid schema for generated alerts.") - - time.sleep(self.server.generate_idle_time_between_bursts) diff --git a/monasca_analytics/spark/__init__.py b/monasca_analytics/spark/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/spark/aggregator.py b/monasca_analytics/spark/aggregator.py deleted file mode 100644 index 143baeb..0000000 --- a/monasca_analytics/spark/aggregator.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import logging -import six - -import numpy as np - -logger = logging.getLogger(__name__) - - -@six.add_metaclass(abc.ABCMeta) -class Aggregator(object): - """Aggregator that accumulates data and sends it to SMLs""" - - def __init__(self, driver): - """BaseAggregator constructor. - - :type driver: monasca_analytics.spark.driver.DriverExecutor - :param driver: The driver that manages spark - """ - self._combined_stream = None - self._smls = [] - self._samples = None - self._driver = driver - - def append_sml(self, l): - """The given sml will now be owned and receive the accumulated data - - :type l: monasca_analytics.sml.base.BaseSML - :param l: The sml to connect to. - """ - self._smls.append(l) - - def accumulate_dstream_samples(self, stream): - """Accumulate the samples coming from a stream - - The first time this function is called it sets the _combined_stream - to be the _stream parameter, and the _output_stream to be the - transformed version (according to the logic implemented by children - of this class) of the _combined_stream. - The consecutive times, it joins the _stream to the aggregated stream, - so at runtime _combined_stream is a funnel of all streams passed - to this function. - - :type stream: pyspark.streaming.DStream - :param stream: stream to be collected - """ - if self._combined_stream is None: - self._combined_stream = stream - else: - self._combined_stream = self._combined_stream.union(stream) - - def prepare_final_accumulate_stream_step(self): - """Accumulate each sample into an ndarray. - - This can only be called once accumulate_dstream_samples has been - called on every stream that need to be accumulated together. - """ - if self._combined_stream is not None: - self._combined_stream.foreachRDD( - lambda _, rdd: self._processRDD(rdd)) - - def _processRDD(self, rdd): - """Process the RDD - - :type rdd: pyspark.RDD - :param rdd: A Spark Resilient Distributed Dataset - """ - if len(self._smls) > 0: - rdd_entries = rdd.collect() - for rdd_entry in rdd_entries: - if self._samples is not None: - self._samples = np.vstack([self._samples, rdd_entry]) - else: - self._samples = rdd_entry - self._check_smls() - else: - self._samples = None - - def _check_smls(self): - """Detect if a SML is ready to learn from the set. - - If it is, for simplicity we remove it from the list of SMLs. - """ - if self._samples is None: - return - - def has_learn(sml, samples): - nb_samples = samples.shape[0] - tst = sml.number_of_samples_required() <= nb_samples - if tst: - sml.learn(samples) - return not tst - - logger.debug(self._samples.shape) - self._smls[:] = [l for l in self._smls if has_learn(l, self._samples)] - if len(self._smls) == 0: - self._driver.move_to_phase2() diff --git a/monasca_analytics/spark/driver.py b/monasca_analytics/spark/driver.py deleted file mode 100644 index e55a6dd..0000000 --- a/monasca_analytics/spark/driver.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import pyspark - -import monasca_analytics.config.config as config -import monasca_analytics.ingestor.base as bi -import monasca_analytics.ldp.base as mldp -import monasca_analytics.sink.base as msink -import monasca_analytics.sml.base as bml -import monasca_analytics.spark.aggregator as agg -import monasca_analytics.spark.streaming_context as streamingctx -import monasca_analytics.voter.base as mvoter - -logger = logging.getLogger(__name__) - - -class DriverExecutor(object): - """Driver part of the job submitted to spark. - - This is where we control what is submitted to workers, - what is driver specific and how the pipeline is constructed. - We also execute the pipeline from here. - """ - - def __init__(self, _config): - self._links = None - self._sources = None - self._orchestrator = None - self.set_links(config.instantiate_components(_config)) - - def restart_spark(): - self._ssc = streamingctx.create_streaming_context( - self._sc, - _config) - - self._restart_spark = restart_spark - self._sc = pyspark.SparkContext( - appName=_config["spark_config"]["appName"]) - self._ssc = streamingctx.create_streaming_context(self._sc, _config) - - def set_links(self, links): - """Set new set of links - - This function has no effect on the current pipeline. - In order to use them, you need to restart the pipeline. - """ - self._links = links - logger.debug("Collect sources...") - self._sources = config.collect_sources(self._links) - logger.debug("New list of sources: {}".format(self._sources)) - self._orchestrator = agg.Aggregator(self) - logger.debug("Propagating feature list...") - self._propagate_feature_list() - - def start_pipeline(self): - """Start the pipeline""" - - # Start by connecting the source - if self._phase1_required(): - logger.info("Phase 1 required, ldp won't produce data until" - " smls have finished.") - # Connect sources to ingestors - self._prepare_phase(self._connect_dependents_phase1) - - # Preparation step for the orchestrator: - # Accumulate everything from the sources - self._orchestrator.prepare_final_accumulate_stream_step() - - # Then prepare the orchestrator - self._prepare_orchestrator() - - else: - # Connect sources to ldps - logger.info("Phase 1 was not required, skipping it.") - self._prepare_phase(self._connect_dependents_phase2) - - logger.info("Start the streaming context") - self._ssc.start() - - def stop_pipeline(self): - logger.debug("Stop spark context.") - self._ssc.stop(False, False) - logger.debug("Terminate sources.") - self._terminate_sources() - logger.debug("Restart spark context.") - self._restart_spark() - - def move_to_phase2(self): - if self._ssc is not None: - logger.debug("Phase 2: Stop SparkStreamingContext.") - self._ssc.stop(False, False) - logger.debug("Phase 2: Stop sources") - self._terminate_sources() - logger.debug("Phase 2: Restart streaming...") - self._restart_spark() - logger.debug("Phase 2: Create new connections") - self._prepare_phase(self._connect_dependents_phase2) - self._ssc.start() - # ? - self._ssc.awaitTermination() - - def _terminate_sources(self): - """Terminates the sources.""" - for source in self._sources: - source.terminate_source() - - def _phase1_required(self): - for src in self._sources: - if any(isinstance(el, bi.BaseIngestor) for el in self._links[src]): - return True - - return False - - def _prepare_orchestrator(self): - """ - This is a part of phase 1. The orchestrator collects - input from all ingestors and then orchestrate the sml - pipeline to solve it and provide to LDPs the learned - data structure. - """ - smls = filter(lambda c: isinstance(c, bml.BaseSML), - self._links.keys()) - sml_with_no_dependents = filter( - lambda c: set(self._links[c]).isdisjoint(smls), - smls) - for sml in sml_with_no_dependents: - logger.debug("Append {} to orchestrator".format(sml)) - self._orchestrator.append_sml(sml) - self._connect_sml_dependents(sml) - - def _prepare_phase(self, connect_dependent): - """Prepare given phase by starting sources. - - :type connect_dependent: (pyspark.streaming.DStream, - monasca_analytics.source.base.BaseSource) -> None - :param connect_dependent: Callback that is going to selectively connect - the appropriate dependencies of each sources. - """ - for src in self._sources: - logger.debug("Prepare source {}".format(src)) - dstream = src.create_dstream(self._ssc) - connect_dependent(dstream, src) - - def _connect_sml_dependents(self, from_component): - """Connect an sml component with all its dependencies. - - During phase 1 this code is running exclusively by the driver - at the moment. - - :type from_component: bml.BaseSML | mvoter.BaseVoter - :param from_component: Where we came from. - """ - for connected_node in self._links[from_component]: - - # SML can, for now, only be connected to voter. - if isinstance(connected_node, mvoter.BaseVoter) and \ - isinstance(from_component, bml.BaseSML): - logger.debug("Set {} to {}" - .format(connected_node, from_component)) - from_component.set_voter(connected_node) - - # Voter can only be connected to LDPs - if isinstance(from_component, mvoter.BaseVoter) and \ - isinstance(connected_node, mldp.BaseLDP): - logger.debug("Append {} to {}" - .format(connected_node, from_component)) - from_component.append_ldp(connected_node) - # We don't connect LDP to anything - continue - - # Only SML can be connected to a sink - if isinstance(connected_node, msink.BaseSink): - logger.debug("Sink {} into {}" - .format(from_component, connected_node)) - connected_node.sink_ml(from_component) - # Sink can't be connected to anything - continue - - self._connect_sml_dependents(connected_node) - - def _connect_dependents_phase2(self, dstream, from_component): - """Connect a component to its dependencies. - - During phase 2, only live data processors are considered. - All ingestors are shutdown. - - :type dstream: pyspark.streaming.DStream | None - :param dstream: Dstream that will be modified by dependent. - It can be None, only if from_component is aggregator, - sml or voter. - :type from_component: monasca_analytics.component.base.BaseComponent - :param from_component: Where we came from. - """ - for connected_node in self._links[from_component]: - # Live data processors are also doing a map, they add - # the causality bit to each element in the stream. - if isinstance(connected_node, mldp.BaseLDP): - logger.debug("Connecting {} to {}".format(from_component, - connected_node)) - new_dstream = connected_node.map_dstream(dstream) - self._connect_dependents_phase2(new_dstream, connected_node) - - # Sink are at the end of the branch! - if isinstance(connected_node, msink.BaseSink): - logger.debug("Sink {} into {}".format(from_component, - connected_node)) - connected_node.sink_dstream(dstream) - - def _connect_dependents_phase1(self, dstream, from_component): - """Connect a component to its dependencies for phase 1. - - All live data processors are ignored during that phase. - - :type dstream: pyspark.streaming.DStream | None - :param dstream: Dstream that will be modified by dependent. - It can be None, only if from_component is aggregator, - sml or voter. - :type from_component: monasca_analytics.component.base.BaseComponent -- - :param from_component: Where we came from. - """ - for connected_node in self._links[from_component]: - - # Ingestors "map" the dstream. They are mainly doing worker - # specific transformation. Like parsing and vectorizing the - # data. - if isinstance(connected_node, bi.BaseIngestor): - logger.debug("Stream from {} to {}" - .format(from_component, connected_node)) - new_dstream = connected_node.map_dstream(dstream) - - # We then connect directly this stream to the orchestrator - self._orchestrator.accumulate_dstream_samples(new_dstream) - # And we look for sink if any - self._connect_dependents_phase1(new_dstream, connected_node) - - # Sink are at the end of the branch! - if isinstance(connected_node, msink.BaseSink): - logger.debug("Sink {} into {}" - .format(from_component, connected_node)) - connected_node.sink_dstream(dstream) - - def _propagate_feature_list(self): - """Set the appropriate features list on each live data processor.""" - for source in self._sources: - features = source.get_feature_list() - for connected_node in self._links[source]: - propagated = False - if isinstance(connected_node, bi.BaseIngestor): - connected_node.set_feature_list(features) - propagated = True - if isinstance(connected_node, mldp.BaseLDP): - connected_node.set_feature_list(features) - propagated = True - if propagated: - logger.info("Feature list {} propagated from {} to {}" - .format(features, source, connected_node)) diff --git a/monasca_analytics/spark/streaming_context.py b/monasca_analytics/spark/streaming_context.py deleted file mode 100644 index 91b8d60..0000000 --- a/monasca_analytics/spark/streaming_context.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import os.path as os_path -import pyspark.sql as sql -import pyspark.streaming as streaming - -logger = logging.getLogger(__name__) - - -class DriverStreamingListener(streaming.StreamingListener): - - @staticmethod - def onBatchCompleted(batchCompleted): - logger.debug("Batch Completed: \n\t{}\n".format( - batchCompleted)) - - @staticmethod - def onBatchStarted(batchStarted): - logger.debug("Batch Started: \n\t{}\n".format( - batchStarted)) - - @staticmethod - def onBatchSubmitted(batchSubmitted): - logger.debug("Batch submitted: \n\t{}\n".format( - batchSubmitted)) - - @staticmethod - def onOutputOperationCompleted(outputOperationCompleted): - logger.debug("Job of batch has completed: \n\t{}\n".format( - outputOperationCompleted)) - - @staticmethod - def onOutputOperationStarted(outputOperationStarted): - logger.debug("Job of a batch has started: \n\t{}\n".format( - outputOperationStarted)) - - @staticmethod - def onReceiverError(receiverError): - logger.warn("Receiver has reported an error: \n\t{}\n".format( - receiverError)) - - @staticmethod - def onReceiverStarted(receiverStarted): - logger.debug("Receiver has been started: \n\t{}\n".format( - receiverStarted)) - - @staticmethod - def onReceiverStopped(receiverStopped): - logger.debug("Receiver has stopped: \n\t{}\n".format( - receiverStopped)) - - @staticmethod - def onStreamingStarted(streamingStarted): - logger.debug("Streaming has been started: \n\t{}\n".format( - streamingStarted)) - - -def create_streaming_context(spark_context, config): - """ - Create a streaming context with a custom Streaming Listener - that will log every event. - :param spark_context: Spark context - :type spark_context: pyspark.SparkContext - :param config: dict - :return: Returns a new streaming context from the given context. - :rtype: pyspark.streaming.StreamingContext - """ - ssc = streaming.StreamingContext(spark_context, config[ - "spark_config"]["streaming"]["batch_interval"]) - ssc.addStreamingListener(DriverStreamingListener) - directory = os_path.expanduser("~/checkpointing") - logger.info("Checkpointing to `{}`".format(directory)) - # Commented out to fix a crash occurring when - # phase 1 is used. The reason of the crash is still unclear - # but Spark complains about the SSC being transferred - # to workers. - # ssc.checkpoint(directory) - return ssc - - -def get_sqlcontext_instance(spark_context): - """ - :type spark_context: pyspark.SparkContext - :param spark_context: The currently active Spark Context - :return: Returns the SQLContext - :rtype: sql.SQLContext - """ - if 'sqlContextSingletonInstance' not in globals(): - globals()['sqlContextSingletonInstance'] = sql.SQLContext( - spark_context) - return globals()['sqlContextSingletonInstance'] diff --git a/monasca_analytics/util/__init__.py b/monasca_analytics/util/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/util/common_util.py b/monasca_analytics/util/common_util.py deleted file mode 100644 index ef34e05..0000000 --- a/monasca_analytics/util/common_util.py +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Common util functions.""" - -import importlib -import inspect -import json -import logging -from logging import config as log_conf -import os -import pkgutil - -from monasca_analytics.config import const -from monasca_analytics.exception import monanas as err -from monasca_analytics import ingestor -from monasca_analytics import ldp -from monasca_analytics import sink -from monasca_analytics import sml -from monasca_analytics import source -from monasca_analytics import voter - -logger = logging.getLogger(__name__) - - -def parse_json_file(filename): - """Parses json and return a dict. - - :type filename: str - :param filename: Filename to be parsed to a dictionary. - :rtype: dict - :returns: Parsed data. - :raises: IOError --If the file does not exist. - :raises: ValueError -- If the file is an invalid json file. - """ - try: - with open(filename, "rt") as f: - return json.load(f) - except (IOError, ValueError) as e: - logger.error("Exception parsing json file : " + str(e)) - raise - - -def setup_logging(filename): - """Setup logging based on a json string. - - :type filename: str - :param filename: Log configuration file. - :raises: IOError -- If the file does not exist. - :raises: ValueError -- If the file is an invalid json file. - """ - try: - config = parse_json_file(filename) - log_conf.dictConfig(config) - logpy4j = logging.getLogger("py4j") - logpy4j.setLevel(logging.ERROR) - logkafka = logging.getLogger("kafka") - logkafka.setLevel(logging.ERROR) - except (IOError, ValueError): - raise - - -def get_available_inherited_classes(pkg, base_class): - """Gets all inherited classes in modules for a given package - - This does not include subpackages. - - :type pkg: str - :param pkg: a package name. - :type base_class: object - :param base_class: a base class. - :rtype: list - :returns: a list of inherited classes. - """ - available_classes = [] - pkg_path = os.path.dirname(pkg.__file__) - - for _, mod_name, _ in pkgutil.iter_modules([pkg_path]): - if not mod_name.startswith("_"): - try: - module = importlib.import_module("{0}.{1}".format(pkg.__name__, - mod_name)) - - for clazz in inspect.getmembers(module, inspect.isclass): - if clazz is not base_class: - if issubclass(clazz[1], base_class) and\ - not inspect.isabstract(clazz[1]) and\ - clazz[1] != base_class: - available_classes.append(clazz[1]) - except Exception as e: - logger.warn(e.__str__()) - - return set(available_classes) - - -def get_available_classes(class_type=None): - """Creates a dictionary containing pipeline available classes of each type - - :type class_type: str | None - :param class_type: if provided, only this type of classes - will be returned - :rtype: dict - :returns: all subclasses keyed by type - """ - _classes = {} - - if not class_type or class_type == const.SOURCES: - from monasca_analytics.source.base import BaseSource - _classes[const.SOURCES] = get_available_inherited_classes(source, - BaseSource) - if not class_type or class_type == const.INGESTORS: - from monasca_analytics.ingestor.base import BaseIngestor - _classes[const.INGESTORS] = \ - get_available_inherited_classes(ingestor, BaseIngestor) - - if not class_type or class_type == const.SMLS: - from monasca_analytics.sml.base import BaseSML - _classes[const.SMLS] = get_available_inherited_classes(sml, BaseSML) - - if not class_type or class_type == const.VOTERS: - from monasca_analytics.voter.base import BaseVoter - _classes[const.VOTERS] = get_available_inherited_classes(voter, - BaseVoter) - if not class_type or class_type == const.SINKS: - from monasca_analytics.sink.base import BaseSink - _classes[const.SINKS] = get_available_inherited_classes(sink, BaseSink) - - if not class_type or class_type == const.LDPS: - from monasca_analytics.ldp.base import BaseLDP - _classes[const.LDPS] = \ - get_available_inherited_classes(ldp, BaseLDP) - - return _classes - - -def get_component_type(class_name): - for cls_type in const.components_types: - names = get_available_class_names(cls_type) - if class_name in names: - return cls_type - - -def get_class_by_name(class_name, class_type=None): - """Gets the class by class name. - - :type class_name: str - :param class_name: a class name to look for - :type class_type: str - :param class_type: the type of the class to look for - (e.g. data_sources, ingestors, etc.). - :returns: class -- the source class requested. - :raises: MonanasNoSuchSourceError -- If no source class found. - :raises: MonanasDuplicateSourceError -- If the system has multiple sources - of the same class name. - """ - classes = get_available_classes(class_type) - if class_type: - clazz = list(filter(lambda t_class: t_class.__name__ == class_name, - classes[class_type])) - else: - for c_type in classes.keys(): - clazz = list(filter(lambda t_class: t_class.__name__ == class_name, - classes[c_type])) - if clazz: - break - if not clazz: - raise err.MonanasNoSuchClassError(class_name) - elif len(clazz) > 1: - raise err.MonanasDuplicateClassError(class_name) - else: - return clazz[0] - - -def get_available_class_names(class_type): - """Gets available class names of type class_type. - - :type class_type: str - :param class_type: type of classes to look for - :rtype: list - :returns: a list of available source class names. - """ - classes = get_available_classes(class_type) - return [Clazz.__name__ for Clazz in classes[class_type]] - - -def get_source_class_by_name(class_name): - """Gets the source class by class name. - - :type class_name: str - :param class_name: name of the source class requested. - :raises: MonanasNoSuchIngestorError -- if no source class found. - :raises: MonanasDuplicateIngestorError -- if the system has multiple - source of the same class name. - """ - return get_class_by_name(class_name, const.SOURCES) - - -def get_available_source_class_names(): - """Gets available source class names. - - :rtype: list - :returns: a list of available source class names. - """ - return get_available_class_names(const.SOURCES) - - -def get_ingestor_class_by_name(class_name): - """Gets the ingestor class by class name. - - :type class_name: str - :param class_name: name of the ingestor class requested. - :raises: MonanasNoSuchIngestorError -- if no ingestor class found. - :raises: MonanasDuplicateIngestorError -- if the system has multiple - ingestors of the same class name. - """ - return get_class_by_name(class_name, const.INGESTORS) - - -def get_available_ingestor_class_names(): - """Gets available ingestor class names. - - :rtype: list - :returns: a list of available ingestor class names. - """ - return get_available_class_names(const.INGESTORS) - - -def get_sml_class_by_name(class_name): - """Gets the sml class by class name. - - :type class_name: str - :param class_name: name of the sml class requested. - :raises: MonanasNoSuchIngestorError -- if no sml class found. - :raises: MonanasDuplicateIngestorError -- if the system has multiple sml - algorithms of the same class - name. - """ - return get_class_by_name(class_name, const.SMLS) - - -def get_available_sml_class_names(): - """Gets available sml class names. - - :rtype: list - :returns: a list of available sml class names. - """ - return get_available_class_names(const.SMLS) - - -def get_voter_class_by_name(class_name): - """Gets the voter class by class name. - - :type class_name: str - :param class_name: name of the voter class requested. - :raises: MonanasNoSuchIngestorError -- If no voter class found. - :raises: MonanasDuplicateIngestorError: If the system has multiple - voter of the same class name. - """ - return get_class_by_name(class_name, const.VOTERS) - - -def get_available_voter_class_names(): - """Gets available voter class names. - - :rtype: list - :returns: a list of available voter class names. - """ - return get_available_class_names(const.VOTERS) - - -def get_available_ldp_class_names(): - """Gets available ldp class names. - - :rtype: list - :returns: a list of available ldp class names. - """ - return get_available_class_names(const.LDPS) diff --git a/monasca_analytics/util/math.py b/monasca_analytics/util/math.py deleted file mode 100644 index 1315514..0000000 --- a/monasca_analytics/util/math.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import bisect - - -def interpolate_1d(table, value): - """Return the interpolated result using the table for the passed value. - - :type table: list[(int, float)] - :param table: a list where keys are numbers - :type value: float | int - :param value: the value we want to compute the result from. - """ - table.sort(key=lambda fn: fn[0]) - keys = [fv[0] for fv in table] - index = bisect.bisect(keys, value) - if index == 0: - return table[index][1] - elif index == len(table): - return table[index - 1][1] - else: - lo = table[index - 1] - hi = table[index] - return (hi[1] - lo[1]) * (value - lo[0]) / (hi[0] - lo[0]) + lo[1] diff --git a/monasca_analytics/util/singleton.py b/monasca_analytics/util/singleton.py deleted file mode 100644 index 51ca2ea..0000000 --- a/monasca_analytics/util/singleton.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class Singleton(type): - """A singleton metaclass.""" - - _instances = {} - - def __call__(self, *args, **kwargs): - if self not in self._instances: - self._instances[self] = super(Singleton, self).__call__(*args, - **kwargs) - return self._instances[self] diff --git a/monasca_analytics/util/spark_func.py b/monasca_analytics/util/spark_func.py deleted file mode 100644 index a8e182c..0000000 --- a/monasca_analytics/util/spark_func.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - - -def from_json(rdd_entry): - return json.loads(rdd_entry) diff --git a/monasca_analytics/util/string_util.py b/monasca_analytics/util/string_util.py deleted file mode 100644 index a9edb4e..0000000 --- a/monasca_analytics/util/string_util.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six - - -def array_to_str(array, multiline=False, indent=None): - """ - Convert the provided dictionary into a readable str, by calling - str on both the keys and the values. - :type array: list - :param array: the dictionary to convert. - :type multiline: bool - :param multiline: If each key value pair should be on its own line. - :type indent: int - :param indent: Indentation if multiline is True. - :rtype: str - :return: Returns the converted dict. - """ - if len(array) == 0: - return "[]" - - multiline = multiline or indent is not None - - def dispatch(value): - if isinstance(value, list): - return array_to_str(value, multiline, indent) - if isinstance(value, dict): - return dict_to_str(value, multiline, indent) - return str(value) - - res = "[" - if multiline: - res += "\n" - join_str = "," - if multiline: - join_str += "\n" - else: - join_str += " " - if indent is not None: - join_str = " " * indent + join_str - res += join_str.join(map(dispatch, array)) - if multiline: - res += "\n" - res += "]" - - return res - - -def dict_to_str(dictionary, multiline=False, indent=None): - """ - Convert the provided dictionary into a readable str, by calling - str on both the keys and the values. - :type dictionary: dict - :param dictionary: the dictionary to convert. - :type multiline: bool - :param multiline: If each key value pair should be on its own line. - :type indent: int - :param indent: Indentation if multiline is True. - :rtype: str - :return: Returns the converted dict. - """ - if len(dictionary) == 0: - return "{}" - res = "{" - if multiline: - res += "\n" - multiline = multiline or indent is not None - for k, v in sorted(six.iteritems(dictionary), key=lambda ke: str(ke[0])): - if indent is not None: - res += " " * indent - if isinstance(v, dict): - res += "{}: {}, ".format(str(k), - dict_to_str(v, multiline, indent)) - elif isinstance(v, list): - res += "{}: {}, ".format(str(k), - array_to_str(v, multiline, indent)) - else: - res += "{}: {}, ".format(str(k), str(v)) - if multiline: - res += '\n' - res = res[0:-2] - res += "}" - return res - - -def stable_repr(obj): - """ - Convert the provided dictionary into a 'repr' str, by calling - repr on both the keys and the values. - :type obj: dict | str | float - :param obj: the dictionary to convert. - :rtype: str - :return: Returns the converted dict. - """ - - if isinstance(obj, list): - return "[" + ", ".join(map(stable_repr, obj)) + "]" - elif not isinstance(obj, dict): - return repr(obj) - - if len(obj) == 0: - return "{}" - - res = "{" - - for k, v in sorted(six.iteritems(obj), key=lambda ke: str(ke[0])): - res += "{}: {}, ".format(repr(k), stable_repr(v)) - - res = res[0:-2] - res += "}" - return res diff --git a/monasca_analytics/util/timestamp.py b/monasca_analytics/util/timestamp.py deleted file mode 100644 index 731d5b1..0000000 --- a/monasca_analytics/util/timestamp.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import time - - -def timestamp(date_time_object): - """ - Returns the timestamp associated with the given - datetime object. - - :type date_time_object: datetime.datetime - :param date_time_object: datetime object that will be converted - :rtype int - :return: Returns the appropriate timestamp - """ - return int(time.mktime(date_time_object.timetuple())) diff --git a/monasca_analytics/util/validation_utils.py b/monasca_analytics/util/validation_utils.py deleted file mode 100644 index e7efd78..0000000 --- a/monasca_analytics/util/validation_utils.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os.path as path -import re - -import voluptuous - -from monasca_analytics.config import const -from monasca_analytics.util import common_util as cu - - -def AvailableSink(msg=None): - - available_sink_classes = cu.get_available_classes(const.SINKS)[const.SINKS] - available_sink_names = [Clazz.__name__ for Clazz in available_sink_classes] - - def f(v): - if str(v) in available_sink_names: - return str(v) - else: - raise voluptuous.Invalid(msg or ("Invalid Sink Name: " + str(v))) - return f - - -def NoSpaceCharacter(msg=None): - def f(v): - if not any(c.isspace() for c in str(v)): - return str(v) - else: - raise voluptuous.Invalid(msg or ( - "White space not allowed in: " + str(v))) - return f - - -def ExistingPath(msg=None): - def f(v): - if path.exists(path.expanduser(str(v))) or\ - path.exists(path.dirname(path.expanduser(str(v)))): - return str(v) - else: - raise voluptuous.Invalid(msg or ( - "Path does not exist: " + str(v))) - return f - - -def NumericString(msg=None): - def f(v): - if re.match("[0-9]+", str(v)): - return str(v) - else: - raise voluptuous.Invalid(msg or ( - "string does not represent a number")) - return f - - -def ValidMarkovGraph(msg=None): - def f(v): - if len(str(v).split(":")) == 2 and str(v).split(":")[1] in\ - ["host", "web_service", "switch"]: - return str(v) - else: - raise voluptuous.Invalid(msg or ( - "Key should be of the form 'anything:host' " - "where 'host' can be replaced " - "by 'web_service' or 'switch'.")) - return f - - -def NotEmptyArray(msg=None): - def f(v): - if len(v) > 0: - return v - else: - raise voluptuous.Invalid(msg or ("empty array: " + str(v))) - return f diff --git a/monasca_analytics/voter/__init__.py b/monasca_analytics/voter/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/voter/base.py b/monasca_analytics/voter/base.py deleted file mode 100644 index f13b8ae..0000000 --- a/monasca_analytics/voter/base.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import logging - -from monasca_analytics.component import base - -logger = logging.getLogger(__name__) - - -class BaseVoter(base.BaseComponent): - - def __init__(self, _id, _config): - super(BaseVoter, self).__init__(_id, _config) - self._ldps = [] - self._smls = [] - self._has_receive_structure_from = set([]) - self._structures = [] - - @abc.abstractmethod - def elect_structure(self, structures): - """Elect a structure or combine them to create a new one - - :type structures: list[numpy.ndarray] - :param structures: the list of structure learned over the samples. - :rtype: numpy.ndarray - :returns: this should returns one of the structure or a combination. - """ - pass - - def suggest_structure(self, who, structure): - """Suggest to this voter the given structure. - - :param who: the sml making the proposal - :param structure: the structure proposed - :rtype: numpy.ndarray - """ - if who not in self._has_receive_structure_from: - self._structures.append(structure) - self._has_receive_structure_from.add(who) - if len(self._has_receive_structure_from) == len(self._smls): - candidate = self.elect_structure(self._structures) - for tr in self._ldps: - tr.set_voter_output(candidate) - # TODO(David): feed the sinks with candidate. - self._has_receive_structure_from = set([]) - return - logger.debug("SML algorithm '{}' already suggested a structure" - .format(who)) - - def append_sml(self, l): - """Append a SML to the list of SMLs - - This method is automatically called by the passed sml - when this voter is connected to it and shouldn't be used directly. - - :type l: monasca_analytics.sml.base.BaseSML - :param l: sml just connected to this. - """ - self._smls.append(l) - - def remove_sml(self, l): - """Remove a SML from the list of SMLs - - This method is automatically called by the passed sml - when this voter is disconnected to it and shouldn't be used - directly. - - :type l: monasca_analytics.sml.base.BaseSML - :param l: sml just disconnected to this. - """ - self._smls.remove(l) - - def append_ldp(self, ldp): - """Add the LDP to the LDPs list""" - self._ldps.append(ldp) - - def remove_ldp(self, ldp): - """Remove the LDP from the LDPs list""" - self._ldps.remove(ldp) diff --git a/monasca_analytics/voter/pick_index.py b/monasca_analytics/voter/pick_index.py deleted file mode 100644 index 604b6ba..0000000 --- a/monasca_analytics/voter/pick_index.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import math -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -from monasca_analytics.voter import base - -import six - - -logger = logging.getLogger(__name__) - - -class PickIndexVoter(base.BaseVoter): - - def __init__(self, _id, _config): - super(PickIndexVoter, self).__init__(_id, _config) - self._index = _config["index"] - - @staticmethod - def validate_config(_config): - pick_schema = voluptuous.Schema({ - "module": voluptuous.And( - six.string_types[0], - lambda i: not any(c.isspace() for c in i)), - "index": voluptuous.And( - voluptuous.Or(float, int), - lambda i: i >= 0 and math.ceil(i) == math.floor(i) - ) - }, required=True) - return pick_schema(_config) - - @staticmethod - def get_default_config(): - return { - "module": PickIndexVoter.__name__, - "index": 0 - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor('index', type_util.Number(), 0), - ] - - def elect_structure(self, structures): - return structures[ - min(len(structures) - 1, - self._index)] diff --git a/monasca_analytics/web_service/__init__.py b/monasca_analytics/web_service/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monasca_analytics/web_service/request_handler.py b/monasca_analytics/web_service/request_handler.py deleted file mode 100644 index 2e10ce2..0000000 --- a/monasca_analytics/web_service/request_handler.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import logging -import sys -import traceback - -import six -from tornado import web -import voluptuous - -import monasca_analytics.banana.emitter as emit -import monasca_analytics.exception.monanas as err -import monasca_analytics.util.common_util as introspect -from monasca_analytics.web_service import web_service_model - - -logger = logging.getLogger(__name__) - - -class MonanasHandler(web.RequestHandler): - """Request handler for WebService.""" - - def initialize(self, monanas): - """Initializes the handler. - - :param monanas: Monanas -- A Monanas's instance. - """ - self._monanas = monanas - - @web.asynchronous - def post(self): - """Performs a Monanas's action.""" - terminate = (False, "") - - try: - body = json.loads(self.request.body) - web_service_model.action_model(body) - getattr(self._monanas, body["action"])() - except (AttributeError, voluptuous.Invalid, ValueError): - self.set_status(400, "The request body was malformed.") - except (err.MonanasBindSourcesError, - err.MonanasAlreadyStartedStreaming, - err.MonanasAlreadyStoppedStreaming) as e: - self.set_status(400, e.__str__()) - except err.MonanasStreamingError as e: - self.set_status(500, e.__str__()) - terminate = (True, e.__str__()) - except Exception as e: - logger.error("Unexpected error: {0}. {1}". - format(sys.exc_info()[0], e)) - self.set_status(500, "Internal server error.") - - self.flush() - self.finish() - - if terminate[0]: - logger.error(terminate[1]) - self._monanas.stop_streaming_and_terminate() - - -class BananaHandler(web.RequestHandler): - """ - Request handler to manage the active config using - the banana configuration language. - """ - - def initialize(self, monanas, typeck_only): - """Initialize the handler. - - :param monanas: A Monana's instance. - """ - self._monanas = monanas - self._typeck_only = typeck_only - - @web.asynchronous - def post(self): - """Performs a Monanas's action.""" - terminate = (False, "") - - try: - body = json.loads(self.request.body) - web_service_model.banana_model(body) - emitter = emit.JsonEmitter() - if self._typeck_only: - self._monanas.typeck_configuration(body["content"], - emitter) - else: - self._monanas.try_change_configuration(body["content"], - emitter) - self.write(emitter.result) - except (AttributeError, voluptuous.Invalid, ValueError) as e: - self.set_status(400, "The request body was malformed.") - except Exception as e: - tb = traceback.format_exc() - print(tb) - logger.error("Unexpected error: {0}. {1}". - format(sys.exc_info()[0], e)) - self.set_status(500, "Internal server error.") - - self.flush() - self.finish() - - if terminate[0]: - logger.error(terminate[1]) - self._monanas.stop_streaming_and_terminate() - - -class BananaMetaDataHandler(web.RequestHandler): - - def initialize(self, monanas): - """Initializes the handler. - - :param monanas: Monanas -- A Monanas's instance. - """ - self._monanas = monanas - - @web.asynchronous - def get(self): - all_components = introspect.get_available_classes() - result = {"components": []} - for kind, components in six.iteritems(all_components): - for component in components: - result["components"].append({ - "name": component.__name__, - "description": component.__doc__, - "params": map(lambda x: x.to_json(), - component.get_params()), - }) - self.write(result) - self.flush() - self.finish() - - @web.asynchronous - def post(self): - - try: - body = json.loads(self.request.body) - web_service_model.banana_model(body) - type_table = self._monanas.compute_type_table(body["content"]) - self.write(type_table) - except (AttributeError, voluptuous.Invalid, ValueError) as e: - logger.warn("Wrong request: {}.". - format(e)) - self.set_status(400, "The request body was malformed.") - except Exception as e: - tb = traceback.format_exc() - print(tb) - logger.error("Unexpected error: {}. {}". - format(sys.exc_info()[0], e)) - self.set_status(500, "Internal server error.") - - self.flush() - self.finish() diff --git a/monasca_analytics/web_service/web_service.py b/monasca_analytics/web_service/web_service.py deleted file mode 100644 index 9de9c6b..0000000 --- a/monasca_analytics/web_service/web_service.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from tornado import web - -from monasca_analytics.web_service import request_handler - - -class WebService(web.Application): - """WebService serving REST API for Monanas.""" - - def __init__(self, monanas, config): - """WebService constructor.""" - self._monanas = monanas - self._config = config - params = {"monanas": self._monanas} - handlers = [ - (r"/", request_handler.MonanasHandler, params), - (r"/banana", request_handler.BananaHandler, { - "monanas": self._monanas, - "typeck_only": False - }), - (r"/banana/typeck", request_handler.BananaHandler, { - "monanas": self._monanas, - "typeck_only": True - }), - (r"/banana/metadata", request_handler.BananaMetaDataHandler, { - "monanas": self._monanas, - }) - ] - - settings = {} - - web.Application.__init__(self, - handlers, - debug=config["debug"], - **settings) diff --git a/monasca_analytics/web_service/web_service_model.py b/monasca_analytics/web_service/web_service_model.py deleted file mode 100644 index 7028756..0000000 --- a/monasca_analytics/web_service/web_service_model.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A list of functions to validate web_service models.""" - -import voluptuous - - -import six - - -def action_model(value): - """Validates the data against action_model schema.""" - action_model_schema = voluptuous.Schema({ - "action": voluptuous.And(six.string_types[0], - lambda o: not o.startswith("_")) - }, required=True) - - return action_model_schema(value) - - -def banana_model(value): - """Validates the data against the banana_model schema.""" - banana_model_schema = voluptuous.Schema({ - "content": six.string_types[0] - }, required=True) - - return banana_model_schema(value) diff --git a/releasenotes/notes/drop-py-2-7-e99c2ab58c9d978a.yaml b/releasenotes/notes/drop-py-2-7-e99c2ab58c9d978a.yaml deleted file mode 100644 index 5f4b0cb..0000000 --- a/releasenotes/notes/drop-py-2-7-e99c2ab58c9d978a.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -upgrade: - - | - Python 2.7 support has been dropped. Last release of monasca-analytics - to support python 2.7 is OpenStack Train. The minimum version of Python now - supported by monasca-analytics is Python 3.6. diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b40c4cd..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -numpy -scipy < 1.2.0 -tornado < 6.0 -scikit-learn -kafka-python -pyparsing -voluptuous>=0.8.9 # BSD License diff --git a/run.py b/run.py deleted file mode 100644 index a1a9068..0000000 --- a/run.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Monanas Runner. - -This script checks for appropriate arguments and starts Monanas to use -data coming from one or more given sources. The source(s) can be configured -using the optional argument --sources. However, a default source using random -data generator is provided in the config folder. -""" - -import json -import logging -import logging.config as log_conf -import os -import subprocess -import sys - -import argparse -import textwrap - -import setup_property - - -class RunnerError(Exception): - def __init__(self, value): - self._value = value - - def __str__(self): - return repr(self._value) - - -def main(arguments): - spark_submit = "{0}/bin/spark-submit".format(arguments.spark_path) - monanas_path = os.environ.get('MONANAS_HOME', "") - kafka_jar = None - - try: - for filename in os.listdir("{0}/external/kafka-assembly/target". - format(arguments.spark_path)): - if filename.startswith("spark-streaming-kafka-assembly") and\ - not any(s in filename for s in ["source", "test"]): - kafka_jar = filename - break - - if not kafka_jar: - raise OSError("Spark's external library required does not exist.") - except OSError as e: - raise RunnerError(e.__str__()) - - spark_kafka_jar = "{0}/external/kafka-assembly/target/{1}".\ - format(arguments.spark_path, kafka_jar) - command = [ - spark_submit, "--master", "local[2]", - "--jars", spark_kafka_jar, monanas_path + "/monasca_analytics/monanas.py", - arguments.config, arguments.log_config - ] - - if arguments.sources is not None: - command += arguments.sources - - try: - logger.info("Executing `{}`...".format(" ".join(command))) - subprocess.Popen(command).communicate() - except OSError as e: - raise RunnerError(e.__str__()) - - -def setup_logging(filename): - """Setup logging based on a json string.""" - with open(filename, "rt") as f: - config = json.load(f) - - log_conf.dictConfig(config) - -def setup_parser(): - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent(__doc__.strip()), - add_help=False) - - parser.add_argument('-c', '--config', - help='Config file.', required=True) - # "-d" currently unused - parser.add_argument('-d', '--debug', - help='Show debug messages.', action='store_true') - parser.add_argument('-h', '--help', - help='Show this screen.', action='help') - parser.add_argument('-l', '--log_config', - help='Log config file\'s path.', required=True) - parser.add_argument('-p', '--spark_path', - help='Spark\'s path.', required=True) - parser.add_argument('-s', '--sources', - help='A list of data sources.', nargs='*') - parser.add_argument('-v', '--version', - help='Show version.', action='version', - version=setup_property.VERSION) - - return parser - -if __name__ == "__main__": - arguments = setup_parser().parse_args() - - try: - setup_logging(arguments.log_config) - except IOError: - raise RunnerError("File not found: {0}.". - format(arguments.log_config)) - except ValueError: - raise RunnerError("{0} is not a valid logging config file.". - format(arguments.log_config)) - - logger = logging.getLogger(__name__) - - try: - main(arguments) - except KeyboardInterrupt: - logger.info("Monanas run script stopped.") - except RunnerError as e: - logger.error(e.__str__()) - except Exception as e: - logger.error("Unexpected error: {0}. {1}.". - format(sys.exc_info()[0], e)) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 513f837..0000000 --- a/setup.cfg +++ /dev/null @@ -1,41 +0,0 @@ -[metadata] -name = monanas -#name = monasca-analytics - -summary = Monanas - Monasca Analytics -description-file = - README.md -author = OpenStack -author-email = openstack-discuss@lists.openstack.org -home-page = https://launchpad.net/monasca -classifier = - Environment :: OpenStack - Intended Audience :: Information Technology - Intended Audience :: System Administrators - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - -[files] -packages = - monasca_analytics - test - -#data_files = -# etc/monasca = -# etc/monasca/analytics-config.conf -# etc/monasca/analytics-config.ini - -[entry_points] -#console_scripts = -# monasca-analytics = monasca_analytics.server:launch -# -#tempest.test_plugins = -# monasca_analytics_tests = monasca_analytics_tests.plugin:MonascaAnalyticsTempestPlugin - -[pbr] -warnerrors = True - diff --git a/setup.py b/setup.py deleted file mode 100644 index bfcb5e5..0000000 --- a/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Monanas setup script. -""" - -from setuptools import setup - -setup( - setup_requires=['pbr>=1.8'], - pbr=True, -) diff --git a/setup_property.py b/setup_property.py deleted file mode 100644 index 7b02f74..0000000 --- a/setup_property.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Setup property for the Machine Learning Framework.""" - -VERSION = "0.1" diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index b7bf5b4..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# mock object framework -coverage>=3.6 # Apache-2.0 -hacking>=1.1.0,<1.2.0 -flake8>=2.5.4,<=3.5.0 # MIT -nose==1.3.0 -mock>=1.0.1 diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/aggregator/__init__.py b/test/aggregator/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/banana/__init__.py b/test/banana/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/banana/deadpathck/__init__.py b/test/banana/deadpathck/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/banana/deadpathck/test_at_least_one_sink_or_source.py b/test/banana/deadpathck/test_at_least_one_sink_or_source.py deleted file mode 100644 index a89ef46..0000000 --- a/test/banana/deadpathck/test_at_least_one_sink_or_source.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import monasca_analytics.banana.deadpathck.config as deadpathck -import monasca_analytics.banana.grammar.config as grammar -import monasca_analytics.banana.typeck.config as typeck -import monasca_analytics.exception.banana as exception - -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class PassOneSinkSourceTestCase(MonanasTestCase): - - def setUp(self): - super(PassOneSinkSourceTestCase, self).setUp() - - def tearDown(self): - super(PassOneSinkSourceTestCase, self).tearDown() - - def test_banana_should_fail_when_no_source(self): - banana_str = "" +\ - "a = CloudMarkovChainSource()\n" +\ - "b = StdoutSink()\n" +\ - "c = CloudIngestor()\n" +\ - "d = LiNGAM()\n" +\ - "c -> d -> b" - # Convert the grammar into an AST - parser = grammar.banana_grammar() - ast = parser.parse(banana_str) - # Compute the type table for the given AST - type_table = typeck.typeck(ast) - # Remove from the tree path that are "dead" - deadpathck.deadpathck(ast, type_table) - self.assertRaises( - exception.BananaNoFullPath, - deadpathck.contains_at_least_one_path_to_a_sink, - ast, - type_table - ) - - def test_banana_should_fail_when_no_sink(self): - banana_str = "" +\ - "a = CloudMarkovChainSource()\n" +\ - "b = StdoutSink()\n" +\ - "c = CloudIngestor()\n" +\ - "d = LiNGAM()\n" +\ - "a -> c -> d" - # Convert the grammar into an AST - parser = grammar.banana_grammar() - ast = parser.parse(banana_str) - # Compute the type table for the given AST - type_table = typeck.typeck(ast) - # Remove from the tree path that are "dead" - deadpathck.deadpathck(ast, type_table) - self.assertRaises( - exception.BananaNoFullPath, - deadpathck.contains_at_least_one_path_to_a_sink, - ast, - type_table - ) - - def test_banana_should_pass_when_more_source_sink(self): - banana_str = "" +\ - "a = CloudMarkovChainSource()\n" +\ - "b = StdoutSink()\n" +\ - "c = CloudIngestor()\n" +\ - "d = LiNGAM()\n" +\ - "a -> c -> d -> b" - # Convert the grammar into an AST - parser = grammar.banana_grammar() - ast = parser.parse(banana_str) - # Compute the type table for the given AST - type_table = typeck.typeck(ast) - # Remove from the tree path that are "dead" - deadpathck.deadpathck(ast, type_table) - deadpathck.contains_at_least_one_path_to_a_sink(ast, type_table) - # We should reach this line. - self.assertTrue(True) diff --git a/test/banana/deadpathck/test_deadpathck.py b/test/banana/deadpathck/test_deadpathck.py deleted file mode 100644 index 04e52af..0000000 --- a/test/banana/deadpathck/test_deadpathck.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import monasca_analytics.banana.deadpathck.config as deadpathck -import monasca_analytics.banana.emitter as emit -import monasca_analytics.banana.grammar.config as grammar -import monasca_analytics.banana.typeck.config as typeck - -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class DeadPathTestCase(MonanasTestCase): - - def setUp(self): - super(DeadPathTestCase, self).setUp() - - def tearDown(self): - super(DeadPathTestCase, self).tearDown() - - def test_banana_should_remove_everything(self): - banana_str = "" +\ - "a = CloudMarkovChainSource()\n" +\ - "b = StdoutSink()\n" +\ - "c = CloudIngestor()\n" +\ - "d = LiNGAM()\n" +\ - "a -> c -> d" - emitter = CustomEmitter() - # Convert the grammar into an AST - parser = grammar.banana_grammar(emitter) - ast = parser.parse(banana_str) - # Compute the type table for the given AST - type_table = typeck.typeck(ast) - # Remove from the tree path that are "dead" - deadpathck.deadpathck(ast, type_table, emitter) - self.assertEqual(emitter.nb_errors, 0) - self.assertEqual(emitter.nb_warnings, 4) - self.assertEqual(len(ast.components), 0) - self.assertEqual(len(list(ast.connections.connections)), 0) - - def test_banana_should_remove_one(self): - banana_str = "" +\ - "a = CloudMarkovChainSource()\n" +\ - "b = StdoutSink()\n" +\ - "c = CloudIngestor()\n" +\ - "d = LiNGAM()\n" +\ - "a -> c -> [d, b]" - emitter = CustomEmitter() - # Convert the grammar into an AST - parser = grammar.banana_grammar(emitter) - ast = parser.parse(banana_str) - # Compute the type table for the given AST - type_table = typeck.typeck(ast) - # Remove from the tree path that are "dead" - deadpathck.deadpathck(ast, type_table, emitter) - self.assertEqual(emitter.nb_errors, 0) - self.assertEqual(emitter.nb_warnings, 1) - self.assertEqual(len(ast.components), 3) - self.assertEqual(len(list(ast.connections.connections)), 2) - - def test_banana_should_not_remove_anything(self): - banana_str = "" +\ - "a = CloudMarkovChainSource()\n" +\ - "b = StdoutSink()\n" +\ - "c = CloudIngestor()\n" +\ - "d = LiNGAM()\n" +\ - "a -> c -> d -> b" - emitter = CustomEmitter() - # Convert the grammar into an AST - parser = grammar.banana_grammar(emitter) - ast = parser.parse(banana_str) - # Compute the type table for the given AST - type_table = typeck.typeck(ast) - # Remove from the tree path that are "dead" - deadpathck.deadpathck(ast, type_table, emitter) - self.assertEqual(emitter.nb_errors, 0) - self.assertEqual(emitter.nb_warnings, 0) - self.assertEqual(len(ast.components), 4) - self.assertEqual(len(list(ast.connections.connections)), 3) - - -class CustomEmitter(emit.Emitter): - - def __init__(self): - super(CustomEmitter, self).__init__() - self.nb_warnings = 0 - self.nb_errors = 0 - - def emit_warning(self, span, message): - print(span.get_line(), str(span), message) - self.nb_warnings += 1 - - def emit_error(self, span, message): - print(span.get_line(), str(span), message) - self.nb_errors += 1 diff --git a/test/banana/eval/__init__.py b/test/banana/eval/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/banana/eval/should_pass/eval_components_only/cloud_markov_chain_source.banana b/test/banana/eval/should_pass/eval_components_only/cloud_markov_chain_source.banana deleted file mode 100644 index e725ac9..0000000 --- a/test/banana/eval/should_pass/eval_components_only/cloud_markov_chain_source.banana +++ /dev/null @@ -1,3 +0,0 @@ -a = CloudMarkovChainSource() # LHS_EQ IGNORE -c = a.sleep # LHS_EQ 0.01 -c = a.transitions.web_service."run=>slow"."0" # LHS_EQ 0.001 \ No newline at end of file diff --git a/test/banana/eval/should_pass/eval_each_stmt/calc_expressions.banana b/test/banana/eval/should_pass/eval_each_stmt/calc_expressions.banana deleted file mode 100644 index 97596da..0000000 --- a/test/banana/eval/should_pass/eval_each_stmt/calc_expressions.banana +++ /dev/null @@ -1,8 +0,0 @@ -a = 23 # LHS_EQ 23.0 -a = a - a # LHS_EQ 0.0 -a = 12 - 12 - 12 # LHS_EQ -12.0 -a = 3 * 4 - 2 # LHS_EQ 10.0 -a = 2 - 3 * 4 # LHS_EQ -10.0 -b = 21 # LHS_EQ 21.0 -d = 12 # LHS_EQ 12.0 -e = b * (d + a) # LHS_EQ 42.0 diff --git a/test/banana/eval/should_pass/eval_each_stmt/json_expressions.banana b/test/banana/eval/should_pass/eval_each_stmt/json_expressions.banana deleted file mode 100644 index 59e635b..0000000 --- a/test/banana/eval/should_pass/eval_each_stmt/json_expressions.banana +++ /dev/null @@ -1,4 +0,0 @@ -a = {a.b.c: 12} # LHS_EQ {'a': {'b': {'c': 12.0}}} -b = a.a.b.c # LHS_EQ 12.0 -a = {a.b: 21, a.c: "test"} # LHS_EQ {'a': {'b': 21.0, 'c': 'test'}} -d = a.a.c + " " + a.a.b # LHS_EQ 'test 21.0' \ No newline at end of file diff --git a/test/banana/eval/test_eval_config.py b/test/banana/eval/test_eval_config.py deleted file mode 100644 index 9e5c285..0000000 --- a/test/banana/eval/test_eval_config.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os -import re - -from monasca_analytics.banana.pass_manager import compute_evaluation_context -from monasca_analytics.util.string_util import stable_repr -from test.util_for_testing import MonanasTestCase - - -logger = logging.getLogger(__name__) - -_has_some_file_that_should = dict() -_has_some_file_that_should["pass_eval_stmt"] = False - - -class EvalTestCase(MonanasTestCase): - - def setUp(self): - super(EvalTestCase, self).setUp() - - def tearDown(self): - super(EvalTestCase, self).tearDown() - - def test_files_have_been_found(self): - self.assertTrue(_has_some_file_that_should["pass_eval_stmt"]) - - -def upgrade_test_case(): - regex_var_eq = re.compile("#(?: )*LHS_EQ(?: )+([^\n]+)") - - for root, dirs, files in os.walk('./banana/eval/should_pass/'): - for filename in files: - name_no_ext, _ = os.path.splitext(filename) - _has_some_file_that_should["pass_eval_stmt"] = True - with open(os.path.join(root, filename), 'r') as f: - content = f.read() - - expected_values = regex_var_eq.findall(content) - - def create_test(test_str, expect_values): - def should_pass(self): - # Custom checks runned after each statement - box = {"counter": 0} - - def custom_check(ctx, stmt_type, lhs_node, rhs_value): - if expect_values[box["counter"]] != "IGNORE": - self.assertEqual( - expect_values[box["counter"]], - stable_repr(rhs_value)) - box["counter"] += 1 - # Evaluate the file. - compute_evaluation_context(test_str, custom_check) - - should_pass.__name__ = "test_banana_eval_" + name_no_ext - return should_pass - - setattr(EvalTestCase, "test_banana_eval_" + name_no_ext, - create_test(content, expected_values)) - -# Fill the test case with generated test case from banana files -upgrade_test_case() diff --git a/test/banana/grammar/__init__.py b/test/banana/grammar/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/banana/grammar/should_fail/component_cant_be_in_object.banana b/test/banana/grammar/should_fail/component_cant_be_in_object.banana deleted file mode 100644 index 74899f5..0000000 --- a/test/banana/grammar/should_fail/component_cant_be_in_object.banana +++ /dev/null @@ -1,2 +0,0 @@ -a = { b: MyComponent() } -# RAISE ParseSyntaxException \ No newline at end of file diff --git a/test/banana/grammar/should_fail/json_obj_props_cant_shadow.banana b/test/banana/grammar/should_fail/json_obj_props_cant_shadow.banana deleted file mode 100644 index 973c008..0000000 --- a/test/banana/grammar/should_fail/json_obj_props_cant_shadow.banana +++ /dev/null @@ -1,4 +0,0 @@ -# This one shadow the a.b, so this is still an error. -a = { a.b: "test", a.b: 23 } - -# RAISE ParseSyntaxException \ No newline at end of file diff --git a/test/banana/grammar/should_fail/wrong_symbol_in_connection.banana b/test/banana/grammar/should_fail/wrong_symbol_in_connection.banana deleted file mode 100644 index 27977ab..0000000 --- a/test/banana/grammar/should_fail/wrong_symbol_in_connection.banana +++ /dev/null @@ -1,3 +0,0 @@ -# This should fail -a -> b * c -# RAISE ParseException \ No newline at end of file diff --git a/test/banana/grammar/should_fail/wrong_symbol_in_expr.banana b/test/banana/grammar/should_fail/wrong_symbol_in_expr.banana deleted file mode 100644 index 42a5362..0000000 --- a/test/banana/grammar/should_fail/wrong_symbol_in_expr.banana +++ /dev/null @@ -1,2 +0,0 @@ -a = b -> c -# RAISE ParseException \ No newline at end of file diff --git a/test/banana/grammar/should_pass/component_creation.banana b/test/banana/grammar/should_pass/component_creation.banana deleted file mode 100644 index 3e261ed..0000000 --- a/test/banana/grammar/should_pass/component_creation.banana +++ /dev/null @@ -1,12 +0,0 @@ -# Creating some components -a = Test() -b = Test(param={}) -c = Test(param="test") -d = Test(param=234) - -# Assigning parameters -b.param = {} -c.param = "test" -d.param = 234 - -# STMT_EQ { DotPath< Ident< a > > = Component { type_name: Ident< Test >, args: []}, DotPath< Ident< b > > = Component { type_name: Ident< Test >, args: [Ident< param > = JsonObj< {} >]}, DotPath< Ident< c > > = Component { type_name: Ident< Test >, args: [Ident< param > = Expr< [StringLit< "test" >] >]}, DotPath< Ident< d > > = Component { type_name: Ident< Test >, args: [Ident< param > = Expr< [Number< 234.0 >] >]}, DotPath< Ident< b >.Ident< param > > = JsonObj< {} >, DotPath< Ident< c >.Ident< param > > = Expr< [StringLit< "test" >] >, DotPath< Ident< d >.Ident< param > > = Expr< [Number< 234.0 >] > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/connection.banana b/test/banana/grammar/should_pass/connection.banana deleted file mode 100644 index e4a02f9..0000000 --- a/test/banana/grammar/should_pass/connection.banana +++ /dev/null @@ -1,13 +0,0 @@ -# This example would not pass later pass -# but exercise the grammar - -a = Test() -b = Test() - -a -> [b, c] -a -> b -> c -a -> [a -> b, b] -> e -a -> b -[[a]] -> b - -# CONN_EQ Connection< [('Ident< a >', 'Ident< b >'), ('Ident< a >', 'Ident< c >'), ('Ident< b >', 'Ident< c >'), ('Ident< a >', 'Ident< a >'), ('Ident< b >', 'Ident< e >')] > \ No newline at end of file diff --git a/test/banana/grammar/should_pass/dot_accesses.banana b/test/banana/grammar/should_pass/dot_accesses.banana deleted file mode 100644 index 1e1430f..0000000 --- a/test/banana/grammar/should_pass/dot_accesses.banana +++ /dev/null @@ -1,10 +0,0 @@ -a = {} -a.param1 = 23 -a."param3" = 1 -a."param4"."para mmm" = a."param3" -a.param2 = { - "a": 34 + 12, - a.b.c: a.param1 -} - -# STMT_EQ { DotPath< Ident< a > > = JsonObj< {} >, DotPath< Ident< a >.Ident< param1 > > = Expr< [Number< 23.0 >] >, DotPath< Ident< a >.StringLit< "param3" > > = Expr< [Number< 1.0 >] >, DotPath< Ident< a >.StringLit< "param4" >.StringLit< "para mmm" > > = Expr< [DotPath< Ident< a >.StringLit< "param3" > >] >, DotPath< Ident< a >.Ident< param2 > > = JsonObj< {DotPath< Ident< a >.Ident< b >.Ident< c > >: Expr< [DotPath< Ident< a >.Ident< param1 > >] >, StringLit< "a" >: Expr< [Number< 34.0 >, +, Number< 12.0 >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/eval_json_obj.banana b/test/banana/grammar/should_pass/eval_json_obj.banana deleted file mode 100644 index 55d05ca..0000000 --- a/test/banana/grammar/should_pass/eval_json_obj.banana +++ /dev/null @@ -1,6 +0,0 @@ -a = { a.b: "test" } -c = { e: a } -# VAR a EQ_JSON { "a": {"b": "test"} } -# VAR c EQ_JSON { "e": {"a": {"b": "test"}} } - -# STMT_EQ { DotPath< Ident< a > > = JsonObj< {DotPath< Ident< a >.Ident< b > >: Expr< [StringLit< "test" >] >} >, DotPath< Ident< c > > = JsonObj< {DotPath< Ident< e > >: Expr< [DotPath< Ident< a > >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/eval_simple_expression.banana b/test/banana/grammar/should_pass/eval_simple_expression.banana deleted file mode 100644 index ea8082c..0000000 --- a/test/banana/grammar/should_pass/eval_simple_expression.banana +++ /dev/null @@ -1,8 +0,0 @@ -a = 23 + 2 -b = "test " + a -c = a + 4.0 * 2 -# VAR a EQ_NUM 25 -# VAR b EQ_STR "test 25" -# VAR c EQ_NUM 33 - -# STMT_EQ { DotPath< Ident< a > > = Expr< [Number< 23.0 >, +, Number< 2.0 >] >, DotPath< Ident< b > > = Expr< [StringLit< "test " >, +, DotPath< Ident< a > >] >, DotPath< Ident< c > > = Expr< [DotPath< Ident< a > >, +, Expr< [Number< 4.0 >, *, Number< 2.0 >] >] > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/full_example.banana b/test/banana/grammar/should_pass/full_example.banana deleted file mode 100644 index 706164c..0000000 --- a/test/banana/grammar/should_pass/full_example.banana +++ /dev/null @@ -1,14 +0,0 @@ -# Connections -a -> b - -# Some vars -d = 23 -e = d - 23 -c = 23 + 1.5 * 3 - d - -# Components -a = Test(a=23, b=c+10) -b = Test(23, 10) - -# STMT_EQ { DotPath< Ident< d > > = Expr< [Number< 23.0 >] >, DotPath< Ident< e > > = Expr< [DotPath< Ident< d > >, -, Number< 23.0 >] >, DotPath< Ident< c > > = Expr< [Number< 23.0 >, +, Expr< [Number< 1.5 >, *, Number< 3.0 >] >, -, DotPath< Ident< d > >] >, DotPath< Ident< a > > = Component { type_name: Ident< Test >, args: [Ident< a > = Expr< [Number< 23.0 >] >, Ident< b > = Expr< [DotPath< Ident< c > >, +, Number< 10.0 >] >]}, DotPath< Ident< b > > = Component { type_name: Ident< Test >, args: [Expr< [Number< 23.0 >] >, Expr< [Number< 10.0 >] >]} } -# CONN_EQ Connection< [('Ident< a >', 'Ident< b >')] > \ No newline at end of file diff --git a/test/banana/grammar/should_pass/json_obj.banana b/test/banana/grammar/should_pass/json_obj.banana deleted file mode 100644 index 562449f..0000000 --- a/test/banana/grammar/should_pass/json_obj.banana +++ /dev/null @@ -1,8 +0,0 @@ -# Various check for edge cases -a = {} -b = { a: {} } -c = { "a": {} } -d = { "": {} } -e = { "": "" } - -# STMT_EQ { DotPath< Ident< a > > = JsonObj< {} >, DotPath< Ident< b > > = JsonObj< {DotPath< Ident< a > >: JsonObj< {} >} >, DotPath< Ident< c > > = JsonObj< {StringLit< "a" >: JsonObj< {} >} >, DotPath< Ident< d > > = JsonObj< {StringLit< "" >: JsonObj< {} >} >, DotPath< Ident< e > > = JsonObj< {StringLit< "" >: Expr< [StringLit< "" >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/jsonlike_objects.banana b/test/banana/grammar/should_pass/jsonlike_objects.banana deleted file mode 100644 index de15920..0000000 --- a/test/banana/grammar/should_pass/jsonlike_objects.banana +++ /dev/null @@ -1,6 +0,0 @@ -a = {} -b = { "a": a } -c = { a.b: "b" } -d = { a.v."t t": 12 + 23 } - -# STMT_EQ { DotPath< Ident< a > > = JsonObj< {} >, DotPath< Ident< b > > = JsonObj< {StringLit< "a" >: Expr< [DotPath< Ident< a > >] >} >, DotPath< Ident< c > > = JsonObj< {DotPath< Ident< a >.Ident< b > >: Expr< [StringLit< "b" >] >} >, DotPath< Ident< d > > = JsonObj< {DotPath< Ident< a >.Ident< v >.StringLit< "t t" > >: Expr< [Number< 12.0 >, +, Number< 23.0 >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/scary_json_ops.banana b/test/banana/grammar/should_pass/scary_json_ops.banana deleted file mode 100644 index 2621d07..0000000 --- a/test/banana/grammar/should_pass/scary_json_ops.banana +++ /dev/null @@ -1,4 +0,0 @@ -a = { b.c: 21 } -b = { a: { a.b: { a.b.c: 10 }}, b.c: 20 } - -# STMT_EQ { DotPath< Ident< a > > = JsonObj< {DotPath< Ident< b >.Ident< c > >: Expr< [Number< 21.0 >] >} >, DotPath< Ident< b > > = JsonObj< {DotPath< Ident< a > >: JsonObj< {DotPath< Ident< a >.Ident< b > >: JsonObj< {DotPath< Ident< a >.Ident< b >.Ident< c > >: Expr< [Number< 10.0 >] >} >} >, DotPath< Ident< b >.Ident< c > >: Expr< [Number< 20.0 >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/valid_expression.banana b/test/banana/grammar/should_pass/valid_expression.banana deleted file mode 100644 index 3c262f4..0000000 --- a/test/banana/grammar/should_pass/valid_expression.banana +++ /dev/null @@ -1,5 +0,0 @@ -c = SomeComponent() -b = 12 + c.param1."foo bar constant" -a = b * 12 - -# STMT_EQ { DotPath< Ident< c > > = Component { type_name: Ident< SomeComponent >, args: []}, DotPath< Ident< b > > = Expr< [Number< 12.0 >, +, DotPath< Ident< c >.Ident< param1 >.StringLit< "foo bar constant" > >] >, DotPath< Ident< a > > = Expr< [DotPath< Ident< b > >, *, Number< 12.0 >] > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/various_literal_assignments.banana b/test/banana/grammar/should_pass/various_literal_assignments.banana deleted file mode 100644 index a82c461..0000000 --- a/test/banana/grammar/should_pass/various_literal_assignments.banana +++ /dev/null @@ -1,5 +0,0 @@ -a = "some string" -b = 12.0 -c = {"a": b} - -# STMT_EQ { DotPath< Ident< a > > = Expr< [StringLit< "some string" >] >, DotPath< Ident< b > > = Expr< [Number< 12.0 >] >, DotPath< Ident< c > > = JsonObj< {StringLit< "a" >: Expr< [DotPath< Ident< b > >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/test_config.py b/test/banana/grammar/test_config.py deleted file mode 100644 index 3fa49ae..0000000 --- a/test/banana/grammar/test_config.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os -import pyparsing as p -import re - -from monasca_analytics.banana.grammar.config import banana_grammar -from test.util_for_testing import MonanasTestCase - - -logger = logging.getLogger(__name__) - -_has_some_file_that_should = dict() -_has_some_file_that_should["pass"] = False -_has_some_file_that_should["fail"] = False - - -class GrammarTestCase(MonanasTestCase): - - def setUp(self): - super(GrammarTestCase, self).setUp() - - def tearDown(self): - super(GrammarTestCase, self).tearDown() - - def test_files_have_been_found(self): - self.assertTrue(_has_some_file_that_should["pass"]) - self.assertTrue(_has_some_file_that_should["fail"]) - - -def upgrade_test_case(): - grammar = banana_grammar() - regex_raise = re.compile("#(?: )*RAISE(?: )*([^\n]+)") - regex_ast_eq = re.compile("#(?: )*AST_EQ(?: )(?: )*([^\n]+)") - regex_stmt_eq = re.compile("#(?: )*STMT_EQ(?: )(?: )*([^\n]+)") - regex_conn_eq = re.compile("#(?: )*CONN_EQ(?: )(?: )*([^\n]+)") - - for root, dirs, files in os.walk('./banana/grammar/should_pass'): - for filename in files: - name_no_ext, _ = os.path.splitext(filename) - _has_some_file_that_should["pass"] = True - with open(os.path.join(root, filename), 'r') as f: - content = f.read() - - expected_ast = regex_ast_eq.search(content) - if expected_ast is not None: - expected_ast = expected_ast.group(1) - expected_stmt = regex_stmt_eq.search(content) - if expected_stmt is not None: - expected_stmt = expected_stmt.group(1) - expected_conn = regex_conn_eq.search(content) - if expected_conn is not None: - expected_conn = expected_conn.group(1) - - def create_test(test_str, expect_ast, expect_stmt, exp_conn): - def should_pass(self): - tree = grammar.parse(test_str) - if expect_ast is not None: - self.assertEqual(str(tree), - expect_ast) - if expect_stmt is not None: - self.assertEqual(tree.statements_to_str(), - expect_stmt) - if exp_conn is not None: - self.assertEqual(str(tree.connections), - exp_conn) - if exp_conn is None and expect_ast is None and\ - expect_stmt is None: - raise Exception("Expected at least one check!") - - should_pass.__name__ = "test_banana_pass_" + name_no_ext - return should_pass - - setattr(GrammarTestCase, "test_banana_pass_" + name_no_ext, - create_test(content, expected_ast, expected_stmt, - expected_conn)) - - for root, dirs, files in os.walk('./banana/grammar/should_fail'): - for filename in files: - name_no_ext, _ = os.path.splitext(filename) - _has_some_file_that_should["fail"] = True - with open(os.path.join(root, filename), 'r') as f: - content = f.read() - - def create_test(test_str): - def should_fail(self): - expected_error = regex_raise.search(test_str).group(1) - expected_exception = get_exception_from_str( - expected_error) - self.assertRaises( - expected_exception, - grammar.parse, - test_str) - should_fail.__name__ = "test_banana_fail_" + name_no_ext - return should_fail - - setattr(GrammarTestCase, "test_banana_fail_" + name_no_ext, - create_test(content)) - -# Fill the test case with generated test case from banana files -upgrade_test_case() - - -def get_exception_from_str(string): - if string == p.ParseSyntaxException.__name__: - return p.ParseSyntaxException - if string == p.ParseFatalException.__name__: - return p.ParseFatalException - if string == p.ParseException.__name__: - return p.ParseException - raise Exception("Invalid exception name: '{}'".format(string)) diff --git a/test/banana/test_api.py b/test/banana/test_api.py deleted file mode 100644 index 323e795..0000000 --- a/test/banana/test_api.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.exception.banana import BananaEnvironmentError -from monasca_analytics.exception.banana import BananaInvalidExpression -from monasca_analytics.parsing.api import validate_environment -from monasca_analytics.parsing.api import validate_expression -from monasca_analytics.parsing.api import validate_name_binding -from test.util_for_testing import MonanasTestCase - - -class TestBananaAPI(MonanasTestCase): - - def setUp(self): - super(TestBananaAPI, self).setUp() - - def tearDown(self): - super(TestBananaAPI, self).tearDown() - - def test_validate_expression_is_valid(self): - validate_expression("a + b") - validate_expression("a * b") - validate_expression("a - b") - validate_expression("a / b") - validate_expression("a / b + 12 * (1 - a)") - - def test_validate_expression_is_invalid(self): - self.assertRaises(BananaInvalidExpression, validate_expression, - "a123") - self.assertRaises(BananaInvalidExpression, validate_expression, - "a n + 15") - self.assertRaises(BananaInvalidExpression, validate_expression, - "a * exp(b)") - self.assertRaises(BananaInvalidExpression, validate_expression, - "-a") - self.assertRaises(BananaInvalidExpression, validate_expression, - "- a") - self.assertRaises(BananaInvalidExpression, validate_expression, - "+ b") - - def test_validate_name_binding_is_valid(self): - validate_name_binding( - validate_expression("a + b * c"), - {"a": "foo", "b": "foo", "c": "bar"} - ) - - def test_validate_name_binding_is_invalid(self): - self.assertRaises(BananaInvalidExpression, - validate_name_binding, - validate_expression("a + b * c"), - {"a": "foo", "c": "bar"}) - - def test_validate_environment_is_valid(self): - validate_environment({"a": "foo", "c": "bar"}) - - def test_validate_environment_is_invalid(self): - self.assertRaises(BananaEnvironmentError, - validate_environment, {"a": 0}) diff --git a/test/banana/test_config_examples.py b/test/banana/test_config_examples.py deleted file mode 100644 index 215f984..0000000 --- a/test/banana/test_config_examples.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os - -import monasca_analytics.banana.emitter as emit -import monasca_analytics.banana.pass_manager as p - -from test.util_for_testing import MonanasTestCase - - -logger = logging.getLogger(__name__) - -_has_some_examples_been_found = {0: False} - - -class ConfigExamplesTestCase(MonanasTestCase): - - def setUp(self): - super(ConfigExamplesTestCase, self).setUp() - - def tearDown(self): - super(ConfigExamplesTestCase, self).tearDown() - - def test_files_has_been_found(self): - self.assertTrue(_has_some_examples_been_found[0]) - - -def upgrade_test_case(): - for root, dirs, files in os.walk('../config'): - for filename in files: - name_no_ext, ext = os.path.splitext(filename) - if ext == '.banana': - _has_some_examples_been_found[0] = True - with open(os.path.join(root, filename), 'r') as f: - content = f.read() - - def create_test(test_str): - def should_pass(self): - fake_driver = FakeDriver() - emitter = CustomEmitter() - p.execute_banana_string( - test_str, fake_driver, emitter) - self.assertEqual(emitter.nb_errors, 0) - self.assertEqual(emitter.nb_warnings, 0) - self.assertTrue(fake_driver.set_links_called) - self.assertTrue(fake_driver.start_pipeline_called) - self.assertTrue(fake_driver.stop_pipeline_called) - - should_pass.__name__ = "test_banana_examples_config" - return should_pass - - setattr(ConfigExamplesTestCase, - "test_banana_examples_config_" + name_no_ext, - create_test(content)) - -upgrade_test_case() - - -class FakeDriver(object): - - def __init__(self): - self.start_pipeline_called = False - self.stop_pipeline_called = False - self.set_links_called = False - - def stop_pipeline(self): - self.stop_pipeline_called = True - - def start_pipeline(self): - self.start_pipeline_called = True - - def set_links(self, _): - self.set_links_called = True - - -class CustomEmitter(emit.Emitter): - - def __init__(self): - super(CustomEmitter, self).__init__() - self.nb_warnings = 0 - self.nb_errors = 0 - - def emit_warning(self, span, message): - print(span.get_line(), str(span), message) - self.nb_warnings += 1 - - def emit_error(self, span, message): - print(span.get_line(), str(span), message) - self.nb_errors += 1 diff --git a/test/banana/typeck/__init__.py b/test/banana/typeck/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/banana/typeck/should_fail/comp_var_cant_be_shadowed.banana b/test/banana/typeck/should_fail/comp_var_cant_be_shadowed.banana deleted file mode 100644 index 610c34e..0000000 --- a/test/banana/typeck/should_fail/comp_var_cant_be_shadowed.banana +++ /dev/null @@ -1,3 +0,0 @@ -c = IptablesIngestor() -c = 23 -# RAISE BananaShadowingComponentError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/component_cant_be_in_object.banana b/test/banana/typeck/should_fail/component_cant_be_in_object.banana deleted file mode 100644 index fb8fcb1..0000000 --- a/test/banana/typeck/should_fail/component_cant_be_in_object.banana +++ /dev/null @@ -1,4 +0,0 @@ -c = IptablesIngestor() -a = { b: c } -# TODO(Joan): rename Comp to Components -# RAISE BananaAssignCompError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/component_unkown_param_in_arg.banana b/test/banana/typeck/should_fail/component_unkown_param_in_arg.banana deleted file mode 100644 index 8ed0288..0000000 --- a/test/banana/typeck/should_fail/component_unkown_param_in_arg.banana +++ /dev/null @@ -1,2 +0,0 @@ -a = IPTablesSource(sleep=0.01, test="foobar") -# RAISE BananaComponentTooManyParams \ No newline at end of file diff --git a/test/banana/typeck/should_fail/component_unkown_param_in_stmt.banana b/test/banana/typeck/should_fail/component_unkown_param_in_stmt.banana deleted file mode 100644 index 939ad99..0000000 --- a/test/banana/typeck/should_fail/component_unkown_param_in_stmt.banana +++ /dev/null @@ -1,9 +0,0 @@ -a = IPTablesSource(sleep=0.01) -a.test = "foobar" - -# NEW_TEST - -a = CloudMarkovChainSource() -a.transitions.web_servic = {} - -# RAISE BananaPropertyDoesNotExists \ No newline at end of file diff --git a/test/banana/typeck/should_fail/impossible_connection.banana b/test/banana/typeck/should_fail/impossible_connection.banana deleted file mode 100644 index 218018e..0000000 --- a/test/banana/typeck/should_fail/impossible_connection.banana +++ /dev/null @@ -1,5 +0,0 @@ -a -> b - -a = IPTablesSource(sleep=0.01) -b = IPTablesSource(sleep=0.01) -# RAISE BananaConnectionError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/incompatible_json_props_types.banana b/test/banana/typeck/should_fail/incompatible_json_props_types.banana deleted file mode 100644 index 7e97243..0000000 --- a/test/banana/typeck/should_fail/incompatible_json_props_types.banana +++ /dev/null @@ -1,8 +0,0 @@ -# Incompatible type between string and object required by "a.b" -a = { a: "test", a.b: 23 } - -# NEW_TEST - -a = { "a"."b": "test", a.b.c: 23 } - -# RAISE BananaTypeError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/invalid_str_concat.banana b/test/banana/typeck/should_fail/invalid_str_concat.banana deleted file mode 100644 index 352f8ef..0000000 --- a/test/banana/typeck/should_fail/invalid_str_concat.banana +++ /dev/null @@ -1,14 +0,0 @@ -a = "test" -a = a - a - -# NEW_TEST - -a = "test" -a = a * a - -# NEW_TEST - -a = "test" -a = a / a - -# RAISE BananaUnknownOperator \ No newline at end of file diff --git a/test/banana/typeck/should_fail/type_has_no_nested_properties.banana b/test/banana/typeck/should_fail/type_has_no_nested_properties.banana deleted file mode 100644 index 835dd5f..0000000 --- a/test/banana/typeck/should_fail/type_has_no_nested_properties.banana +++ /dev/null @@ -1,9 +0,0 @@ -a = IPTablesSource(sleep=0.01) -b = a.sleep.test - -# NEW_TEST -a = CloudMarkovChainSource() -# Typo: runs -> run -b = a.transitions.web_service."runs=>slow" - -# RAISE BananaPropertyDoesNotExists \ No newline at end of file diff --git a/test/banana/typeck/should_fail/type_is_not_a_subtype.banana b/test/banana/typeck/should_fail/type_is_not_a_subtype.banana deleted file mode 100644 index cccb3de..0000000 --- a/test/banana/typeck/should_fail/type_is_not_a_subtype.banana +++ /dev/null @@ -1,18 +0,0 @@ -a = RandomSource() -a.model = { - params: { - "origin_types": {} - }, - extra_param: 42 -} - -# NEW_TEST - -a = RandomSource(model = { - params: { - "origin_types": {} - }, - extra_param: 42 -}) - -# RAISE BananaArgumentTypeError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/wrong_param_type.banana b/test/banana/typeck/should_fail/wrong_param_type.banana deleted file mode 100644 index 63fd975..0000000 --- a/test/banana/typeck/should_fail/wrong_param_type.banana +++ /dev/null @@ -1,7 +0,0 @@ -a = IPTablesSource(sleep="foobar") - -# NEW_TEST - -a = IPTablesSource(sleep={}) - -# RAISE BananaArgumentTypeError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/wrong_parameter_name_throw.banana b/test/banana/typeck/should_fail/wrong_parameter_name_throw.banana deleted file mode 100644 index b58752b..0000000 --- a/test/banana/typeck/should_fail/wrong_parameter_name_throw.banana +++ /dev/null @@ -1,9 +0,0 @@ -# slep instead of sleep -a = IPTablesSource(slep=0.01) - -# NEW_TEST - -# priod instead of period -a = MonascaDerivativeLDP(priod=0.01) - -# RAISE BananaComponentIncorrectParamName \ No newline at end of file diff --git a/test/banana/typeck/should_pass/compatible_json_props_types.banana b/test/banana/typeck/should_pass/compatible_json_props_types.banana deleted file mode 100644 index cfb9fd0..0000000 --- a/test/banana/typeck/should_pass/compatible_json_props_types.banana +++ /dev/null @@ -1,3 +0,0 @@ -a = { a.b: "test", a.c: "test" } - -# TYPE_TABLE_EQ {Ident< a >: TypeObject < {a: TypeObject < {b: TypeString, c: TypeString} >} >} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/component_param_has_a_type.banana b/test/banana/typeck/should_pass/component_param_has_a_type.banana deleted file mode 100644 index cb87f0c..0000000 --- a/test/banana/typeck/should_pass/component_param_has_a_type.banana +++ /dev/null @@ -1,3 +0,0 @@ -a = IPTablesSource(sleep=0.01) -b = a.sleep -# TYPE_TABLE_EQ {Ident< a >: IPTablesSource(sleep=TypeNumber), Ident< b >: TypeNumber} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/expression_shenanigan.banana b/test/banana/typeck/should_pass/expression_shenanigan.banana deleted file mode 100644 index 7cd8eb0..0000000 --- a/test/banana/typeck/should_pass/expression_shenanigan.banana +++ /dev/null @@ -1,19 +0,0 @@ -a = 12 * 13 + 1 - -# NEW_TEST - -a = 1 + 12 * 13 - -# NEW_TEST - -a = ((12 * 13) + 1) - -# NEW_TEST - -a = (1 + 3) * 3 - -# NEW_TEST - -a = 4 -5 + 22 - 22 - -# TYPE_TABLE_EQ {Ident< a >: TypeNumber} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/monasca_combine.banana b/test/banana/typeck/should_pass/monasca_combine.banana deleted file mode 100644 index d713dca..0000000 --- a/test/banana/typeck/should_pass/monasca_combine.banana +++ /dev/null @@ -1,22 +0,0 @@ -# Checking that parameters are working as expected - -a = MonascaCombineLDP() -a.metric = "cpu.logical_cores_actives" -a.lambda = "a * b - b - a + c" -a.bindings = { - a: "cpu.idle_perc", - b: "cpu.total_logical_cores", - c: "some_val" -} - -# NEW_TEST - -# Same test as above - -a = MonascaCombineLDP( - metric = "cpu.logical_cores_actives", - lambda = "a * b", - bindings = {a:"cpu.idle_perc", b:"cpu.total_logical_cores", c:"some_val"} -) - -# TYPE_TABLE_EQ {Ident< a >: MonascaCombineLDP(metric=TypeString,period=TypeNumber,lambda=TypeString,bindings=TypeAny)} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/nested_value_in_component_params.banana b/test/banana/typeck/should_pass/nested_value_in_component_params.banana deleted file mode 100644 index 48331bc..0000000 --- a/test/banana/typeck/should_pass/nested_value_in_component_params.banana +++ /dev/null @@ -1,9 +0,0 @@ -a = CloudMarkovChainSource() -b = a.transitions.web_service - -# NEW_TEST - -a = CloudMarkovChainSource() -b = a."transitions"."web_service" - -# TYPE_TABLE_EQ {Ident< a >: CloudMarkovChainSource(sleep=TypeNumber,min_event_per_burst=TypeNumber,transitions=TypeStruct < {host: TypeStruct < {off=>on: TypeNumber, on=>off: TypeNumber} >, switch: TypeStruct < {off=>on: TypeNumber, on=>off: TypeNumber} >, web_service: TypeStruct < {run=>slow: TypeAny, slow=>run: TypeAny, stop=>run: TypeAny} >} >,triggers=TypeStruct < {support: TypeStruct < {get_called: TypeAny} >} >,graph=TypeAny), Ident< b >: TypeStruct < {run=>slow: TypeAny, slow=>run: TypeAny, stop=>run: TypeAny} >} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/number_can_be_shadowed_by_object.banana b/test/banana/typeck/should_pass/number_can_be_shadowed_by_object.banana deleted file mode 100644 index 3d78957..0000000 --- a/test/banana/typeck/should_pass/number_can_be_shadowed_by_object.banana +++ /dev/null @@ -1,4 +0,0 @@ -a = {a: "test"} -a = 23 - -# TYPE_TABLE_EQ {Ident< a >: TypeNumber} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/object_can_be_shadowed_by_number.banana b/test/banana/typeck/should_pass/object_can_be_shadowed_by_number.banana deleted file mode 100644 index 6d478c4..0000000 --- a/test/banana/typeck/should_pass/object_can_be_shadowed_by_number.banana +++ /dev/null @@ -1,4 +0,0 @@ -a = 23 -a = {} - -# TYPE_TABLE_EQ {Ident< a >: TypeObject < {} >} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/simple_connection.banana b/test/banana/typeck/should_pass/simple_connection.banana deleted file mode 100644 index bb96895..0000000 --- a/test/banana/typeck/should_pass/simple_connection.banana +++ /dev/null @@ -1,7 +0,0 @@ -# Simple connection -a -> b - -a = IPTablesSource(sleep=0.01) -b = IptablesIngestor() - -# TYPE_TABLE_EQ {Ident< a >: IPTablesSource(sleep=TypeNumber), Ident< b >: IptablesIngestor()} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/simple_literal.banana b/test/banana/typeck/should_pass/simple_literal.banana deleted file mode 100644 index bfe87a2..0000000 --- a/test/banana/typeck/should_pass/simple_literal.banana +++ /dev/null @@ -1,4 +0,0 @@ -a = "test" -b = 213 -c = {} -# TYPE_TABLE_EQ {Ident< a >: TypeString, Ident< b >: TypeNumber, Ident< c >: TypeObject < {} >} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/type_is_a_subtype.banana b/test/banana/typeck/should_pass/type_is_a_subtype.banana deleted file mode 100644 index 3ae970c..0000000 --- a/test/banana/typeck/should_pass/type_is_a_subtype.banana +++ /dev/null @@ -1,20 +0,0 @@ -a = RandomSource() -a.model = { - name: "test", - params: { - "origin_types": {} - }, - extra_param: 42 -} - -# NEW_TEST - -a = RandomSource(model = { - name: "test", - params: { - "origin_types": {} - }, - extra_param: 42 -}) - -# TYPE_TABLE_EQ {Ident< a >: RandomSource(host=TypeString,port=TypeNumber,model=TypeStruct < {name: TypeString, params: TypeStruct < {origin_types: TypeObject < {} >} >} >,alert_per_burst=TypeNumber,idle_time_between_bursts=TypeNumber)} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/valid_str_concat.banana b/test/banana/typeck/should_pass/valid_str_concat.banana deleted file mode 100644 index fb6799c..0000000 --- a/test/banana/typeck/should_pass/valid_str_concat.banana +++ /dev/null @@ -1,4 +0,0 @@ -a = "test" -a = a + a - -# TYPE_TABLE_EQ {Ident< a >: TypeString} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/various_object_use_case.banana b/test/banana/typeck/should_pass/various_object_use_case.banana deleted file mode 100644 index 3bea1bc..0000000 --- a/test/banana/typeck/should_pass/various_object_use_case.banana +++ /dev/null @@ -1,7 +0,0 @@ -a = {} -b = {a: a} -c.d.e = {} -a.test = 12 -e = a -a.test = "test" -# TYPE_TABLE_EQ {Ident< a >: TypeObject < {test: TypeString} >, Ident< b >: TypeObject < {a: TypeObject < {test: TypeString} >} >, Ident< c >: TypeObject < {d: TypeObject < {e: TypeObject < {} >} >} >, Ident< e >: TypeObject < {test: TypeString} >} \ No newline at end of file diff --git a/test/banana/typeck/test_typeck_config.py b/test/banana/typeck/test_typeck_config.py deleted file mode 100644 index f7551d3..0000000 --- a/test/banana/typeck/test_typeck_config.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os -import re - -import monasca_analytics.banana.pass_manager as p -import monasca_analytics.exception.banana as exception -from test.util_for_testing import MonanasTestCase - - -logger = logging.getLogger(__name__) - -_has_some_file_that_should = dict() -_has_some_file_that_should["pass"] = False -_has_some_file_that_should["fail"] = False - - -class TypecheckTestCase(MonanasTestCase): - - def setUp(self): - super(TypecheckTestCase, self).setUp() - - def tearDown(self): - super(TypecheckTestCase, self).tearDown() - - def test_files_have_been_found(self): - self.assertTrue(_has_some_file_that_should["pass"]) - self.assertTrue(_has_some_file_that_should["fail"]) - - -def upgrade_test_case(): - regex_pass = re.compile("#(?: )*TYPE_TABLE_EQ(?: )*([^\n]+)") - regex_fail = re.compile("#(?: )*RAISE(?: )*([^\n]+)") - regex_split = re.compile("#(?: )*NEW_TEST(?:[^\n]*)\n") - - for root, dirs, files in os.walk('./banana/typeck/should_pass'): - for filename in files: - name_no_ext, _ = os.path.splitext(filename) - _has_some_file_that_should["pass"] = True - with open(os.path.join(root, filename), 'r') as f: - content = f.read() - expect_type_table = regex_pass.search(content) - if expect_type_table is not None: - expect_type_table = expect_type_table.group(1) - - def create_test(test_str, expected_type_table): - def should_pass(self): - type_table = p.compute_type_table(test_str) - if expected_type_table is not None: - self.assertEqual(str(type_table), - expected_type_table) - else: - raise Exception( - "Missing # TYPE_TABLE_EQ <...> in test." - ) - - should_pass.__name__ = "test_banana_pass_" + name_no_ext - return should_pass - - many_tests = regex_split.split(content) - if len(many_tests) == 1: - setattr(TypecheckTestCase, - "test_banana_pass_" + name_no_ext, - create_test(many_tests[0], expect_type_table)) - else: - suffix = 0 - for test_case in many_tests: - suffix += 1 - setattr(TypecheckTestCase, - "test_banana_pass_" + name_no_ext + - str(suffix), - create_test(test_case, expect_type_table)) - - for root, dirs, files in os.walk('./banana/typeck/should_fail'): - for filename in files: - name_no_ext, _ = os.path.splitext(filename) - _has_some_file_that_should["fail"] = True - with open(os.path.join(root, filename), 'r') as f: - content = f.read() - expect_error = regex_fail.search(content) - if expect_error is not None: - expect_error = expect_error.group(1) - - def create_test(s, test_str, expected_error): - def should_fail(self): - if expected_error is not None: - expected_exception = get_exception_from_str( - expected_error) - self.assertRaises( - expected_exception, - p.compute_type_table, - test_str) - else: - raise Exception("Missing # RAISE in test") - should_fail.__name__ = "test_banana_fail_" + name_no_ext +\ - str(s) - return should_fail - - many_tests = regex_split.split(content) - if len(many_tests) == 1: - setattr(TypecheckTestCase, - "test_banana_fail_" + name_no_ext, - create_test("", many_tests[0], expect_error)) - else: - suffix = 0 - for test_case in many_tests: - suffix += 1 - setattr(TypecheckTestCase, - "test_banana_fail_" + name_no_ext + - str(suffix), - create_test(suffix, test_case, expect_error)) - -# Fill the test case with generated test case from banana files -upgrade_test_case() - - -def get_exception_from_str(string): - if string == exception.BananaAssignCompError.__name__: - return exception.BananaAssignCompError - if string == exception.BananaConnectionError.__name__: - return exception.BananaConnectionError - if string == exception.BananaAssignmentError.__name__: - return exception.BananaAssignmentError - if string == exception.BananaTypeError.__name__: - return exception.BananaTypeError - if string == exception.BananaArgumentTypeError.__name__: - return exception.BananaArgumentTypeError - if string == exception.BananaUnknown.__name__: - return exception.BananaUnknown - if string == exception.BananaPropertyDoesNotExists.__name__: - return exception.BananaPropertyDoesNotExists - if string == exception.BananaComponentIncorrectParamName.__name__: - return exception.BananaComponentIncorrectParamName - if string == exception.BananaComponentTooManyParams.__name__: - return exception.BananaComponentTooManyParams - if string == exception.BananaShadowingComponentError.__name__: - return exception.BananaShadowingComponentError - if string == exception.BananaUnknownOperator.__name__: - return exception.BananaUnknownOperator - raise Exception("Invalid exception name: '{}'".format(string)) diff --git a/test/cli/__init__.py b/test/cli/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/cli/test_dsl.py b/test/cli/test_dsl.py deleted file mode 100644 index dfc6ae9..0000000 --- a/test/cli/test_dsl.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import json -import logging.config -import os -import unittest - -from monasca_analytics.banana.cli.dsl import MonanasDSL -from monasca_analytics.config import const -from monasca_analytics.exception import dsl as dsl_err -from monasca_analytics.exception import monanas as mon_err - - -class TestMonanasDSL(unittest.TestCase): - - def setup_logging(self): - current_dir = os.path.dirname(__file__) - logging_config_file = os.path.join(current_dir, - "../resources/logging.json") - with open(logging_config_file, "rt") as f: - config = json.load(f) - logging.config.dictConfig(config) - - def get_test_config_file_path(self): - current_dir = os.path.dirname(__file__) - return os.path.join(current_dir, "../resources/test_json.json") - - def setUp(self): - self.setup_logging() - config_file_path = self.get_test_config_file_path() - self.dsl = MonanasDSL(config_file_path) - self.original_config = copy.deepcopy(self.dsl._config) - self.testing_new_source_config = { - "module": "KafkaSource", - "params": { - "zk_host": "myHost", - "zk_port": 1234, - "group_id": "myGroupId", - "topics": { - "myTopic1": 1, - "myTopic2": 2 - } - } - } - self.tmp_file = "tmp_config_file.json" - - def tearDown(self): - if os.path.exists(self.tmp_file): - os.remove(self.tmp_file) - - def test_generate_id(self): - self.assertEqual("src4", self.dsl._generate_id(const.SOURCES)) - self.assertEqual("ing2", self.dsl._generate_id(const.INGESTORS)) - self.assertEqual("sml2", self.dsl._generate_id(const.SMLS)) - self.assertEqual("vot2", self.dsl._generate_id(const.VOTERS)) - self.assertEqual("snk3", self.dsl._generate_id(const.SINKS)) - self.assertEqual("ldp2", self.dsl._generate_id(const.LDPS)) - self.assertEqual("src5", self.dsl._generate_id(const.SOURCES)) - self.assertEqual("ing3", self.dsl._generate_id(const.INGESTORS)) - self.assertEqual("sml3", self.dsl._generate_id(const.SMLS)) - self.assertEqual("vot3", self.dsl._generate_id(const.VOTERS)) - self.assertEqual("snk4", self.dsl._generate_id(const.SINKS)) - self.assertEqual("ldp3", self.dsl._generate_id(const.LDPS)) - self.assertEqual(self.original_config, self.dsl._config) - - def test_generate_id_wrong_type(self): - self.assertRaises(KeyError, self.dsl._generate_id, "wrong_type") - - def test_is_connected(self): - self.assertTrue(self.dsl._is_connected("src1")) - self.assertFalse(self.dsl._is_connected("src2")) - self.assertTrue(self.dsl._is_connected("ing1")) - self.assertTrue(self.dsl._is_connected("sml1")) - self.assertTrue(self.dsl._is_connected("vot1")) - self.assertTrue(self.dsl._is_connected("snk1")) - self.assertTrue(self.dsl._is_connected("snk2")) - self.assertTrue(self.dsl._is_connected("ldp1")) - self.assertEqual(self.original_config, self.dsl._config) - - def test_component_defined(self): - self.assertTrue(self.dsl._component_defined("src1")) - self.assertFalse(self.dsl._component_defined("fake_id")) - self.assertEqual(self.original_config, self.dsl._config) - - def test_validate_connection_by_ids(self): - self.assertTrue(self.dsl._validate_connection_by_ids("src1", "ing1")) - self.assertTrue(self.dsl._validate_connection_by_ids("src1", "ldp1")) - self.assertFalse(self.dsl._validate_connection_by_ids("src2", "sml1")) - self.assertFalse(self.dsl._validate_connection_by_ids("ing1", "vot1")) - self.assertTrue(self.dsl._validate_connection_by_ids("ldp1", "snk1")) - self.assertEqual(self.original_config, self.dsl._config) - - def test_add_component(self): - new_id = self.dsl.add_component(self.testing_new_source_config) - self.assertEqual("src4", new_id) - expected_config = self.original_config - expected_config[const.SOURCES]["src4"] = self.testing_new_source_config - expected_config[const.CONNECTIONS]["src4"] = [] - self.assertEqual(expected_config, self.dsl._config) - - def test_add_component_string(self): - conf_str = json.dumps(self.testing_new_source_config) - new_id = self.dsl.add_component(conf_str) - self.assertEqual("src4", new_id) - expected_config = self.original_config - expected_config[const.SOURCES]["src4"] = self.testing_new_source_config - expected_config[const.CONNECTIONS]["src4"] = [] - self.assertEqual(expected_config, self.dsl._config) - - def test_add_component_wrong_module(self): - self.testing_new_source_config["module"] = "fake_module" - self.assertRaises(mon_err.MonanasNoSuchClassError, - self.dsl.add_component, - self.testing_new_source_config) - self.assertEqual(self.original_config, self.dsl._config) - - def test_modify_component(self): - self.assertTrue(self.dsl.modify_component( - "src3", ["sleep"], 0.02)) - expected_config = self.original_config - expected_config[const.SOURCES]["src3"]["sleep"] = 0.02 - self.assertEqual(expected_config, self.dsl._config) - - def test_modify_component_inexistent(self): - self.assertFalse(self.dsl.modify_component( - "src8", ["params", "server_sleep_in_seconds"], 0.02)) - self.assertEqual(self.original_config, self.dsl._config) - - def test_modify_component_to_invalid_config(self): - self.assertFalse(self.dsl.modify_component( - "src3", ["fake", "fake_param"], 123)) - self.assertEqual(self.original_config, self.dsl._config) - - def test_modify_dictionary_new_path(self): - original = self.original_config["voters"]["vot1"] - modified = self.dsl._modify_dictionary( - original, ["params", "param1", "subparam1A"], "new_value") - expected = { - "module": "voter_module", - "params": { - "param1": { - "subparam1A": "new_value" - } - } - } - self.assertEqual(expected, modified) - - def test_modify_dictioinary_overwrite_value(self): - original = self.original_config["sources"]["src1"] - modified = self.dsl._modify_dictionary( - original, ["params", "param1"], "new_value") - expected = { - "module": "src_module1", - "params": { - "param1": "new_value", - "param2": "val2", - "model_id": 3 - } - } - self.assertEqual(expected, modified) - - def test_modify_dictionary_overwrite_path(self): - original = self.original_config["sources"]["src1"] - modified = self.dsl._modify_dictionary( - original, ["params"], "new_value") - expected = { - "module": "src_module1", - "params": "new_value" - } - self.assertEqual(expected, modified) - - def test_remove_unconnected_component(self): - self.assertTrue(self.dsl.remove_component("src2")) - expected_config = self.original_config - del(expected_config[const.SOURCES]["src2"]) - del(expected_config[const.CONNECTIONS]["src2"]) - self.assertEqual(expected_config, self.dsl._config) - - def test_remove_connected_component(self): - self.assertRaises(dsl_err.DSLExistingConnection, - self.dsl.remove_component, "src1") - self.assertEqual(self.original_config, self.dsl._config) - - def test_remove_component_wrong_id(self): - self.assertFalse(self.dsl.remove_component("fake_id")) - self.assertEqual(self.original_config, self.dsl._config) - - def test_connect_component_new_allowed(self): - self.assertTrue(self.dsl.connect_components("src2", "ing1")) - expected_config = self.original_config - expected_config[const.CONNECTIONS]["src2"].append("ing1") - self.assertEqual(self.original_config, self.dsl._config) - - def test_connect_existing(self): - self.assertFalse(self.dsl.connect_components("src1", "ing1")) - self.assertEqual(self.original_config, self.dsl._config) - - def test_connect_component_new_forbidden(self): - self.assertRaises(dsl_err.DSLInvalidConnection, - self.dsl.connect_components, "src1", "vot1") - self.assertEqual(self.original_config, self.dsl._config) - - def test_disconnect(self): - self.assertTrue(self.dsl.disconnect_components("src1", "ing1")) - expected_config = self.original_config - expected_config[const.CONNECTIONS]["src1"] = ["ldp1"] - self.assertEqual(self.original_config, self.dsl._config) - - def test_disconnect_inexistent_components(self): - self.assertFalse(self.dsl.disconnect_components("fake_1", "fake_2")) - self.assertFalse(self.dsl.disconnect_components("fake_id", "snk1")) - self.assertFalse(self.dsl.disconnect_components("src1", "fake_id")) - self.assertEqual(self.original_config, self.dsl._config) - - def test_disconnect_inexistent_connection(self): - self.assertFalse(self.dsl.disconnect_components("src2", "ing1")) - self.assertEqual(self.original_config, self.dsl._config) - - def test_save_configuration_overwrite_no_file(self): - self.assertTrue( - self.dsl.save_configuration(self.tmp_file, overwrite_file=True)) - self.dsl._config = None - self.dsl.load_configuration(self.tmp_file) - self.assertEqual(self.original_config, self.dsl._config) - - def test_save_configuration_not_overwrite_no_file(self): - self.assertTrue( - self.dsl.save_configuration(self.tmp_file, overwrite_file=False)) - self.dsl._config = None - self.dsl.load_configuration(self.tmp_file) - self.assertEqual(self.original_config, self.dsl._config) - - def _create_dirty_fyle(self, fname): - with open(fname, "w") as f: - f.write("This content may be overwritten") - - def test_save_configuration_overwrite_file(self): - self._create_dirty_fyle(self.tmp_file) - self.assertTrue( - self.dsl.save_configuration(self.tmp_file, overwrite_file=True)) - self.dsl._config = None - self.dsl.load_configuration("tmp_config_file.json") - self.assertEqual(self.original_config, self.dsl._config) - - def test_save_configuration_not_overwrite_file(self): - self._create_dirty_fyle(self.tmp_file) - old_size = os.stat(self.tmp_file).st_size - self.assertFalse( - self.dsl.save_configuration(self.tmp_file, overwrite_file=False)) - self.assertEqual(old_size, os.stat(self.tmp_file).st_size) diff --git a/test/cli/test_interpreter.py b/test/cli/test_interpreter.py deleted file mode 100644 index 9cc9808..0000000 --- a/test/cli/test_interpreter.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import json -import logging.config -import os -import unittest - -from monasca_analytics.banana.cli import interpreter -from monasca_analytics.config import const -from monasca_analytics.exception import dsl as dsl_err -from monasca_analytics.exception import monanas as mon_err -from monasca_analytics.source import iptables_markov_chain as ipt_src -from monasca_analytics.source import kafka - - -class TestDSLInterpreter(unittest.TestCase): - - def setup_logging(self): - current_dir = os.path.dirname(__file__) - logging_config_file = os.path.join(current_dir, - "../resources/logging.json") - with open(logging_config_file, "rt") as f: - config = json.load(f) - logging.config.dictConfig(config) - - def setUp(self): - current_dir = os.path.dirname(__file__) - self.dsl_file_path = os.path.join(current_dir, - "../resources/dsl_code.txt") - self.test_config_path = os.path.join(current_dir, - "../resources/test_json.json") - self.tmp_save_file = "my_config.json" - self.setup_logging() - self.inter = interpreter.DSLInterpreter() - - def tearDown(self): - if os.path.exists(self.tmp_save_file): - os.remove(self.tmp_save_file) - - def test_cmd_create(self): - comp_id = self.inter.execute_string("A = IPTablesSource") - self.assertEqual("src1", comp_id) - self.assertEqual({"A": "src1"}, self.inter.mappings) - self.assertEqual(ipt_src.IPTablesSource.get_default_config(), - self.inter.dsl._config[const.SOURCES]["src1"]) - - def test_cmd_create_inexistent(self): - self.assertRaises(mon_err.MonanasNoSuchClassError, - self.inter.execute_string, - "A = fakeModule") - - def test_cmd_connect(self): - self.inter.execute_string("A = IPTablesSource") - self.inter.execute_string("B = IptablesIngestor") - self.assertTrue(self.inter.execute_string("A -> B")) - self.assertListEqual(["ing1"], - self.inter.dsl._config[const.CONNECTIONS]["src1"]) - - def test_cmd_connect_connected(self): - self.inter.execute_string("A = IPTablesSource") - self.inter.execute_string("B = IptablesIngestor") - self.inter.execute_string("A -> B") - self.assertFalse(self.inter.execute_string("A -> B")) - self.assertListEqual(["ing1"], - self.inter.dsl._config[const.CONNECTIONS]["src1"]) - - def test_cmd_connect_inexistent(self): - self.assertRaises(dsl_err.DSLInterpreterException, - self.inter.execute_string, "A -> B") - - def test_cmd_connect_forbidden(self): - self.inter.execute_string("A = IPTablesSource") - self.inter.execute_string("B = PickIndexVoter") - self.assertRaises(dsl_err.DSLInvalidConnection, - self.inter.execute_string, "A -> B") - - def test_cmd_disconnect(self): - self.inter.execute_string("A = IPTablesSource") - self.inter.execute_string("B = IptablesIngestor") - self.inter.execute_string("A -> B") - self.assertTrue(self.inter.execute_string("A !-> B")) - self.assertListEqual([], - self.inter.dsl._config[const.CONNECTIONS]["src1"]) - - def test_cmd_load(self): - self.inter.execute_string("load(" + self.test_config_path + ")") - with open(self.test_config_path) as f: - expected_config = json.loads(f.read()) - self.assertEqual(expected_config, self.inter.dsl._config) - - def test_cmd_save(self): - self.inter.execute_string("save(" + self.tmp_save_file + ")") - self.inter.execute_string("load(" + self.tmp_save_file + ")") - self.inter.execute_string("A = IPTablesSource") - self.inter.execute_string("save()") - expected = const.get_default_base_config() - expected[const.SOURCES]["src1"] = ipt_src.IPTablesSource.\ - get_default_config() - expected[const.CONNECTIONS]["src1"] = [] - with open(self.tmp_save_file) as f: - saved_config = json.loads(f.read()) - self.assertEqual(expected, saved_config) - pass - - def test_cmd_save_as(self): - self.inter.execute_string("save(" + self.tmp_save_file + ")") - with open(self.tmp_save_file) as f: - saved_config = json.loads(f.read()) - self.assertEqual(const.get_default_base_config(), saved_config) - - def test_cmd_remove(self): - self.inter.execute_string("A = IPTablesSource") - self.inter.execute_string("rm(A)") - self.assertEqual({}, self.inter.dsl._config[const.SOURCES]) - - def test_cmd_remove_inexistent(self): - self.assertRaises(dsl_err.DSLInterpreterException, - self.inter.execute_string, "rm(A)") - - def test_cmd_remove_connected(self): - self.inter.execute_string("A = IPTablesSource") - self.inter.execute_string("B = IptablesIngestor") - self.inter.execute_string("A -> B") - self.assertRaises(dsl_err.DSLExistingConnection, - self.inter.execute_string, "rm(A)") - - def test_cmd_modify_to_valid_float(self): - self.inter.execute_string("A = IPTablesSource") - self.assertTrue(self.inter.execute_string( - "A.sleep = 0.02")) - self.assertEqual({ - "module": "IPTablesSource", - "sleep": 0.02 - }, self.inter.dsl._config[const.SOURCES]["src1"]) - - def test_cmd_modify_to_invalid_float(self): - self.inter.execute_string("A = IPTablesSource") - self.assertFalse(self.inter.execute_string( - "A.sleep = 1.2")) - - def test_cmd_modify_int(self): - self.inter.execute_string("A = KafkaSource") - self.assertTrue(self.inter.execute_string( - "A.params.zk_port = 1234")) - expected = copy.deepcopy(kafka.KafkaSource.get_default_config()) - expected["params"]["zk_port"] = 1234 - self.assertEqual(expected, - self.inter.dsl._config[const.SOURCES]["src1"]) - - def test_cmd_modify_str(self): - self.inter.execute_string("A = KafkaSource") - self.assertTrue(self.inter.execute_string( - "A.params.zk_host = my_host")) - expected = copy.deepcopy(kafka.KafkaSource.get_default_config()) - expected["params"]["zk_host"] = "my_host" - self.assertEqual(expected, - self.inter.dsl._config[const.SOURCES]["src1"]) - - def test_get_id_from_name(self): - self.inter.execute_string("A = IPTablesSource") - _id = self.inter._get_id("A") - self.assertEqual("src1", _id) - - def test_get_id_from_id(self): - self.inter.execute_string("A = IPTablesSource") - _id = self.inter._get_id("src1") - self.assertEqual("src1", _id) diff --git a/test/cli/test_parser.py b/test/cli/test_parser.py deleted file mode 100644 index 7460a2e..0000000 --- a/test/cli/test_parser.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import logging.config -import os -import six -import unittest - -from monasca_analytics.banana.cli import const -from monasca_analytics.banana.cli import parser - - -class TestMonanasDSL(unittest.TestCase): - - def setup_logging(self): - current_dir = os.path.dirname(__file__) - logging_config_file = os.path.join(current_dir, - "../resources/logging.json") - with open(logging_config_file, "rt") as f: - config = json.load(f) - logging.config.dictConfig(config) - - def setUp(self): - current_dir = os.path.dirname(__file__) - self.dsl_file_path = os.path.join(current_dir, - "../resources/dsl_code.txt") - self.setup_logging() - - def tearDown(self): - pass - - def assert_create(self, info, varname, module): - self.assertEqual(varname, info[const.CREATE][0]) - self.assertEqual(module, info[const.CREATE][1]) - - def assert_modify(self, info, varname, params, value): - self.assertEqual(varname, info[const.MODIFY][0]) - self.assertListEqual(info[const.MODIFY][1:-1], params) - self.assertEqual(value, info[const.MODIFY][-1]) - - def assert_connect(self, info, varname1, varname2): - self.assertEqual(varname1, info[const.CONNECT][0]) - self.assertEqual(varname2, info[const.CONNECT][1]) - - def assert_disconnect(self, info, varname1, varname2): - self.assertEqual(varname1, info[const.DISCONNECT][0]) - self.assertEqual(varname2, info[const.DISCONNECT][1]) - - def assert_remove(self, info, varname): - self.assertEqual(varname, info[const.REMOVE][0]) - - def assert_load(self, info, filepaht): - self.assertEqual(filepaht, info[const.LOAD][0]) - - def assert_save(self, info): - self.assertIn(const.SAVE, info) - - def assert_save_as(self, info, filepaht): - self.assertEqual(filepaht, info[const.SAVE_AS][0]) - - def test_parse_create(self): - info = parser.get_parser().parseString("A = my_module") - self.assertEqual(1, len(info)) - six.assertCountEqual(self, [const.CREATE], info[0].keys()) - self.assert_create(info[0], "A", "my_module") - - def test_parse_modify(self): - info = parser.get_parser().parseString("_A1.params.123._inf- = my.val") - self.assertEqual(1, len(info)) - six.assertCountEqual(self, [const.MODIFY], info[0].keys()) - self.assert_modify(info[0], "_A1", ["params", "123", "_inf-"], - "my.val") - - def test_parse_connect(self): - info = parser.get_parser().parseString("_A1->B") - self.assertEqual(1, len(info)) - six.assertCountEqual(self, [const.CONNECT], info[0].keys()) - self.assert_connect(info[0], "_A1", "B") - - def test_parse_disconnect(self): - info = parser.get_parser().parseString("_A1!->B") - self.assertEqual(1, len(info)) - six.assertCountEqual(self, [const.DISCONNECT], info[0].keys()) - self.assert_disconnect(info[0], "_A1", "B") - - def test_parse_remove(self): - info = parser.get_parser().parseString("rM(A)") - self.assertEqual(1, len(info)) - six.assertCountEqual(self, [const.REMOVE], info[0].keys()) - self.assert_remove(info[0], "A") - - def test_parse_load(self): - info = parser.get_parser().parseString("LoAd(_some/path/123.json)") - self.assertEqual(1, len(info)) - six.assertCountEqual(self, [const.LOAD], info[0].keys()) - self.assert_load(info[0], "_some/path/123.json") - - def test_parse_save(self): - info = parser.get_parser().parseString("sAVe()") - self.assertEqual(1, len(info)) - six.assertCountEqual(self, [const.SAVE], info[0].keys()) - self.assert_save(info[0]) - - def test_parse_save_as(self): - info = parser.get_parser().parseString("sAVE(/root/0/path_/f.conf)") - self.assertEqual(1, len(info)) - six.assertCountEqual(self, [const.SAVE_AS], info[0].keys()) - self.assert_save_as(info[0], "/root/0/path_/f.conf") - - def test_parse_multiline(self): - info = parser.get_parser().parseString( - "load(_path/conf.json)\n\nA=MySource1\nA->ing1") - self.assertEqual(3, len(info)) - self.assert_load(info[0], "_path/conf.json") - self.assert_create(info[1], "A", "MySource1") - self.assert_connect(info[2], "A", "ing1") - - def test_parse_comments(self): - info = parser.get_parser().parseString( - "load(_path/conf.json)\n#Comment1\nA=MySource1#Comment2\nA->ing1") - self.assertEqual(3, len(info)) - self.assert_load(info[0], "_path/conf.json") - self.assert_create(info[1], "A", "MySource1") - self.assert_connect(info[2], "A", "ing1") - - def test_parse_file(self): - info = parser.get_parser().parseFile(self.dsl_file_path) - self.assertEqual(13, len(info)) - self.assert_create(info[0], "A", "CloudMarkovChainSource") - self.assert_modify(info[1], "A", ["params", "server_sleep_in_seconds"], - "0.1") - self.assert_create(info[2], "B", "CloudIngestor") - self.assert_create(info[3], "C", "LiNGAM") - self.assert_modify(info[4], "C", ["params", "threshold"], "0.1") - self.assert_create(info[5], "D", "PickIndexVoter") - self.assert_create(info[6], "E", "KafkaSink") - self.assert_create(info[7], "F", "CloudCausalityLDP") - self.assert_connect(info[8], "A", "B") - self.assert_connect(info[9], "A", "F") - self.assert_connect(info[10], "C", "D") - self.assert_connect(info[11], "D", "F") - self.assert_connect(info[12], "F", "E") diff --git a/test/config/__init__.py b/test/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/config/test_create_components.py b/test/config/test_create_components.py deleted file mode 100644 index 2f092f4..0000000 --- a/test/config/test_create_components.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from monasca_analytics.config import const -from monasca_analytics.config import creation -from monasca_analytics.exception import monanas as err -import monasca_analytics.util.common_util as cu -from test.mocks import sml_mocks -from test.util_for_testing import MonanasTestCase - - -class CreateComponentsTest(MonanasTestCase): - - def setUp(self): - """ - Keep a copy of the original functions that will be mocked, then - mock them, reset variables, and initialize ML_Framework. - """ - super(CreateComponentsTest, self).setUp() - self._backup_functions() - self._mock_functions() - sml_mocks.sml_mocks.reset() - self.init_sml_config() - - def tearDown(self): - """ - Restore the potentially mocked functions to the original ones - """ - super(CreateComponentsTest, self).tearDown() - self._restore_functions() - - def _backup_functions(self): - self.original_get_class_by_name = cu.get_class_by_name - - def _restore_functions(self): - cu.get_class_by_name = self.original_get_class_by_name - - def _mock_functions(self): - cu.get_class_by_name = sml_mocks.mock_get_class_by_name - - def init_sml_config(self): - """ - Initialize the ML_Framework object with the test_json config - """ - current_dir = os.path.dirname(__file__) - test_json_file = os.path.join(current_dir, - "../resources/test_json.json") - self._config = cu.parse_json_file(test_json_file) - - def test_create_component_by_module(self): - sml_mocks.sml_mocks.reset() - component_id = "ing1" - component_config = {"module": "ingestor_module", "sink_id": 1} - creation._create_component_by_module(component_id, component_config, - "ingestors") - self.assert_only_instantiated("ingestor_module") - - def test_create_component_by_module_inexistent_module(self): - sml_mocks.sml_mocks.reset() - component_id = "inex1" - component_config = {"module": "inexistent_module"} - self.assertRaises( - err.MonanasNoSuchClassError, - creation._create_component_by_module, - component_id, component_config, "ingestors") - self.assert_instantiated_no_classes() - - def test_create_components_by_module(self): - sml_mocks.sml_mocks.reset() - creation._create_comps_by_module(const.LDPS, self._config) - self.assert_only_instantiated("ldp_module1") - - def test_create_components_by_module_inexistent1(self): - sml_mocks.sml_mocks.reset() - self._config["voter"] = {"inex1": {"module": "inexistent_voter"}} - self.assertRaises(err.MonanasNoSuchClassError, - creation._create_comps_by_module, - "voter", self._config) - self.assert_instantiated_no_classes() - - def test_create_components_by_module_inexistent2(self): - sml_mocks.sml_mocks.reset() - self._config["ingestors"] = { - "inex2": {"module": "inexistent_ingestor", "param": {}} - } - self.assertRaises(err.MonanasNoSuchClassError, - creation._create_comps_by_module, - "ingestors", self._config) - self.assert_instantiated_no_classes() - - def test_create_components_by_module_mixed_existance(self): - sml_mocks.sml_mocks.reset() - self._config[const.INGESTORS]["inex2"] =\ - {"module": "inexistent_ingestor", "params": {}} - self.assertRaises(err.MonanasNoSuchClassError, - creation._create_comps_by_module, - const.INGESTORS, self._config) - - def assert_instantiated_no_classes(self): - for n in sml_mocks.sml_mocks.instantiated.keys(): - self.assertEqual(0, len(sml_mocks.sml_mocks.instantiated[n])) - - def assert_only_instantiated(self, name): - self.assertEqual(1, len(sml_mocks.sml_mocks.instantiated[name])) - for n in sml_mocks.sml_mocks.instantiated.keys(): - if n != name: - self.assertEqual(0, len(sml_mocks.sml_mocks.instantiated[n])) diff --git a/test/ingestor/__init__.py b/test/ingestor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/ingestor/test_base.py b/test/ingestor/test_base.py deleted file mode 100644 index 3a6c742..0000000 --- a/test/ingestor/test_base.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import logging.config -import os -import unittest - -from test.mocks import ingestors - - -class BaseIngestorTest(unittest.TestCase): - """ - Class that tests the BaseIngestor. It uses the Mock as a - testing target, because it extends the abstract class BaseIngestor, - so the base logic can be tested. - """ - - def setup_logging(self): - current_dir = os.path.dirname(__file__) - logging_config_file = os.path.join(current_dir, - "../resources/logging.json") - with open(logging_config_file, "rt") as f: - config = json.load(f) - logging.config.dictConfig(config) - - def setUp(self): - self.setup_logging() - self.bi = ingestors.IngestorBasicChild("fake_id", "fake_config") - - def tearDown(self): - pass diff --git a/test/ingestor/test_cloud.py b/test/ingestor/test_cloud.py deleted file mode 100644 index 95f2de9..0000000 --- a/test/ingestor/test_cloud.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import logging.config -import os -import unittest - -from monasca_analytics.ingestor import cloud - - -class TestIptablesIngestor(unittest.TestCase): - - def setup_logging(self): - current_dir = os.path.dirname(__file__) - logging_config_file = os.path.join(current_dir, - "../resources/logging.json") - with open(logging_config_file, "rt") as f: - config = json.load(f) - logging.config.dictConfig(config) - - def setUp(self): - self.setup_logging() - - def tearDown(self): - pass - - def test_get_default_config(self): - default_config = cloud.CloudIngestor.get_default_config() - cloud.CloudIngestor.validate_config(default_config) - self.assertEqual("CloudIngestor", default_config["module"]) diff --git a/test/ingestor/test_iptables_ingestor.py b/test/ingestor/test_iptables_ingestor.py deleted file mode 100644 index eca290e..0000000 --- a/test/ingestor/test_iptables_ingestor.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import logging.config -import os -import unittest - -import numpy as np - -from monasca_analytics.ingestor import iptables as ipt_ing -from monasca_analytics.source import iptables_markov_chain as ipt_src - - -class TestIptablesIngestor(unittest.TestCase): - - def setup_logging(self): - current_dir = os.path.dirname(__file__) - logging_config_file = os.path.join(current_dir, - "../resources/logging.json") - with open(logging_config_file, "rt") as f: - config = json.load(f) - logging.config.dictConfig(config) - - def setUp(self): - self.setup_logging() - self.rdd_entry = [{ - "ctime": "Mon Apr 11 19:59:12 2016", - "event": { - "msg": "OUTPUT -p icmp --icmp-type echo-request -j ACCEPT", - "id": "1" - } - }, { - "ctime": "Mon Apr 11 19:59:12 2016", - "event": { - "msg": "OUTPUT -o eth0 -p tcp --sport 80 -j ACCEPT", - "id": "1" - } - }] - self.ip_ing = ipt_ing.IptablesIngestor("fake_id", - {"module": "fake_config"}) - self.ip_ing.set_feature_list(["ssh", "ip", "http", "ping"]) - - def tearDown(self): - pass - - def test_get_default_config(self): - default_config = ipt_ing.IptablesIngestor.get_default_config() - ipt_ing.IptablesIngestor.validate_config(default_config) - self.assertEqual("IptablesIngestor", default_config["module"]) - - def test_process_data(self): - rdd_entry = [] - for iptable in ipt_src.iptables: - rdd_entry.append({ - 'ctime': "Mon Apr 11 19:59:12 2016", - 'event': { - "msg": iptable, - "id": 1 - } - }) - processed = self.ip_ing._process_data(rdd_entry, self.ip_ing._features) - np.testing.assert_array_equal( - processed, np.array([2, 4, 2, 4])) diff --git a/test/ldp/__init__.py b/test/ldp/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/ldp/test_cloud_causality.py b/test/ldp/test_cloud_causality.py deleted file mode 100644 index 7ce8f52..0000000 --- a/test/ldp/test_cloud_causality.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.ldp import cloud_causality as cloud -from test.util_for_testing import MonanasTestCase - - -class TestCloudCausalityLDP(MonanasTestCase): - - def setUp(self): - super(TestCloudCausalityLDP, self).setUp() - - def tearDown(self): - super(TestCloudCausalityLDP, self).tearDown() - - def test_get_default_config(self): - default_config = cloud.CloudCausalityLDP.get_default_config() - cloud.CloudCausalityLDP.validate_config(default_config) - self.assertEqual("CloudCausalityLDP", default_config["module"]) diff --git a/test/ldp/test_iptables_ldp.py b/test/ldp/test_iptables_ldp.py deleted file mode 100644 index 7dab201..0000000 --- a/test/ldp/test_iptables_ldp.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.ldp import iptables_ldp -from test.mocks import classifier_mock -from test.util_for_testing import MonanasTestCase - - -class TestIptablesLDP(MonanasTestCase): - - def setUp(self): - super(TestIptablesLDP, self).setUp() - self.rdd_entry = [{ - "ctime": "Mon Apr 11 19:59:12 2016", - "event": { - "msg": "OUTPUT -p icmp --icmp-type echo-request -j ACCEPT", - "id": "1" - } - }, { - "ctime": "Mon Apr 11 19:59:12 2016", - "event": { - "msg": "OUTPUT -o eth0 -p tcp --sport 80 -j ACCEPT", - "id": "1" - } - }] - self.raw_events = [x["event"] for x in self.rdd_entry] - self.ip_ldp = iptables_ldp.IptablesLDP("fake_id", - {"module": "fake_config"}) - - def tearDown(self): - super(TestIptablesLDP, self).tearDown() - - def assert_anomalous_events(self, events, anomalous=True): - expected_events = self.raw_events - for exv in expected_events: - exv["anomalous"] = anomalous - self.assertEqual(expected_events, events) - - def test_detect_anomalies_no_features(self): - self.clf = classifier_mock.MockClassifier(False) - self.ip_ldp.set_voter_output(self.clf) - ret = self.ip_ldp._detect_anomalies(self.rdd_entry, - self.ip_ldp._data) - self.assertEqual(self.raw_events, ret) - - def test_detect_anomalies_no_classifier(self): - self.clf = classifier_mock.MockClassifier(False) - self.ip_ldp.set_feature_list(["ssh", "ip", "http", "ping"]) - ret = self.ip_ldp._detect_anomalies(self.rdd_entry, - self.ip_ldp._data) - self.assertEqual(self.raw_events, ret) - - def test_detect_anomalies_non_anomalous(self): - self.clf = classifier_mock.MockClassifier(False) - self.ip_ldp.set_feature_list(["ssh", "ip", "http", "ping"]) - self.ip_ldp.set_voter_output(self.clf) - ret = self.ip_ldp._detect_anomalies(self.rdd_entry, - self.ip_ldp._data) - self.assert_anomalous_events(ret, False) - - def test_detect_anomalies_anomalous(self): - self.clf = classifier_mock.MockClassifier(True) - self.ip_ldp.set_feature_list(["ssh", "ip", "http", "ping"]) - self.ip_ldp.set_voter_output(self.clf) - ret = self.ip_ldp._detect_anomalies(self.rdd_entry, - self.ip_ldp._data) - self.assert_anomalous_events(ret, True) diff --git a/test/ldp/test_monasca_aggregator.py b/test/ldp/test_monasca_aggregator.py deleted file mode 100644 index 4712b19..0000000 --- a/test/ldp/test_monasca_aggregator.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.ldp.monasca_aggregate import MonascaAggregateLDP -from test.util_for_testing import gen_metric -from test.util_for_testing import MonanasTestCase - - -class TestMonascaAggregateLDP(MonanasTestCase): - - def setUp(self): - super(TestMonascaAggregateLDP, self).setUp() - self.all_metrics = [ - gen_metric("memory", 1.5, 0, "h1"), - gen_metric("memory", 2.0, 1, "h1"), - gen_metric("memory", 1.0, 0, "h2"), - gen_metric("memory", 3.0, 1, "h2"), - ] - - def tearDown(self): - super(TestMonascaAggregateLDP, self).tearDown() - - def _reduce(self, fn, iterable): - # iterable = map(lambda i: {"metric": {"value": i}}, iterable) - cnt = len(iterable) - acc = fn[0](iterable[0], cnt) - for index in range(1, cnt): - acc = fn[1](acc, iterable[index], cnt) - return acc - - def _conf(self, fn_name): - return { - "func": fn_name - } - - def test_reducer_avg(self): - avg_reducer = MonascaAggregateLDP.select_reducer(self._conf("avg")) - self.assertEqual( - self._reduce(avg_reducer, [1.0, 2.0, 3.0, 6.0]), - 3 - ) - - def test_reducer_max(self): - max_reducer = MonascaAggregateLDP.select_reducer(self._conf("max")) - self.assertEqual( - self._reduce(max_reducer, [1, 2, 3, 6]), - 6 - ) - - def test_reducer_min(self): - min_reducer = MonascaAggregateLDP.select_reducer(self._conf("min")) - self.assertEqual( - self._reduce(min_reducer, [1, 2, 3, 6]), - 1 - ) - - def test_reducer_sum(self): - sum_reducer = MonascaAggregateLDP.select_reducer(self._conf("sum")) - self.assertEqual( - self._reduce(sum_reducer, [1, 2, 3, 6]), - 12 - ) - - def test_reducer_cnt(self): - cnt_reducer = MonascaAggregateLDP.select_reducer(self._conf("cnt")) - self.assertEqual( - self._reduce(cnt_reducer, [1, 2, 3, 6]), - 4 - ) - - def test_aggregate_with_avg(self): - reducer = MonascaAggregateLDP.select_reducer(self._conf("avg")) - res = MonascaAggregateLDP.aggregate(self.all_metrics, reducer, "_avg") - res = [m["metric"]["value"] for m in res] - self.assertEqual(res, [1.25, 2.5]) - - def test_aggregate_with_min(self): - reducer = MonascaAggregateLDP.select_reducer(self._conf("min")) - res = MonascaAggregateLDP.aggregate(self.all_metrics, reducer, "_min") - res = [m["metric"]["value"] for m in res] - self.assertEqual(res, [1.0, 2.0]) - - def test_aggregate_with_max(self): - reducer = MonascaAggregateLDP.select_reducer(self._conf("max")) - res = MonascaAggregateLDP.aggregate(self.all_metrics, reducer, "_max") - res = [m["metric"]["value"] for m in res] - self.assertEqual(res, [1.5, 3.0]) - - def test_aggregate_with_sum(self): - reducer = MonascaAggregateLDP.select_reducer(self._conf("sum")) - res = MonascaAggregateLDP.aggregate(self.all_metrics, reducer, "_sum") - res = [m["metric"]["value"] for m in res] - self.assertEqual(res, [2.5, 5.0]) - - def test_aggregate_with_cnt(self): - reducer = MonascaAggregateLDP.select_reducer(self._conf("cnt")) - res = MonascaAggregateLDP.aggregate(self.all_metrics, reducer, "_cnt") - res = [m["metric"]["value"] for m in res] - self.assertEqual(res, [2, 2]) diff --git a/test/ldp/test_monasca_combine.py b/test/ldp/test_monasca_combine.py deleted file mode 100644 index 44cda9d..0000000 --- a/test/ldp/test_monasca_combine.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from test.util_for_testing import MonanasTestCase - - -class TestMonascaAggregateLDP(MonanasTestCase): - - def setUp(self): - super(TestMonascaAggregateLDP, self).setUp() - self.all_metrics = [ - gen_metric("nb_cores", 1.2, 0, "h1"), - gen_metric("nb_cores", 2.0, 1, "h1"), - gen_metric("idl_perc", 0.2, 0, "h1"), - gen_metric("idl_perc", 0.8, 1, "h1"), - ] - - def tearDown(self): - super(TestMonascaAggregateLDP, self).tearDown() diff --git a/test/ldp/test_monasca_derivative.py b/test/ldp/test_monasca_derivative.py deleted file mode 100644 index e6c008e..0000000 --- a/test/ldp/test_monasca_derivative.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.ldp.monasca_derivative import MonascaDerivativeLDP -from pyspark.resultiterable import ResultIterable -from test.util_for_testing import gen_metric -from test.util_for_testing import MonanasTestCase - - -class TestMonascaAggregateLDP(MonanasTestCase): - - def setUp(self): - super(TestMonascaAggregateLDP, self).setUp() - self.all_metrics = [ - gen_metric("memory", 1.5, 0, "h1"), - gen_metric("memory", 2.0, 1, "h1"), - gen_metric("memory", 1.0, 0, "h1"), - gen_metric("memory", 3.0, 1, "h1"), - gen_metric("memory", 1.5, 4, "h1"), - gen_metric("memory", 2.0, 2, "h1"), - gen_metric("memory", 1.0, 3, "h1"), - gen_metric("memory", 3.0, 5, "h1"), - ] - - def _values(self, values): - return [m["metric"]["value"] for m in values] - - def tearDown(self): - super(TestMonascaAggregateLDP, self).tearDown() - - def test_derivative_should_do_nothing_with_1_value(self): - self.assertEqual(MonascaDerivativeLDP.derivative( - ResultIterable(self.all_metrics[0:1])), - []) - - def test_derivative_should_work_on_first_and_last_values(self): - new_values = MonascaDerivativeLDP.derivative( - ResultIterable(self.all_metrics[0:2])) - new_values = self._values(new_values) - self.assertEqual(new_values, - [0.5, 0.5]) - - def test_derivative_should_remove_duplicate(self): - new_values = MonascaDerivativeLDP.derivative( - ResultIterable(self.all_metrics[0:4])) - new_values = self._values(new_values) - self.assertEqual(new_values, - [0.5, 0.5]) - - def test_derivative_should_work(self): - new_values = MonascaDerivativeLDP.derivative( - ResultIterable(self.all_metrics)) - new_values = self._values(new_values) - self.assertEqual(new_values, - [0.5, 0.25, -0.5, -0.25, 1.0, 1.5]) diff --git a/test/mocks/__init__.py b/test/mocks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/mocks/classifier_mock.py b/test/mocks/classifier_mock.py deleted file mode 100644 index c8f5e36..0000000 --- a/test/mocks/classifier_mock.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import numpy as np - -from monasca_analytics.sml import svm_one_class as svm - - -class MockClassifier(object): - - def __init__(self, predict_anomaly=False): - self._predict_anomaly = predict_anomaly - self.predict_cnt = 0 - - def predict(self, _): - self.predict_cnt += 1 - if self._predict_anomaly: - return np.array([svm.ANOMALY]) - return np.array([svm.NON_ANOMALY]) diff --git a/test/mocks/ingestors.py b/test/mocks/ingestors.py deleted file mode 100644 index c31cc47..0000000 --- a/test/mocks/ingestors.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.ingestor import base - - -class IngestorBasicChild(base.BaseIngestor): - """ - BaseIngestor with abstract classes implemented in order to test it - """ - - validation_cnt = 0 - - def __init__(self, _id, _config): - IngestorBasicChild.validation_cnt = 0 - self.process_stream_cnt = 0 - self.before_stop_ingesting_cnt = 0 - super(IngestorBasicChild, self).__init__(_id, _config) - - @staticmethod - def validate_config(_config): - IngestorBasicChild.validation_cnt += 1 - - @staticmethod - def get_default_config(): - {"module": IngestorBasicChild.__name__} - - def map_dstream(self): - self.process_stream_cnt += 1 - - def before_stop_ingesting(self): - self.before_stop_ingesting_cnt += 1 - - -class MockIngestor(IngestorBasicChild): - """ - Mock of BaseIngestor in order to test code that uses it - """ - - def __init__(self, _id, _config): - self.ingest_cnt = 0 - super(MockIngestor, self).__init__(_id, _config) - - def ingest(self, dstream, ssc): - self.ingest_cnt += 1 diff --git a/test/mocks/markov.py b/test/mocks/markov.py deleted file mode 100644 index 695d40b..0000000 --- a/test/mocks/markov.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import monasca_analytics.source.markov_chain.base as bmkv - - -class MockRequestBuilder(bmkv.RequestBuilder): - - def __init__(self, events): - super(MockRequestBuilder, self).__init__(request=None) - self.events = events - - def send(self, data): - self.events.append(data) - - def finalize(self): - pass diff --git a/test/mocks/sml_mocks.py b/test/mocks/sml_mocks.py deleted file mode 100644 index baf6401..0000000 --- a/test/mocks/sml_mocks.py +++ /dev/null @@ -1,337 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import numpy as np - -from monasca_analytics.exception import monanas as err -from monasca_analytics.ingestor import base as base_ing -from monasca_analytics.ldp import base as base_ldp -from monasca_analytics.sink import base as base_snk -from monasca_analytics.sml import base as base_sml -from monasca_analytics.source import base as base_src -from monasca_analytics.spark import aggregator -from monasca_analytics.voter import base as base_voter - - -class SMLMocks(object): - """ - Auxiliary class to track modules instantiations, classes got by name, - and validated configuration. - """ - - def __init__(self): - self.reset() - - def reset(self): - """ - Resets all the elements (counts and lists) - """ - self.instantiated = {"src_module1": [], - "src_module2": [], - "IPTablesSource": [], - "ingestor_module": [], - "sml_module": [], - "voter_module": [], - "sink_module1": [], - "sink_module2": [], - "ldp_module1": [] - } - self.classes_got_by_name = [] - self.bound_sources = [] - self.terminated_sources = [] - self.killed = False - - def reset_connections(self): - """ - Resets the variables of the instantiated elements only - """ - for comp_type in self.instantiated.values(): - for comp in comp_type: - comp.reset_vars() - -sml_mocks = SMLMocks() - - -class MockClass_src(base_src.BaseSource): - - def __init__(self, _id, _config): - super(MockClass_src, self).__init__(_id, _config) - self.reset_vars() - - def reset_vars(self): - self.get_feature_list_cnt = 0 - self.create_dstream_cnt = 0 - self.terminate_source_cnt = 0 - - @staticmethod - def validate_config(_config): - pass - - @staticmethod - def get_default_config(): - return {"module": MockClass_src.__name__} - - @staticmethod - def get_params(): - return [] - - def get_feature_list(self): - self.get_feature_list_cnt += 1 - return ["a", "b"] - - def create_dstream(self, ssc): - self.create_dstream_cnt += 1 - return ssc.mockDStream() - - def terminate_source(self): - self.terminate_source_cnt += 1 - - -class MockClass_src_module1(MockClass_src): - - def __init__(self, _id, _config): - sml_mocks.instantiated["src_module1"].append(self) - super(MockClass_src_module1, self).__init__(_id, _config) - - -class MockClass_src_module2(MockClass_src): - - def __init__(self, _id, _config): - sml_mocks.instantiated["src_module2"].append(self) - super(MockClass_src_module2, self).__init__(_id, _config) - - -class MockClass_src_module3(MockClass_src): - - def __init__(self, _id, _config): - sml_mocks.instantiated["IPTablesSource"].append(self) - super(MockClass_src_module3, self).__init__(_id, _config) - - -class MockClass_ingestor_module(base_ing.BaseIngestor): - - def __init__(self, _id, _config): - super(MockClass_ingestor_module, self).__init__(_id, _config) - sml_mocks.instantiated["ingestor_module"].append(self) - self.reset_vars() - - @staticmethod - def validate_config(_config): - pass - - @staticmethod - def get_default_config(): - return {"module": MockClass_ingestor_module.__name__} - - @staticmethod - def get_params(): - return [] - - def reset_vars(self): - self.map_dstream_cnt = 0 - - def map_dstream(self, dstream): - self.map_dstream_cnt += 1 - return dstream - - -class MockClass_aggr_module(aggregator.Aggregator): - - def __init__(self, driver): - super(MockClass_aggr_module, self).__init__(driver) - self.reset_vars() - - @staticmethod - def validate_config(_config): - pass - - @staticmethod - def get_default_config(): - return {"module": MockClass_aggr_module.__name__} - - @staticmethod - def get_params(): - return [] - - def reset_vars(self): - self.accumulate_dstream_samples_cnt = 0 - self.append_sml_cnt = 0 - self._smls = [] - - def append_sml(self, sml): - super(MockClass_aggr_module, self).append_sml(sml) - self.append_sml_cnt += 1 - - def accumulate_dstream_samples(self, dstream): - self._samples = np.array([]) - self._combined_stream = None - super(MockClass_aggr_module, self).accumulate_dstream_samples(dstream) - self.accumulate_dstream_samples_cnt += 1 - - -class MockClass_sml_module(base_sml.BaseSML): - - def __init__(self, _id, _config): - super(MockClass_sml_module, self).__init__(_id, _config) - sml_mocks.instantiated["sml_module"].append(self) - self.reset_vars() - - @staticmethod - def validate_config(_config): - pass - - @staticmethod - def get_default_config(): - return {"module": MockClass_sml_module.__name__} - - @staticmethod - def get_params(): - return [] - - def reset_vars(self): - self._voter = None - self.learn_structure_cnt = 0 - - def learn_structure(self, _): - self.learn_structure_cnt += 1 - - def number_of_samples_required(self): - return 0 - - -class MockClass_voter_module(base_voter.BaseVoter): - - def __init__(self, _id, _config): - super(MockClass_voter_module, self).__init__(_id, _config) - sml_mocks.instantiated["voter_module"].append(self) - self.reset_vars() - - @staticmethod - def validate_config(_config): - pass - - @staticmethod - def get_default_config(): - return {"module": MockClass_voter_module.__name__} - - @staticmethod - def get_params(): - return [] - - def reset_vars(self): - self.elect_structure_cnt = 0 - - def elect_structure(self, _): - self.elect_structure_cnt += 1 - - -class MockClass_sink(base_snk.BaseSink): - - def __init__(self, _id, _config): - super(MockClass_sink, self).__init__(_id, _config) - self.reset_vars() - - def reset_vars(self): - self.sink_dstream_cnt = 0 - self.sink_sml_cnt = 0 - - @staticmethod - def validate_config(_config): - pass - - @staticmethod - def get_default_config(): - return {"module": MockClass_sink.__name__} - - @staticmethod - def get_params(): - return [] - - def sink_dstream(self, _): - self.sink_dstream_cnt += 1 - - def sink_ml(self, *_): - self.sink_sml_cnt += 1 - - -class MockClass_sink_module1(MockClass_sink): - - def __init__(self, _id, _config): - sml_mocks.instantiated["sink_module1"].append(self) - super(MockClass_sink_module1, self).__init__(_id, _config) - - -class MockClass_sink_module2(MockClass_sink): - - def __init__(self, _id, _config): - sml_mocks.instantiated["sink_module2"].append(self) - super(MockClass_sink_module2, self).__init__(_id, _config) - - -class MockClass_ldp_module1(base_ldp.BaseLDP): - - def __init__(self, _id, _config): - super(MockClass_ldp_module1, self).__init__(_id, _config) - sml_mocks.instantiated["ldp_module1"].append(self) - self.reset_vars() - - @staticmethod - def validate_config(_config): - pass - - @staticmethod - def get_default_config(): - return {"module": MockClass_ldp_module1.__name__} - - @staticmethod - def get_params(): - return [] - - def map_dstream(self, dstream): - self.map_dstream_cnt += 1 - return dstream - - def reset_vars(self): - self.map_dstream_cnt = 0 - - -def mock_kill(pid, code): - sml_mocks.killed = True - - -def mock_get_class_by_name(module, class_type): - sml_mocks.classes_got_by_name.append([module, class_type]) - if module == "src_module1": - return MockClass_src_module1 - elif module == "src_module2": - return MockClass_src_module2 - elif module == "IPTablesSource": - return MockClass_src_module3 - elif module == "ingestor_module": - return MockClass_ingestor_module - elif module == "aggr_module": - return MockClass_aggr_module - elif module == "sml_module": - return MockClass_sml_module - elif module == "voter_module": - return MockClass_voter_module - elif module == "sink_module1": - return MockClass_sink_module1 - elif module == "sink_module2": - return MockClass_sink_module2 - elif module == "ldp_module1": - return MockClass_ldp_module1 - raise err.MonanasNoSuchClassError("testing NoSuchClassError") diff --git a/test/mocks/sources.py b/test/mocks/sources.py deleted file mode 100644 index 1cc7103..0000000 --- a/test/mocks/sources.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.source import base - - -class MockBaseSource(base.BaseSource): - - validation_cnt = 0 - - def __init__(self, _id, _config): - MockBaseSource.validation_cnt = 0 - self.before_binding_cnt = 0 - self.after_binding_cnt = 0 - self.before_unbinding_cnt = 0 - self.after_unbinding_cnt = 0 - super(MockBaseSource, self).__init__(_id, _config) - - @staticmethod - def validate_config(_config): - MockBaseSource.validation_cnt += 1 - - @staticmethod - def get_default_config(): - return {"module": MockBaseSource.__name__} - - @staticmethod - def get_params(): - return [] - - def create_dstream(self, ssc): - self.before_binding_cnt += 1 - return None - - def terminate_source(self): - pass - - def get_feature_list(self): - pass diff --git a/test/mocks/spark_mocks.py b/test/mocks/spark_mocks.py deleted file mode 100644 index fe77aa1..0000000 --- a/test/mocks/spark_mocks.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import time - - -class MockSparkContext(object): - - def __init__(self, appName): - self.parallelize_cnt = 0 - pass - - def parallelize(self, param): - self.parallelize_cnt += 1 - pass - - -class MockStreamingContext(object): - - def __init__(self, sc, streaming_batch_interval): - self.sparkContext = sc - self.started_cnt = 0 - self.stopped_cnt = 0 - self._textFileStreamDirectory = None - self._port = None - - def start(self): - self.started_cnt += 1 - - def stop(self, stopSparkContext=True, stopGraceFully=False): - self.stopped_cnt += 1 - - def awaitTermination(self): - self.stopped_cnt += 1 - - def textFileStream(self, directory): - self._textFileStreamDirectory = directory - return "file_dstream" - - def socketTextStream(self, host, port): - self._port = port - self._host = host - return MockDStream(None, None, None) - - def mockDStream(self): - return MockDStream(None, None, None) - - def addStreamingListener(self, *arg, **kwargs): - pass - - def checkpoint(self, directory): - pass - - -class MockKafkaUtils(object): - - @staticmethod - def createStream(ssc, hostport, groupid, topics): - return "kafka_dstream" - - -class MockDStream(object): - - def __init__(self, jdstream, ssc, jrdd_deserializer): - self.transform_cnt = 0 - self.join_cnt = 0 - self.foreachRDD_cnt = 0 - self.map_cnt = 0 - self.jdstream = jdstream - self.ssc = ssc - self.fake_rdd = None - self.jrdd_deserializer = jrdd_deserializer - - def transform(self, func): - self.transform_cnt += 1 - self.fake_rdd = MockRDD( - self.jdstream, self.ssc, self.jrdd_deserializer) - timestamp = time.time() - func(timestamp, self.fake_rdd) - - def join(self, other_dstream): - self.join_cnt += 1 - - def foreachRDD(self, func): - self.foreachRDD_cnt += 1 - self.fake_rdd = MockRDD( - self.jdstream, self.ssc, self.jrdd_deserializer) - timestamp = time.time() - func(timestamp, self.fake_rdd) - - def map(self, func): - self.map_cnt += 1 - self.fake_rdd = MockRDD( - self.jdstream, self.ssc, self.jrdd_deserializer) - for rdd_entry in self.fake_rdd.collect(): - func(rdd_entry) - - def pprint(self, num=10): - pass - - -class MockRDD(object): - - def __init__(self, jdstream, ssc, jrdd_deserializer): - self.collect_cnt = 0 - self._rdd_entries = [] - - def set_rdd_entries(self, rdd_entries): - self._rdd_entries = rdd_entries - - def collect(self): - self.collect_cnt += 1 - return self._rdd_entries diff --git a/test/resources/dsl_code.txt b/test/resources/dsl_code.txt deleted file mode 100644 index 6373b91..0000000 --- a/test/resources/dsl_code.txt +++ /dev/null @@ -1,15 +0,0 @@ -#target generated_config.json - -A = CloudMarkovChainSource -A.params.server_sleep_in_seconds = 0.1 -B = CloudIngestor -C = LiNGAM -C.params.threshold = 0.1 -D = PickIndexVoter -E = KafkaSink -F = CloudCausalityLDP -A -> B -A -> F -C -> D -D -> F -F -> E \ No newline at end of file diff --git a/test/resources/logging.json b/test/resources/logging.json deleted file mode 100644 index a4f4e8c..0000000 --- a/test/resources/logging.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "standard": { - "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s" - } - }, - "handlers": { - "default": { - "level": "DEBUG", - "class": "logging.StreamHandler", - "formatter": "standard" - } - }, - "loggers": { - "": { - "handlers": ["default"], - "level": "ERROR", - "propagate": true - } - } -} diff --git a/test/resources/test_json.json b/test/resources/test_json.json deleted file mode 100644 index af7fbff..0000000 --- a/test/resources/test_json.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "spark_config": { - "appName": "testApp", - "streaming": { - "batch_interval": 1 - } - }, - "server": { - "port": 3000, - "debug": true - }, - "sources": { - "src1": { - "module": "src_module1", - "params": { - "param1": "val1", - "param2": "val2", - "model_id": 3 - } - }, - "src2": { - "module": "src_module2" - }, - "src3": { - "module": "IPTablesSource", - "sleep": 0.01 - } - }, - "ingestors": { - "ing1": { - "module": "ingestor_module" - } - }, - "smls": { - "sml1": { - "module": "sml_module" - } - }, - "voters": { - "vot1": { - "module": "voter_module" - } - }, - "sinks": { - "snk1": { - "module": "sink_module1" - }, - "snk2": { - "module": "sink_module2" - } - }, - "ldps": { - "ldp1": { - "module": "ldp_module1" - } - }, - "connections": { - "src1": ["ing1", "ldp1"], - "src2": [], - "ing1": [], - "sml1": ["vot1", "snk1"], - "vot1": ["ldp1"], - "ldp1": ["snk2"], - "snk2": [], - "snk1": [] - }, - "feedback": { - "snk1": ["sml1"], - "snk2": ["vot1"] - } -} diff --git a/test/sink/__init__.py b/test/sink/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/sink/test_base_sqlite.py b/test/sink/test_base_sqlite.py deleted file mode 100644 index c35d9af..0000000 --- a/test/sink/test_base_sqlite.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import sqlite3 -import unittest - -import numpy as np -import six -from six.moves import cPickle -import voluptuous - -import monasca_analytics.banana.typeck.type_util as type_util -import monasca_analytics.component.params as params -from monasca_analytics.sink import base_sqlite as bsql -from test.mocks import spark_mocks - - -class BaseSQLiteSinkDummyExtension(bsql.BaseSQLiteSink): - - def _rdds_table_create_query(self): - return """CREATE TABLE IF NOT EXISTS rdds - (fake_col1 TEXT, fake_col2 TEXT)""" - - def _rdd_insert_query(self, rdd_json): - return ('INSERT INTO rdds VALUES("' + rdd_json["one"] + - '", "' + rdd_json["two"] + '")') - - @staticmethod - def get_default_config(): - return { - "module": BaseSQLiteSinkDummyExtension.__name__, - "params": { - "db_name": "sqlite_sink.db" - } - } - - @staticmethod - def get_params(): - return [ - params.ParamDescriptor( - "db_name", - type_util.String(), - "sqlite_sink.db" - ) - ] - - @staticmethod - def validate_config(_config): - base_schema = voluptuous.Schema({ - "module": voluptuous.And( - six.string_types[0], - lambda i: not any(c.isspace() for c in i)), - voluptuous.Optional("db_name"): voluptuous.And( - six.string_types[0], - lambda i: not any(c.isspace() for c in i)), - }, required=True) - return base_schema(_config) - - -class TestSQLiteSink(unittest.TestCase): - - def setUp(self): - unittest.TestCase.setUp(self) - self._valid_config = {"module": "BaseSQLiteSinkDummyExtension"} - self.snk = BaseSQLiteSinkDummyExtension("fake_id", self._valid_config) - - def get_rdd_mock(self): - rdd = spark_mocks.MockRDD(None, None, None) - rdd_entries = [{"one": "row1col1", "two": "row1col2"}, - {"one": "row2col1", "two": "row2col2"}, - {"one": "row3col1", "two": "row3col2"}] - rdd.set_rdd_entries(rdd_entries) - return rdd - - def assert_rdd_written_to_db(self, rdd): - with sqlite3.connect("sqlite_sink.db") as conn: - c = conn.cursor() - for row in c.execute('SELECT * FROM rdds'): - if self._find_row_in_rdd(row, rdd): - return - self.fail("Did not find rdd in database") - - def _find_row_in_rdd(self, row, rdd): - for rdd_entry in rdd._rdd_entries: - if rdd_entry["one"] == row[0] and rdd_entry["two"] == row[1]: - return True - return False - - def assert_sml_written_to_db(self, sml, voter_id): - with sqlite3.connect("sqlite_sink.db") as conn: - c = conn.cursor() - c.execute('SELECT sml FROM smls WHERE voter_id = "' + - voter_id + '"') - fetched_sml = c.fetchone() - if six.PY2: - fetched_sml = cPickle.loads(str(fetched_sml[0])) - else: - fetched_sml = cPickle.loads(fetched_sml[0]) - self.assertEqual(len(sml), len(fetched_sml)) - self.assertTrue((sml == fetched_sml).all()) - - def test_validate_valid_config_no_dbname(self): - conf = {"module": "BaseSQLiteSinkDummyExtension"} - self.snk.validate_config(conf) - - def test_validate_valid_config_with_dbname(self): - conf = {"module": "BaseSQLiteSinkDummyExtension", - "db_name": "mySQLite.db"} - self.snk.validate_config(conf) - - def test_validate_config_no_module(self): - conf = {"db_name": "mySQLite.db"} - self.assertRaises(voluptuous.Invalid, self.snk.validate_config, conf) - - def test_validate_config_extra_param(self): - conf = {"module": "BaseSQLiteSinkDummyExtension", - "infiltrated": "I shouldn't be here"} - self.assertRaises(voluptuous.Invalid, self.snk.validate_config, conf) - - def test_get_db_name(self): - conf = {"db_name": "mySQLite.db"} - db_name = self.snk._get_db_name(conf) - self.assertEqual("mySQLite.db", db_name) - - def test_get_db_name_default(self): - conf = {"module": "BaseSQLiteSinkDummyExtension"} - db_name = self.snk._get_db_name(conf) - self.assertEqual(bsql.DB_NAME_DEFAULT, db_name) - - def test_persist(self): - rdd = self.get_rdd_mock() - self.snk._persist(None, rdd) - self.assert_rdd_written_to_db(rdd) - - def test_sink_ml_array(self): - sml = np.array([[1, 2, 3], ["a", "b", "c"], [.1, .5, .9]]) - self.snk.sink_ml("vot1", sml) - self.assert_sml_written_to_db(sml, "vot1") - - def tearDown(self): - unittest.TestCase.tearDown(self) - os.remove("sqlite_sink.db") - -if __name__ == "__main__": - unittest.main() diff --git a/test/sink/test_iptables_sqlite.py b/test/sink/test_iptables_sqlite.py deleted file mode 100644 index 31bf97b..0000000 --- a/test/sink/test_iptables_sqlite.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import unittest - -from monasca_analytics.sink import iptables_sqlite as ipt_snk - - -class TestIptablesSQLiteSink(unittest.TestCase): - - def setUp(self): - unittest.TestCase.setUp(self) - self._valid_config = {"module": "IptablesSQLiteSink"} - self.snk = ipt_snk.IptablesSQLiteSink("fake_id", self._valid_config) - - def test_rdds_table_create_query(self): - query = self.snk._rdds_table_create_query() - self.assertEqual("""CREATE TABLE IF NOT EXISTS rdds - (msg TEXT, anomalous TEXT, msg_id TEXT, ctime TEXT)""", query) - - def test_rdd_insert_query_valid_rdd(self): - rdd_entry = { - "msg": "test message", - "id": 1, - "anomalous": True, - "ctime": "t1" - } - query = self.snk._rdd_insert_query(rdd_entry) - self.assertEqual( - 'INSERT INTO rdds VALUES("test message", "True", "1", "t1")', - query) - - def test_rdd_insert_query_invalid_rdd(self): - rdd_entry = { - "msg": "test message", - "anomalous": True, - "ctime": "t1" - } - self.assertRaises(KeyError, self.snk._rdd_insert_query, rdd_entry) - - def tearDown(self): - unittest.TestCase.tearDown(self) - os.remove("sqlite_sink.db") - -if __name__ == "__main__": - unittest.main() diff --git a/test/sink/test_kafka.py b/test/sink/test_kafka.py deleted file mode 100644 index fec0d69..0000000 --- a/test/sink/test_kafka.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import unittest - -# This file is name kafkas otherwise it will conflict -# with the global import 'kafka' -import monasca_analytics.sink.kafkas as kf - - -class KafkaSinkTest(unittest.TestCase): - def setUp(self): - unittest.TestCase.setUp(self) - - class MockedKafka(object): - def __init__(self): - self.has_been_called = False - - def KafkaProducer(self, *_, **kwargs): - self.has_been_called = True - - self.mock_kafka = MockedKafka() - self.original_kafka = kf.kafka - kf.kafka = self.mock_kafka - - def testKafkaSinkInit(self): - kf.KafkaSink("id", { - "module": "KafkaSink", - "host": "localhost", - "port": 00, - "topic": "boom", - }) - self.assertTrue(self.mock_kafka) - - def tearDown(self): - unittest.TestCase.tearDown(self) - kf.kafka = self.original_kafka - - def test_get_default_config(self): - default_config = kf.KafkaSink.get_default_config() - kf.KafkaSink.validate_config(default_config) - self.assertEqual("KafkaSink", default_config["module"]) - -if __name__ == "__main__": - unittest.main() diff --git a/test/sink/test_sink_config_validator.py b/test/sink/test_sink_config_validator.py deleted file mode 100644 index 514b82b..0000000 --- a/test/sink/test_sink_config_validator.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import unittest -import voluptuous - -import monasca_analytics.sink.sink_config_validator as validator - -kafka = validator.validate_kafka_sink_config - - -class SinkConfigValidatorTest(unittest.TestCase): - def setUp(self): - unittest.TestCase.setUp(self) - self._valid_config = { - "module": "KafkaSink", - "host": "127.0.0.1", - "port": 9092, - "topic": "transformed_data" - } - - def test_validate_kafka_sink_valid_config(self): - try: - kafka(self._valid_config) - except voluptuous.Invalid as e: - self.fail(e.__str__()) - - def test_validate_kafka_sink_invalid_module(self): - invalid_config = self._valid_config - invalid_config["module"] = "invalid_module" - self.assertRaises(voluptuous.Invalid, kafka, invalid_config) - - def test_validate_kafka_sink_invalid_host(self): - invalid_config = self._valid_config - invalid_config["host"] = "invalid host" - self.assertRaises(voluptuous.Invalid, kafka, invalid_config) - - def test_validate_kafka_sink_invalid_port(self): - invalid_config = self._valid_config - invalid_config["port"] = "invalid_port" - self.assertRaises(voluptuous.Invalid, kafka, invalid_config) - - def test_validate_kafka_sink_invalid_topic(self): - invalid_config = self._valid_config - invalid_config["topic"] = "invalid topic" - self.assertRaises(voluptuous.Invalid, kafka, invalid_config) - - def tearDown(self): - unittest.TestCase.tearDown(self) - -if __name__ == "__main__": - unittest.main() diff --git a/test/sml/__init__.py b/test/sml/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/sml/test_decision_tree.py b/test/sml/test_decision_tree.py deleted file mode 100644 index 0675fa9..0000000 --- a/test/sml/test_decision_tree.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -from sklearn import tree - -from monasca_analytics.sml import decision_tree -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class TestDecisionTreeClassifier(MonanasTestCase): - - def setUp(self): - super(TestDecisionTreeClassifier, self).setUp() - self.dt_sml = decision_tree.DecisionTreeClassifier( - "fakeid", {"module": "fake", "nb_samples": 1000}) - - def tearDown(self): - super(TestDecisionTreeClassifier, self).tearDown() - - def get_testing_data(self): - a = np.random.uniform(size=1000) - b = np.random.uniform(size=1000) - c = np.random.uniform(size=1000) - d = np.random.uniform(size=1000) - labels = np.random.randint(2, size=1000) - return np.array([a, b, c, d, labels]).T - - def test_generate_train_test_sets(self): - data = self.get_testing_data() - X_train, X_train_labeled, X_test, X_test_labeled =\ - self.dt_sml._generate_train_test_sets(data, 0.6) - self.assertEqual(600, len(X_train)) - self.assertEqual(600, len(X_train_labeled)) - self.assertEqual(400, len(X_test)) - self.assertEqual(400, len(X_test_labeled)) - - def test_learn_structure(self): - data = self.get_testing_data() - clf = self.dt_sml.learn_structure(data) - self.assertIsInstance(clf, tree.DecisionTreeClassifier) diff --git a/test/sml/test_elliptic_envelope.py b/test/sml/test_elliptic_envelope.py deleted file mode 100644 index 0031ebb..0000000 --- a/test/sml/test_elliptic_envelope.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -from sklearn import covariance - -from monasca_analytics.sml import elliptic_envelope -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class TestEllipticEnvelope(MonanasTestCase): - - def setUp(self): - super(TestEllipticEnvelope, self).setUp() - self.ee_sml = elliptic_envelope.EllipticEnvelope( - "fakeid", {"module": "fake", "nb_samples": 1000}) - - def tearDown(self): - super(TestEllipticEnvelope, self).tearDown() - - def get_testing_data(self): - a = np.random.uniform(size=1000) - b = np.random.uniform(size=1000) - c = np.random.uniform(size=1000) - d = np.random.uniform(size=1000) - return np.array([a, b, c, d]).T - - def test_generate_train_test_sets(self): - data = self.get_testing_data() - train, test = self.ee_sml._generate_train_test_sets(data, 0.6) - self.assertEqual(600, len(train)) - self.assertEqual(400, len(test)) - - def test_learn_structure(self): - data = self.get_testing_data() - clf = self.ee_sml.learn_structure(data) - self.assertIsInstance(clf, covariance.EllipticEnvelope) diff --git a/test/sml/test_isolation_forest.py b/test/sml/test_isolation_forest.py deleted file mode 100644 index 1947e73..0000000 --- a/test/sml/test_isolation_forest.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -from sklearn import ensemble - -from monasca_analytics.sml import isolation_forest -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class TestIsolationForest(MonanasTestCase): - - def setUp(self): - super(TestIsolationForest, self).setUp() - self.if_sml = isolation_forest.IsolationForest( - "fakeid", {"module": "fake", "nb_samples": 1000}) - - def tearDown(self): - super(TestIsolationForest, self).tearDown() - - def get_testing_data(self): - a = np.random.uniform(size=1000) - b = np.random.uniform(size=1000) - c = np.random.uniform(size=1000) - d = np.random.uniform(size=1000) - return np.array([a, b, c, d]).T - - def test_generate_train_test_sets(self): - data = self.get_testing_data() - train, test = self.if_sml._generate_train_test_sets(data, 0.6) - self.assertEqual(600, len(train)) - self.assertEqual(400, len(test)) - - def test_learn_structure(self): - data = self.get_testing_data() - clf = self.if_sml.learn_structure(data) - self.assertIsInstance(clf, ensemble.IsolationForest) diff --git a/test/sml/test_lingam.py b/test/sml/test_lingam.py deleted file mode 100644 index b72bf14..0000000 --- a/test/sml/test_lingam.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np - -from monasca_analytics.sml import lingam -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class LiNGAMTest(MonanasTestCase): - - def setUp(self): - super(LiNGAMTest, self).setUp() - - def tearDown(self): - super(LiNGAMTest, self).tearDown() - - def test_continuous_lingam_algorithm(self): - b = np.random.laplace(size=500) - a = np.random.laplace(size=500) + b - d = np.random.laplace(size=500) + a + b - c = np.random.laplace(size=500) + d - data = np.array([a, b, c, d]) - causality_matrix, causal_order =\ - lingam.LiNGAM._discover_structure(data.T) - - logger.debug("\nb deps (should be almost zero): {}" - .format(np.sum(np.abs(causality_matrix[1, :])))) - logger.debug("\ncausality matrix:\n{}".format(causality_matrix)) - self.assertEqual(np.all(causal_order == np.array([1, 0, 3, 2])), True, - "Algorithm didn't found the causal order!") - - def test_discrete_set_lingam_algorithm(self): - b = np.random.laplace(size=500) - a = np.random.laplace(size=500) + b - d = np.random.laplace(size=500) + a + b - c = np.random.laplace(size=500) + d - data = np.array([a, b, c, d]) - data = np.floor(data) - causality_matrix, causal_order =\ - lingam.LiNGAM._discover_structure(data.T) - - logger.debug("\nb deps (should be almost zero): {}" - .format(np.sum(np.abs(causality_matrix[1, :])))) - logger.debug("\ncausality matrix:\n{}".format(causality_matrix)) - self.assertEqual(np.all(causal_order == np.array([1, 0, 3, 2])), True, - "Algorithm didn't found the causal order!") - - def test_discrete_set_absolute_value_lingam_algorithm(self): - b = np.random.laplace(size=500) - a = np.random.laplace(size=500) + b - d = np.random.laplace(size=500) + a + b - c = np.random.laplace(size=500) + d - data = np.array([a, b, c, d]) - data = np.floor(data) - data = np.abs(data) - causality_matrix, causal_order =\ - lingam.LiNGAM._discover_structure(data.T) - - logger.debug("\nb deps (should be almost zero): {}" - .format(np.sum(np.abs(causality_matrix[1, :])))) - logger.debug("\ncausality matrix:\n{}".format(causality_matrix)) - self.assertEqual(np.all(causal_order == np.array([1, 0, 3, 2])), True, - "Algorithm didn't found the causal order!") - - def test_get_default_config(self): - default_config = lingam.LiNGAM.get_default_config() - lingam.LiNGAM.validate_config(default_config) - self.assertEqual("LiNGAM", default_config["module"]) diff --git a/test/sml/test_logistic_regression.py b/test/sml/test_logistic_regression.py deleted file mode 100644 index 4c3b9b3..0000000 --- a/test/sml/test_logistic_regression.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -from sklearn import linear_model - -from monasca_analytics.sml import logistic_regression -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class TestLogisticRegression(MonanasTestCase): - - def setUp(self): - super(TestLogisticRegression, self).setUp() - self.lr_sml = logistic_regression.LogisticRegression( - "fakeid", {"module": "fake", "nb_samples": 1000}) - - def tearDown(self): - super(TestLogisticRegression, self).tearDown() - - def get_testing_data(self): - a = np.random.uniform(size=1000) - b = np.random.uniform(size=1000) - c = np.random.uniform(size=1000) - d = np.random.uniform(size=1000) - labels = np.random.randint(2, size=1000) - return np.array([a, b, c, d, labels]).T - - def test_generate_train_test_sets(self): - data = self.get_testing_data() - X_train, X_train_labeled, X_test, X_test_labeled =\ - self.lr_sml._generate_train_test_sets(data, 0.6) - self.assertEqual(600, len(X_train)) - self.assertEqual(600, len(X_train_labeled)) - self.assertEqual(400, len(X_test)) - self.assertEqual(400, len(X_test_labeled)) - - def test_learn_structure(self): - data = self.get_testing_data() - clf = self.lr_sml.learn_structure(data) - self.assertIsInstance(clf, linear_model.LogisticRegression) diff --git a/test/sml/test_random_forest_classifier.py b/test/sml/test_random_forest_classifier.py deleted file mode 100644 index 78f2b9d..0000000 --- a/test/sml/test_random_forest_classifier.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -from sklearn import ensemble - -from monasca_analytics.sml import random_forest_classifier -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class TestRandomForestClassifier(MonanasTestCase): - - def setUp(self): - super(TestRandomForestClassifier, self).setUp() - self.rf_sml = random_forest_classifier.RandomForestClassifier( - "fakeid", {"module": "fake", "nb_samples": 1000}) - - def tearDown(self): - super(TestRandomForestClassifier, self).tearDown() - - def get_testing_data(self): - a = np.random.uniform(size=1000) - b = np.random.uniform(size=1000) - c = np.random.uniform(size=1000) - d = np.random.uniform(size=1000) - labels = np.random.randint(2, size=1000) - return np.array([a, b, c, d, labels]).T - - def test_generate_train_test_sets(self): - data = self.get_testing_data() - X_train, X_train_labeled, X_test, X_test_labeled =\ - self.rf_sml._generate_train_test_sets(data, 0.6) - self.assertEqual(600, len(X_train)) - self.assertEqual(600, len(X_train_labeled)) - self.assertEqual(400, len(X_test)) - self.assertEqual(400, len(X_test_labeled)) - - def test_learn_structure(self): - data = self.get_testing_data() - clf = self.rf_sml.learn_structure(data) - self.assertIsInstance(clf, ensemble.RandomForestClassifier) diff --git a/test/sml/test_svc.py b/test/sml/test_svc.py deleted file mode 100644 index 750dc79..0000000 --- a/test/sml/test_svc.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -from sklearn import svm - -from monasca_analytics.sml import svc -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class TestSvc(MonanasTestCase): - - def setUp(self): - super(TestSvc, self).setUp() - self.svc_sml = svc.Svc( - "fakeid", {"module": "fake", "nb_samples": 1000}) - - def tearDown(self): - super(TestSvc, self).tearDown() - - def get_testing_data(self): - a = np.random.uniform(size=1000) - b = np.random.uniform(size=1000) - c = np.random.uniform(size=1000) - d = np.random.uniform(size=1000) - labels = np.random.randint(2, size=1000) - return np.array([a, b, c, d, labels]).T - - def test_generate_train_test_sets(self): - data = self.get_testing_data() - X_train, X_train_labeled, X_test, X_test_labeled =\ - self.svc_sml._generate_train_test_sets(data, 0.6) - self.assertEqual(600, len(X_train)) - self.assertEqual(600, len(X_train_labeled)) - self.assertEqual(400, len(X_test)) - self.assertEqual(400, len(X_test_labeled)) - - def test_learn_structure(self): - data = self.get_testing_data() - clf = self.svc_sml.learn_structure(data) - self.assertIsInstance(clf, svm.SVC) diff --git a/test/sml/test_svm_one_class.py b/test/sml/test_svm_one_class.py deleted file mode 100644 index 75cefe9..0000000 --- a/test/sml/test_svm_one_class.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -import numpy as np -from sklearn import svm - -from monasca_analytics.sml import svm_one_class -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class TestSvmOneClass(MonanasTestCase): - - def setUp(self): - super(TestSvmOneClass, self).setUp() - self.svm = svm_one_class.SvmOneClass("fakeid", { - "module": "fake", - "nb_samples": 1000 - }) - - def tearDown(self): - super(TestSvmOneClass, self).tearDown() - - def get_testing_data(self): - a = np.random.uniform(size=1000) - b = np.random.uniform(size=1000) - c = np.random.uniform(size=1000) - d = np.random.uniform(size=1000) - return np.array([a, b, c, d]).T - - def test_generate_train_test_sets(self): - data = self.get_testing_data() - train, test = self.svm._generate_train_test_sets(data, 0.6) - self.assertEqual(600, len(train)) - self.assertEqual(400, len(test)) - - def test_learn_structure(self): - data = self.get_testing_data() - clf = self.svm.learn_structure(data) - self.assertIsInstance(clf, svm.OneClassSVM) diff --git a/test/source/__init__.py b/test/source/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/source/markov_chain/__init__.py b/test/source/markov_chain/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/source/markov_chain/test_base.py b/test/source/markov_chain/test_base.py deleted file mode 100644 index 4d13dd9..0000000 --- a/test/source/markov_chain/test_base.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - -from monasca_analytics.source.markov_chain import base -import monasca_analytics.source.markov_chain.events as ev -import monasca_analytics.source.markov_chain.prob_checks as pck -import monasca_analytics.source.markov_chain.state_check as dck -import monasca_analytics.source.markov_chain.transition as tr -import test.mocks.markov as markov_mocks -from test.util_for_testing import MonanasTestCase - - -class StateNodeTest(MonanasTestCase): - - def setUp(self): - super(StateNodeTest, self).setUp() - - def tearDown(self): - super(StateNodeTest, self).tearDown() - - def test_collect_events_should_be_populated_by_trigger(self): - some_trigger = ev.Trigger( - node_check=dck.EqCheck(0), - prob_check=pck.NoProbCheck(), - event_builder=ev.EventBuilder("test") - ) - node = base.StateNode(0, None, some_trigger) - events = [] - node.collect_events(1, datetime.datetime.now(), - markov_mocks.MockRequestBuilder(events)) - self.assertTrue(len(events) == 1) - self.assertEqual(events[0]["event"].msg, "test", "a") - - def test_next_state_should_use_available_transitions(self): - tr1 = tr.Transition( - from_state=0, - to_state=1, - deps_check=dck.TrueCheck(), - prob_check=pck.NoProbCheck() - ) - tr2 = tr.Transition( - from_state=1, - to_state=2, - deps_check=dck.EqCheck(1), - prob_check=pck.NoProbCheck() - ) - mc = tr.MarkovChain([tr1, tr2]) - n1 = base.StateNode(0, mc, None) - n2 = base.StateNode(1, mc, None) - n3 = base.StateNode(0, mc, None) - n1.dependencies.append(n2) - n2.dependencies.append(n3) - # First round - n1.next_state(1, set()) - self.assertEqual(n1.state, 1) - self.assertEqual(n2.state, 1) - self.assertEqual(n3.state, 0) - # Second round - n1.next_state(1, set()) - self.assertEqual(n1.state, 2) - self.assertEqual(n2.state, 1) - self.assertEqual(n3.state, 0) - - def test_next_state_update_only_deps_and_deps_in_first(self): - tr1 = tr.Transition( - from_state=0, - to_state=1, - prob_check=pck.NoProbCheck() - ) - tr2 = tr.Transition( - from_state=1, - to_state=2, - deps_check=dck.EqCheck(1), - prob_check=pck.NoProbCheck() - ) - mc = tr.MarkovChain([tr1, tr2]) - n1 = base.StateNode(0, mc, None) - n2 = base.StateNode(1, mc, None) - n3 = base.StateNode(0, mc, None) - n1.dependencies.append(n2) - n2.dependencies.append(n3) - n2.next_state(1, set()) - self.assertEqual(n1.state, 0) - self.assertEqual(n2.state, 2) - self.assertEqual(n3.state, 1) diff --git a/test/source/markov_chain/test_events.py b/test/source/markov_chain/test_events.py deleted file mode 100644 index 8960158..0000000 --- a/test/source/markov_chain/test_events.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - -import monasca_analytics.source.markov_chain.events as ev -import monasca_analytics.source.markov_chain.prob_checks as pck -import monasca_analytics.source.markov_chain.state_check as dck -import test.mocks.markov as markov_mocks - -from test.util_for_testing import MonanasTestCase - - -class DummyState(object): - - def __init__(self, state=0): - self.state = state - self.dependencies = [] - - def id(self): - return 0 - - -class TriggersTest(MonanasTestCase): - - def setUp(self): - super(TriggersTest, self).setUp() - - def tearDown(self): - super(TriggersTest, self).tearDown() - - def test_trigger_should_create_event_when_necessary(self): - some_trigger = ev.Trigger( - node_check=dck.EqCheck(0), - prob_check=pck.NoProbCheck(), - event_builder=ev.EventBuilder("") - ) - events = [] - some_trigger.apply_on( - DummyState(), - 1, - datetime.datetime.now(), - markov_mocks.MockRequestBuilder(events)) - self.assertEqual(len(events), 1) - events = [] - some_trigger.apply_on( - DummyState(1), - 1, - datetime.datetime.now(), - markov_mocks.MockRequestBuilder(events)) - self.assertEqual(len(events), 0) diff --git a/test/source/markov_chain/test_transition.py b/test/source/markov_chain/test_transition.py deleted file mode 100644 index ce3c794..0000000 --- a/test/source/markov_chain/test_transition.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monasca_analytics.source.markov_chain.prob_checks as pck -import monasca_analytics.source.markov_chain.state_check as dck -import monasca_analytics.source.markov_chain.transition as t -from test.util_for_testing import MonanasTestCase - - -class DummyState(object): - - def __init__(self, state=0): - self.state = state - self.dependencies = [] - - -class MarkovChainTransitionsTest(MonanasTestCase): - - def setUp(self): - super(MarkovChainTransitionsTest, self).setUp() - - def tearDown(self): - super(MarkovChainTransitionsTest, self).tearDown() - - def test_first_order_dep_check(self): - state = DummyState() - state.dependencies.append(DummyState(1)) - dc = dck.DepCheck(dck.EqCheck(1)) - self.assertTrue(dc(state)) - state = DummyState() - self.assertFalse(dc(state)) - state = DummyState() - state.dependencies.append(DummyState(2)) - self.assertFalse(dc(state)) - - def test_second_order_dep_check(self): - state = DummyState() - state1 = DummyState() - state.dependencies.append(state1) - state1.dependencies.append(DummyState(1)) - dc = dck.DepCheck(dck.DepCheck(dck.EqCheck(1))) - self.assertTrue(dc(state)) - state = DummyState() - self.assertFalse(dc(state)) - self.assertFalse(dc(state1)) - - def test_combiner_and_dep_check(self): - state = DummyState() - state1 = DummyState(1) - state.dependencies.append(state1) - state1.dependencies.append(DummyState(2)) - dc = dck.AndCheck(c1=dck.DepCheck(dck.EqCheck(1)), - c2=dck.DepCheck(dck.DepCheck(dck.EqCheck(2)))) - self.assertTrue(dc(state)) - self.assertFalse(dc(state1)) - state1.state = 2 - self.assertFalse(dc(state)) - state1.state = 1 - state1.dependencies[0].state = 1 - self.assertFalse(dc(state)) - state = DummyState() - self.assertFalse(dc(state)) - - def test_combiner_or_dep_check(self): - state = DummyState() - state1 = DummyState(1) - state.dependencies.append(state1) - state1.dependencies.append(DummyState(2)) - dc = dck.OrCheck(c1=dck.DepCheck(dck.EqCheck(1)), - c2=dck.DepCheck(dck.DepCheck(dck.EqCheck(2)))) - self.assertTrue(dc(state)) - self.assertFalse(dc(state1)) - state1.dependencies[0].state = 1 - self.assertTrue(dc(state)) - self.assertTrue(dc(state1)) - state1.state = 2 - self.assertFalse(dc(state)) - state = DummyState() - self.assertFalse(dc(state)) - - def test_prob_check(self): - pc = pck.ProbCheck(0.5) - i = 0 - while i < 30: - if pc(0): - break - i += 1 - self.assertTrue(i < 30) - - def test_prob_check_interpolate(self): - pc = pck.ProbCheck({0: 0.0, 1: 0.0, 24: 1.0}) - self.assertFalse(pc(0)) - self.assertFalse(pc(1)) - self.assertTrue(pc(24)) - i = 0 - while i < 30: - if pc(12): - break - i += 1 - self.assertTrue(i < 30) - - def test_transition(self): - tr = t.Transition(0, 1, pck.ProbCheck(1.0)) - state = DummyState(0) - self.assertTrue(tr(state, 1)) - self.assertEqual(state.state, 1) - state = DummyState(2) - self.assertFalse(tr(state, 1)) - self.assertEqual(state.state, 2) - - def test_transition_with_true_check(self): - tr = t.Transition(0, 1, pck.NoProbCheck(), dck.TrueCheck()) - state = DummyState(0) - self.assertFalse(tr(state, 1)) - state1 = DummyState(123456) - state.dependencies.append(state1) - self.assertTrue(tr(state, 1)) - self.assertEqual(state.state, 1) - - def test_markov_chain(self): - tr1 = t.Transition(0, 1, pck.ProbCheck(1.0)) - tr2 = t.Transition(1, 2, pck.ProbCheck(1.0)) - mc = t.MarkovChain([tr1, tr2]) - state1 = DummyState(0) - state2 = DummyState(1) - mc.apply_on(state1, 1) - mc.apply_on(state2, 1) - self.assertEqual(state1.state, 1) - self.assertEqual(state2.state, 2) - mc.apply_on(state1, 1) - mc.apply_on(state2, 1) - self.assertEqual(state1.state, 2) - self.assertEqual(state2.state, 2) diff --git a/test/source/test_base.py b/test/source/test_base.py deleted file mode 100644 index 4aa834f..0000000 --- a/test/source/test_base.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from test.mocks import sources -from test.util_for_testing import MonanasTestCase - - -class BaseSourceTest(MonanasTestCase): - """ - Class that tests the BaseDataSource. It uses the Mock as a testing target, - because it extends the abstract class BaseIngestor, so the base logic - can be tested. - """ - - def setUp(self): - super(BaseSourceTest, self).setUp() - self.bs = sources.MockBaseSource("fake_id", "fake_config") - - def tearDown(self): - super(BaseSourceTest, self).tearDown() - - def test_validate_called(self): - self.assertEqual(1, sources.MockBaseSource.validation_cnt) diff --git a/test/source/test_iptables_markov_chain.py b/test/source/test_iptables_markov_chain.py deleted file mode 100644 index 3783043..0000000 --- a/test/source/test_iptables_markov_chain.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import voluptuous - -from monasca_analytics.source import iptables_markov_chain -from test.util_for_testing import MonanasTestCase - - -import six - - -class TestIPTablesSource(MonanasTestCase): - - def setUp(self): - super(TestIPTablesSource, self).setUp() - self.valid_config = { - "module": "IPTablesSource", - "sleep": 0.01 - } - self.config_extra_param = { - "module": "IPTablesSource", - "sleep": 0.01, - "infiltrated": "wrong_param" - } - self.config_missing_param = { - "module": "IPTablesSource", - } - self.config_wrong_type = { - "module": "IPTablesSource", - "sleep": "I should be an integer" - } - self.config_missing_params = {"module": "IPTablesSource"} - self.ips = iptables_markov_chain.IPTablesSource("fake_id", - self.valid_config) - - def tearDown(self): - super(TestIPTablesSource, self).tearDown() - - def test_validate_valid_config(self): - self.assertEqual(self.valid_config, self.ips._config) - - def test_validate_config_extra_param(self): - self.assertRaises( - voluptuous.Invalid, self.ips.validate_config, - self.config_extra_param) - - def test_validate_config_missing_param(self): - self.assertRaises( - voluptuous.Invalid, self.ips.validate_config, - self.config_missing_param) - - def test_validate_config_wrong_type(self): - self.assertRaises( - voluptuous.Invalid, - self.ips.validate_config, - self.config_wrong_type) - - def test_get_default_config(self): - default_config = iptables_markov_chain.\ - IPTablesSource.get_default_config() - iptables_markov_chain.IPTablesSource.validate_config(default_config) - self.assertEqual("IPTablesSource", default_config["module"]) - - def assertStateNode(self, node, state, num_triggers, states_transitions): - self.assertEqual(node.state, state) - self.assertEqual(len(node._triggers), num_triggers) - self.assertEqual(len(node._markov_chain._transitions), - len(states_transitions.keys())) - for state, num_transitions in six.iteritems(states_transitions): - self.assertEqual(len(node._markov_chain._transitions[state]), - num_transitions) - - def assertTransition(self, transition, from_state, to_state): - self.assertEqual(transition._from_state, from_state) - self.assertEqual(transition._to_state, to_state) - - def assert_all_correct_transitions(self, trs): - self.assertTransition(trs[iptables_markov_chain.STATE_STOP][0], - from_state=iptables_markov_chain.STATE_STOP, - to_state=iptables_markov_chain.STATE_NORMAL) - self.assertTransition(trs[iptables_markov_chain.STATE_NORMAL][0], - from_state=iptables_markov_chain.STATE_NORMAL, - to_state=iptables_markov_chain.STATE_STOP) - self.assertTransition(trs[iptables_markov_chain.STATE_NORMAL][1], - from_state=iptables_markov_chain.STATE_NORMAL, - to_state=iptables_markov_chain.STATE_ATTACK) - self.assertTransition(trs[iptables_markov_chain.STATE_ATTACK][0], - from_state=iptables_markov_chain.STATE_ATTACK, - to_state=iptables_markov_chain.STATE_NORMAL) - - def test_create_system(self): - nodes = self.ips._create_system() - self.assertEqual(1, len(nodes)) - self.assertStateNode(nodes[0], - state=iptables_markov_chain.STATE_STOP, - num_triggers=16, - states_transitions={ - iptables_markov_chain.STATE_STOP: 1, - iptables_markov_chain.STATE_NORMAL: 2, - iptables_markov_chain.STATE_ATTACK: 1}) - self.assert_all_correct_transitions(nodes[0]. - _markov_chain._transitions) - - def test_create_markov_chain_model(self): - markov_chain = self.ips._create_markov_chain_model() - self.assert_all_correct_transitions(markov_chain._transitions) diff --git a/test/source/test_kafka.py b/test/source/test_kafka.py deleted file mode 100644 index 9d43a4a..0000000 --- a/test/source/test_kafka.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import voluptuous - -from monasca_analytics.source import kafka -from test.mocks import spark_mocks -from test.util_for_testing import MonanasTestCase - - -class KafkaSourceTest(MonanasTestCase): - - def _mock_functions(self): - kafka.kafka.KafkaUtils = spark_mocks.MockKafkaUtils - - def setUp(self): - super(KafkaSourceTest, self).setUp() - self._mock_functions() - self.valid_config = { - "module": "kafka", - "params": { - "zk_host": "my_host", - "zk_port": 1234, - "group_id": "my_group_id", - "topics": {"topic1": 1, "topic2": 2} - } - } - self.config_extra_param = { - "module": "kafka", - "params": { - "zk_host": "my_host", - "zk_port": 1234, - "group_id": "my_group_id", - "topics": {"topic1": 1, "topic2": 2}, - "infiltrated": "wrong_param" - } - } - self.config_missing_param = { - "module": "kafka", - "params": { - "zk_host": "my_host", - "group_id": "my_group_id", - "topics": {"topic1": 1, "topic2": 2} - } - } - self.config_wrong_type = { - "module": 123, - "params": { - "zk_host": "my_host", - "zk_port": 1234, - "group_id": "my_group_id", - "topics": {"topic1": 1, "topic2": 2} - } - } - self.config_missing_params = {"module": "file"} - self.ks = kafka.KafkaSource("fake_id", self.valid_config) - - def tearDown(self): - super(KafkaSourceTest, self).tearDown() - - def test_validate_valid_config(self): - self.assertEqual(self.valid_config, self.ks._config) - - def test_validate_config_extra_param(self): - self.assertRaises( - voluptuous.Invalid, - self.ks.validate_config, - self.config_extra_param) - - def test_validate_config_wrong_type(self): - self.assertRaises( - voluptuous.Invalid, - self.ks.validate_config, - self.config_wrong_type) - - def test_validate_config_missing_params(self): - self.assertRaises( - voluptuous.Invalid, - self.ks.validate_config, - self.config_missing_params) - - def test_get_default_config(self): - default_config = kafka.KafkaSource.get_default_config() - kafka.KafkaSource.validate_config(default_config) - self.assertEqual("KafkaSource", default_config["module"]) - - def test_before_bind_source_dstream_created(self): - ssc = spark_mocks.MockStreamingContext(None, None) - self.assertIsNotNone(self.ks.create_dstream(ssc)) diff --git a/test/source/test_markov_chain.py b/test/source/test_markov_chain.py deleted file mode 100644 index 60bed5c..0000000 --- a/test/source/test_markov_chain.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import voluptuous - -from monasca_analytics.source import cloud_markov_chain as cloud -from test.mocks import spark_mocks -from test.util_for_testing import MonanasTestCase - - -class MarkovChainSourceTest(MonanasTestCase): - - def setUp(self): - super(MarkovChainSourceTest, self).setUp() - transitions = { - "web_service": { - "run=>slow": { - 0: 0.01, - 8: 0.2, - 12: 0.7, - 14: 0.7, - 22: 0.3, - 24: 0.1 - }, - "slow=>run": { - 0: 1, - 8: 0.7, - 12: 0.1, - 14: 0.1, - 22: 0.8, - 24: 0.99 - }, - "stop=>run": 0.5 - }, - "switch": { - "on=>off": 0.1, - "off=>on": 0.5 - }, - "host": { - "on=>off": 0.5, - "off=>on": 0.1 - }, - } - self.valid_config = { - "module": "markov_chain_source", - "transitions": dict(transitions), - "triggers": { - "support": { - "get_called": { - 0: 0.1, - 8: 0.2, - 12: 0.8, - 14: 0.8, - 22: 0.5, - 24: 0.0 - } - } - }, - "sleep": 0.1, - "min_event_per_burst": 500, - "graph": { - "h1:host": ["s1"], - "h2:host": ["s1"], - "s1:switch": [], - "w1:web_service": ["h1"], - "w2:web_service": ["h2"] - } - } - self.config_extra_param = dict(self.valid_config) - self.config_extra_param["extra_param"] = "john doe" - self.config_missing_param = dict(self.valid_config) - self.config_missing_param["transitions"] = dict(transitions) - self.config_missing_param["transitions"].pop("host") - self.config_wrong_type = { - "module": 123, - "transitions": dict(self.valid_config["transitions"]), - "sleep": 0.1, - "min_event_per_burst": 500, - "graph": {} - } - self.mcs = cloud.CloudMarkovChainSource("fake_id", self.valid_config) - - def tearDown(self): - super(MarkovChainSourceTest, self).tearDown() - - def test_validate_valid_config(self): - self.assertEqual(self.valid_config, self.mcs._config) - - def test_validate_config_extra_param(self): - self.assertRaises( - voluptuous.Invalid, - self.mcs.validate_config, - self.config_extra_param) - - def test_validate_config_wrong_type(self): - self.assertRaises( - voluptuous.Invalid, - self.mcs.validate_config, - self.config_wrong_type) - - def test_get_default_config(self): - default_config = cloud.CloudMarkovChainSource.get_default_config() - cloud.CloudMarkovChainSource.validate_config(default_config) - self.assertEqual("CloudMarkovChainSource", default_config["module"]) - - def test_create_dstream_created(self): - ssc = spark_mocks.MockStreamingContext(None, None) - self.assertIsNotNone(self.mcs.create_dstream(ssc)) - self.mcs.terminate_source() - self.assertEqual(ssc._host, "localhost") - - def test_create_system(self): - [support_node] = self.mcs._create_system() - ws = support_node.dependencies - self.assertEqual(len(ws), 2) - ws1 = ws[0] - self.assertEqual(len(ws1.dependencies), 1) - ws2 = ws[1] - self.assertEqual(len(ws2.dependencies), 1) - hs1 = ws1.dependencies[0] - self.assertEqual(len(hs1.dependencies), 1) - hs2 = ws2.dependencies[0] - self.assertEqual(len(hs2.dependencies), 1) - self.assertEqual(hs1.dependencies[0], hs2.dependencies[0]) diff --git a/test/source/test_random.py b/test/source/test_random.py deleted file mode 100644 index 40b8a5d..0000000 --- a/test/source/test_random.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.source import randoms -from test.util_for_testing import MonanasTestCase - - -class TestRandomSource(MonanasTestCase): - - def setUp(self): - super(TestRandomSource, self).setUp() - - def tearDown(self): - super(TestRandomSource, self).tearDown() - - def test_get_default_config(self): - default_config = randoms.RandomSource.get_default_config() - randoms.RandomSource.validate_config(default_config) - self.assertEqual("RandomSource", default_config["module"]) diff --git a/test/spark/__init__.py b/test/spark/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/spark/test_aggregator.py b/test/spark/test_aggregator.py deleted file mode 100644 index b894ccd..0000000 --- a/test/spark/test_aggregator.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.spark import aggregator -from test.mocks import spark_mocks -from test.util_for_testing import MonanasTestCase - - -class BaseAggregatorTest(MonanasTestCase): - - def setUp(self): - super(BaseAggregatorTest, self).setUp() - self.da = AggregatorBasicChild(None) - self.set_mocks() - - def tearDown(self): - super(BaseAggregatorTest, self).tearDown() - - def set_mocks(self): - pass - - def test_aggregate_first_time(self): - dstream1 = spark_mocks.MockDStream(None, None, None) - self.da.accumulate_dstream_samples(dstream1) - self.assertEqual(dstream1, self.da._combined_stream) - self.assertEqual(0, self.da._combined_stream.join_cnt) - - def test_aggregate_with_no_smls(self): - dstream1 = spark_mocks.MockDStream(None, None, None) - self.da.accumulate_dstream_samples(dstream1) - - -class AggregatorBasicChild(aggregator.Aggregator): - - def __init__(self, driver): - super(AggregatorBasicChild, self).__init__(driver) - self.accumulation_logic_cnt = 0 - - def prepare_final_accumulate_stream_step(self): - super(AggregatorBasicChild, self)\ - .prepare_final_accumulate_stream_step() - self.accumulation_logic_cnt += 1 diff --git a/test/spark/test_driver.py b/test/spark/test_driver.py deleted file mode 100644 index 6bf5a89..0000000 --- a/test/spark/test_driver.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from monasca_analytics.config import const -import monasca_analytics.spark.driver as driver -import monasca_analytics.util.common_util as cu -from test.mocks import sml_mocks -from test.mocks import spark_mocks -from test.util_for_testing import MonanasTestCase - - -class DriverExecutorTest(MonanasTestCase): - - def setUp(self): - """ - Keep a copy of the original functions that will be mocked, then - mock them, reset variables, and initialize ML_Framework. - """ - super(DriverExecutorTest, self).setUp() - self._backup_functions() - self._mock_functions() - sml_mocks.sml_mocks.reset() - self.init_sml_config() - - def tearDown(self): - """ - Restore the potentially mocked functions to the original ones - """ - super(DriverExecutorTest, self).tearDown() - self._restore_functions() - sml_mocks.sml_mocks.reset_connections() - - def _backup_functions(self): - self.original_get_class_by_name = cu.get_class_by_name - self.original_SparkContext = driver.pyspark.SparkContext - self.original_StreamingContext = \ - driver.streamingctx.streaming.StreamingContext - self.original_Aggregator = driver.agg.Aggregator - - def _restore_functions(self): - cu.get_class_by_name = self.original_get_class_by_name - driver.pyspark.SparkContext = self.original_SparkContext - driver.streamingctx.streaming.StreamingContext = \ - self.original_StreamingContext - driver.agg.Aggregator = self.original_Aggregator - - def _mock_functions(self): - cu.get_class_by_name = sml_mocks.mock_get_class_by_name - driver.pyspark.SparkContext = spark_mocks.MockSparkContext - driver.streamingctx.streaming.StreamingContext = \ - spark_mocks.MockStreamingContext - driver.agg.Aggregator = sml_mocks.MockClass_aggr_module - - def init_sml_config(self): - """ - Initialize the ML_Framework object with the test_json config - """ - current_dir = os.path.dirname(__file__) - test_json_file = os.path.join(current_dir, - "../resources/test_json.json") - config = cu.parse_json_file(test_json_file) - self.mlf = driver.DriverExecutor(config) - - def assert_got_classes_by_name_once(self): - self.assertEqual(9, len(sml_mocks.sml_mocks.classes_got_by_name)) - self.assertIn(["src_module1", const.SOURCES], - sml_mocks.sml_mocks.classes_got_by_name) - self.assertIn(["src_module2", const.SOURCES], - sml_mocks.sml_mocks.classes_got_by_name) - self.assertIn(["IPTablesSource", const.SOURCES], - sml_mocks.sml_mocks.classes_got_by_name) - self.assertIn(["ingestor_module", const.INGESTORS], - sml_mocks.sml_mocks.classes_got_by_name) - self.assertIn(["sml_module", const.SMLS], - sml_mocks.sml_mocks.classes_got_by_name) - self.assertIn(["voter_module", const.VOTERS], - sml_mocks.sml_mocks.classes_got_by_name) - self.assertIn(["sink_module1", const.SINKS], - sml_mocks.sml_mocks.classes_got_by_name) - self.assertIn(["sink_module2", const.SINKS], - sml_mocks.sml_mocks.classes_got_by_name) - self.assertIn(["ldp_module1", const.LDPS], - sml_mocks.sml_mocks.classes_got_by_name) - - def assert_instantiated_classes_once(self): - for n in sml_mocks.sml_mocks.instantiated.keys(): - self.assertEqual(1, len(sml_mocks.sml_mocks.instantiated[n])) - - def assert_instantiated_no_classes(self): - for n in sml_mocks.sml_mocks.instantiated.keys(): - self.assertEqual(0, len(sml_mocks.sml_mocks.instantiated[n])) - - def assert_only_instantiated(self, name): - self.assertEqual(1, len(sml_mocks.sml_mocks.instantiated[name])) - for n in sml_mocks.sml_mocks.instantiated.keys(): - if n != name: - self.assertEqual(0, len(sml_mocks.sml_mocks.instantiated[n])) - - def assert_src_initialized(self, src): - self.assertEqual(src.get_feature_list_cnt, 1) - self.assertEqual(src.create_dstream_cnt, 1) - - def assert_src_termintated(self, src): - self.assertEqual(src.terminate_source_cnt, 1) - - def assert_ingestor(self, ing): - self.assertEqual(ing.map_dstream_cnt, 1) - - def assert_agg(self, agg): - self.assertEqual(agg.accumulate_dstream_samples_cnt, 1) - self.assertEqual(agg.append_sml_cnt, 1) - - def assert_sml(self, sml): - self.assertEqual(sml.learn_structure_cnt, 1) - - def assert_voter(self, voter): - self.assertEqual(voter.elect_structure_cnt, 1) - - def assert_sink(self, sink): - self.assertEqual(sink.sink_dstream_cnt + sink.sink_sml_cnt, 1) - - def assert_sink_dstream(self, sink): - self.assertEqual(sink.sink_dstream_cnt, 1) - - def assert_sink_ml(self, sink): - self.assertEqual(sink.sink_sml_cnt, 1) - - def assert_ldp(self, ldp): - self.assertEqual(ldp.map_dstream_cnt, 1) - - def test_driver_orchestration_at_creation(self): - """ - Tests that the Monanas constructor checks the config json file, - gets all the modules classes by name and instantiates them. - """ - self.assert_got_classes_by_name_once() - self.assert_instantiated_classes_once() - - def test_pipeline_connected(self): - self.mlf.start_pipeline() - self.assert_src_initialized( - sml_mocks.sml_mocks.instantiated["src_module1"][0]) - self.assert_src_initialized( - sml_mocks.sml_mocks.instantiated["src_module2"][0]) - self.assert_ingestor( - sml_mocks.sml_mocks.instantiated["ingestor_module"][0]) - self.assert_agg(self.mlf._orchestrator) - self.mlf._orchestrator.accumulate_dstream_samples( - spark_mocks.MockDStream(None, None, None)) - self.mlf._orchestrator.prepare_final_accumulate_stream_step() - self.assert_sml(sml_mocks.sml_mocks.instantiated["sml_module"][0]) - self.assert_voter(sml_mocks.sml_mocks.instantiated["voter_module"][0]) - self.assert_sink( - sml_mocks.sml_mocks.instantiated["sink_module1"][0]) - self.assert_sink_ml( - sml_mocks.sml_mocks.instantiated["sink_module1"][0]) - - def test_start_pipeline(self): - self.mlf.start_pipeline() - - def test_phase2(self): - self.mlf.start_pipeline() - self.assertEqual(1, self.mlf._ssc.started_cnt) - self.mlf._orchestrator.accumulate_dstream_samples( - spark_mocks.MockDStream(None, None, None)) - self.mlf._orchestrator.prepare_final_accumulate_stream_step() - self.assert_ldp( - sml_mocks.sml_mocks.instantiated["ldp_module1"][0]) - self.assert_sink( - sml_mocks.sml_mocks.instantiated["sink_module2"][0]) - self.assert_sink_dstream(sml_mocks.sml_mocks.instantiated[ - "sink_module2"][0]) - - def assert_stopped_streaming_state(self, ssc=None): - if ssc: - self.assertEqual(1, ssc.stopped_cnt) - - def test_stop_pipeline(self): - self.mlf.start_pipeline() - ssc = self.mlf._ssc - self.mlf.stop_pipeline() - self.assert_stopped_streaming_state(ssc) diff --git a/test/test_monanas.py b/test/test_monanas.py deleted file mode 100644 index f01c961..0000000 --- a/test/test_monanas.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from monasca_analytics.exception import monanas as err -import monasca_analytics.monanas as mnn -import monasca_analytics.spark.driver as driver -import monasca_analytics.util.common_util as cu -from test.mocks import sml_mocks -from test.mocks import spark_mocks -from test.util_for_testing import MonanasTestCase - - -class MonanasTest(MonanasTestCase): - - def setUp(self): - """ - Keep a copy of the original functions that will be mocked, then - mock them, reset variables, and initialize ML_Framework. - """ - super(MonanasTest, self).setUp() - self._backup_functions() - self._mock_functions() - sml_mocks.sml_mocks.reset() - self.init_sml_config() - - def tearDown(self): - """ - Restore the potentially mocked functions to the original ones - """ - super(MonanasTest, self).tearDown() - self._restore_functions() - - def _backup_functions(self): - self.original_kill = mnn.os.kill - self.original_get_class_by_name = cu.get_class_by_name - self.original_SparkContext = driver.pyspark.SparkContext - self.original_StreamingContext = \ - driver.streamingctx.streaming.StreamingContext - self.original_Aggregator = driver.agg.Aggregator - - def _restore_functions(self): - cu.get_class_by_name = self.original_get_class_by_name - mnn.os.kill = self.original_kill - driver.pyspark.SparkContext = self.original_SparkContext - driver.streamingctx.streaming.StreamingContext = \ - self.original_StreamingContext - driver.agg.Aggregator = self.original_Aggregator - - def _mock_functions(self): - cu.get_class_by_name = sml_mocks.mock_get_class_by_name - mnn.os.kill = sml_mocks.mock_kill - driver.pyspark.SparkContext = spark_mocks.MockSparkContext - driver.streamingctx.streaming.StreamingContext = \ - spark_mocks.MockStreamingContext - driver.agg.Aggregator = sml_mocks.MockClass_aggr_module - - def init_sml_config(self): - """ - Initialize the ML_Framework object with the test_json config - """ - current_dir = os.path.dirname(__file__) - test_json_file = os.path.join(current_dir, "resources/test_json.json") - config = cu.parse_json_file(test_json_file) - self.mlf = mnn.Monanas(config) - - def test_is_streaming(self): - self.assertFalse(self.mlf.is_streaming()) - self.mlf._is_streaming = True - self.assertTrue(self.mlf.is_streaming()) - self.mlf._is_streaming = False - self.assertFalse(self.mlf.is_streaming()) - - def test_start_streaming_no_param(self): - self.mlf.start_streaming() - self.assertTrue(self.mlf.is_streaming()) - - def assert_stopped_streaming_state(self, ssc=None): - if ssc: - self.assertEqual(1, ssc.stopped_cnt) - self.assertFalse(self.mlf.is_streaming()) - - def test_stop_streaming(self): - self.mlf.start_streaming() - self.mlf.stop_streaming() - - def test_stop_streaming_no_streaming(self): - self.mlf.start_streaming() - self.mlf.stop_streaming() - self.assertRaises(err.MonanasAlreadyStoppedStreaming, - self.mlf.stop_streaming) - - def test_stop_streaming_and_terminate_from_init_state(self): - self.assertFalse(sml_mocks.sml_mocks.killed) - self.mlf.stop_streaming_and_terminate() - self.assertTrue(sml_mocks.sml_mocks.killed) - self.assert_stopped_streaming_state() - - def test_stop_streaming_and_terminate_from_streaming_state(self): - self.assertFalse(sml_mocks.sml_mocks.killed) - self.mlf.start_streaming() - self.mlf.stop_streaming_and_terminate() - self.assertTrue(sml_mocks.sml_mocks.killed) - - def test_stop_streaming_and_terminate_from_stopped_state(self): - self.assertFalse(sml_mocks.sml_mocks.killed) - self.mlf.start_streaming() - self.mlf.stop_streaming() - self.mlf.stop_streaming_and_terminate() - self.assertTrue(sml_mocks.sml_mocks.killed) diff --git a/test/test_run.py b/test/test_run.py deleted file mode 100644 index 848ad99..0000000 --- a/test/test_run.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2018 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import run -from test.util_for_testing import MonanasTestCase - - -class ParserTest(MonanasTestCase): - """Test argment parser in run.py""" - - def setUp(self): - super(ParserTest, self).setUp() - self.parser = run.setup_parser() - - def tearDown(self): - super(ParserTest, self).tearDown() - - def _get_parser(self, args): - try: - parsed = self.parser.parse_args(args) - except SystemExit: - raise ParserException("Argument parse failed") - - return parsed - - def _check_parser(self, parsed, args, verify_args): - for av in verify_args: - attr, value = av - if attr: - self.assertIn(attr, parsed) - self.assertEqual(value, getattr(parsed, attr)) - - def test_parser_required(self): - arglist = [ - '--config', '/path/to/config_file', - '--log_config', '/path/to/log_file', - '--spark_path', '/path/to/spark', - ] - - verifylist = [ - ('config', '/path/to/config_file'), - ('log_config', '/path/to/log_file'), - ('spark_path', '/path/to/spark'), - ] - - parsed = self._get_parser(arglist) - self._check_parser(parsed, arglist, verifylist) - - def test_parser_optional(self): - arglist = [ - '--config', '/path/to/config_file', - '--log_config', '/path/to/log_file', - '--spark_path', '/path/to/spark', - '--sources', '/path/to/src1', '/path/to/src2', - ] - - verifylist = [ - ('config', '/path/to/config_file'), - ('log_config', '/path/to/log_file'), - ('spark_path', '/path/to/spark'), - ('sources', ['/path/to/src1', '/path/to/src2']), - ] - - parsed = self._get_parser(arglist) - self._check_parser(parsed, arglist, verifylist) - - def test_parser_optional_bool(self): - arglist = [ - '--config', '/path/to/config_file', - '--log_config', '/path/to/log_file', - '--spark_path', '/path/to/spark', - '--debug', - ] - - parsed = self._get_parser(arglist) - self.assertTrue(parsed.debug) diff --git a/test/util/__init__.py b/test/util/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/util/inheritance.py b/test/util/inheritance.py deleted file mode 100644 index 471d543..0000000 --- a/test/util/inheritance.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class Baseclass(object): - pass - - -class Extended_1_1(Baseclass): - pass - - -class Extended_1_2(Baseclass): - pass - - -class Extended_1_3(Baseclass): - pass - - -class Extended_2_1(Extended_1_1): - pass - - -class Extended_3_1(Extended_2_1): - pass diff --git a/test/util/test_common_util.py b/test/util/test_common_util.py deleted file mode 100644 index 91b7d83..0000000 --- a/test/util/test_common_util.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import six -import unittest - -from monasca_analytics.config import const -from monasca_analytics.exception import monanas as err -from monasca_analytics.sml import lingam -from monasca_analytics.source import kafka -from monasca_analytics.util import common_util -from monasca_analytics.voter import pick_index -from test import util -from test.util import inheritance as inh - - -class CommonUtilTest(unittest.TestCase): - - def test_parse_json_file(self): - current_dir = os.path.dirname(__file__) - test_json_file = os.path.join(current_dir, - "../resources/test_json.json") - parsed_json = common_util.parse_json_file(test_json_file) - six.assertCountEqual(self, parsed_json["sources"]["src1"], - {"module": "src_module1", - "params": { - "param1": "val1", - "param2": "val2", - "model_id": 3} - }) - six.assertCountEqual(self, parsed_json["ingestors"]["ing1"], - {"module": "ingestor_module"}) - six.assertCountEqual(self, parsed_json["smls"]["sml1"], - {"module": "sml_module"}) - self.assertEqual(parsed_json["voters"]["vot1"], - {"module": "voter_module"}) - six.assertCountEqual(self, parsed_json["sinks"]["snk1"], - {"module": "sink_module1"}) - six.assertCountEqual(self, parsed_json["sinks"]["snk2"], - {"module": "sink_module2"}) - six.assertCountEqual(self, parsed_json["ldps"]["ldp1"], - {"module": "ldps_module1"}) - six.assertCountEqual(self, parsed_json["connections"], - {"src1": ["ing1"], - "src2": ["ing1"], - "ing1": ["aggr1", "ldp1", "sin1"], - "snk1": [], - "snk2": [], - "sml1": ["vot1", "snk1"], - "vot1": ["ldp1", "snk1"], - "ldp1": ["snk2"]}) - six.assertCountEqual(self, parsed_json["feedback"], - {"snk1": ["sml1"], - "snk2": ["vot1"]}) - - def test_get_class_by_name(self): - common_util.get_class_by_name("RandomSource", const.SOURCES) - - def test_get_class_by_name_no_such_class(self): - self.assertRaises(err.MonanasNoSuchClassError, - common_util.get_class_by_name, - "InventedSource", - const.SOURCES) - - def test_get_available_inherited_classes(self): - children = common_util.get_available_inherited_classes(util, - inh.Baseclass) - classes = [source_class.__name__ for source_class in children] - six.assertCountEqual(self, classes, - ["Extended_1_1", "Extended_1_2", - "Extended_1_3", "Extended_2_1", "Extended_3_1"]) - - def test_get_source_class_by_name(self): - clazz = common_util.get_source_class_by_name("KafkaSource") - self.assertEqual(clazz, kafka.KafkaSource) - - def test_get_available_source_class_names(self): - names = common_util.get_available_source_class_names() - six.assertCountEqual( - self, - ['RandomSource', 'KafkaSource', - 'CloudMarkovChainSource', 'IPTablesSource', - 'MonascaMarkovChainSource'], - names) - - def test_get_available_ingestor_class_names(self): - names = common_util.get_available_ingestor_class_names() - six.assertCountEqual( - self, - ['CloudIngestor', 'IptablesIngestor'], - names) - - def test_get_sml_class_by_name(self): - clazz = common_util.get_sml_class_by_name( - "LiNGAM") - self.assertEqual(clazz, lingam.LiNGAM) - - def test_get_available_sml_class_names(self): - names = common_util.get_available_sml_class_names() - six.assertCountEqual( - self, - ['LiNGAM', - 'SvmOneClass', - 'IsolationForest', - 'EllipticEnvelope', - 'DecisionTreeClassifier', - 'LogisticRegression', - 'RandomForestClassifier', - 'Svc'], - names) - - def test_get_voter_class_by_name(self): - clazz = common_util.get_voter_class_by_name( - "PickIndexVoter") - self.assertEqual(clazz, pick_index.PickIndexVoter) - - def test_get_available_voter_class_names(self): - names = common_util.get_available_voter_class_names() - six.assertCountEqual(self, ["PickIndexVoter"], names) - - def test_get_available_ldp_class_names(self): - names = common_util.get_available_ldp_class_names() - six.assertCountEqual(self, [ - "CloudCausalityLDP", "IptablesLDP", - 'MonascaDerivativeLDP', 'MonascaAggregateLDP', - 'MonascaCombineLDP' - ], names) diff --git a/test/util/test_config_model.py b/test/util/test_config_model.py deleted file mode 100644 index 66cea0b..0000000 --- a/test/util/test_config_model.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os - -import voluptuous - -from monasca_analytics.config import validation -from monasca_analytics.util import common_util -from test.util_for_testing import MonanasTestCase - -logger = logging.getLogger(__name__) - - -class TestConfigModel(MonanasTestCase): - - def get_config(self): - current_dir = os.path.dirname(__file__) - test_json_file = os.path.join(current_dir, - "../resources/test_json.json") - return common_util.parse_json_file(test_json_file) - - def setUp(self): - super(TestConfigModel, self).setUp() - self.comp_types = ["sources", "ingestors", "smls", - "voters", "ldps", "sinks"] - self.config = self.get_config() - - def test_validate_config_valid(self): - ret = validation.validate_config(self.config) - self.assertIsNone(ret) - - def test_validate_config_missing_key(self): - for key in self.comp_types: - del self.config[key] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - - def test_validate_config_extra_key(self): - self.config = self.get_config() - self.config["infiltrated"] = "I should not exist" - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - - def test_validate_config_missing_spark_key(self): - for key in list(self.config["spark_config"].keys()): - del self.config["spark_config"][key] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - del self.config["spark_config"]["streaming"]["batch_interval"] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - - def test_validate_config_spark_wrong_format(self): - self.config["spark_config"]["streaming"][ - "batch_interval"] = "I should not be a string" - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - self.config["spark_config"]["appName"] = 123 - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - - def test_validate_config_server_wrong_format(self): - self.config["server"]["port"] = "I should be an int" - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - self.config["server"]["debug"] = 52 - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - - def test_validate_config_spark_extra_parameters(self): - self.config["spark_config"]["infiltrated"] = "I should not exist" - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - self.config["spark_config"]["streaming"][ - "infiltrated"] = "I should not exist" - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - - def test_validate_config_server_extra_parameters(self): - self.config["server"]["infiltrated"] = "I should not exist" - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - - def test_validate_config_wrong_format_components(self): - for key in self.comp_types: - self.config[key] = ["I", "should", "be", "a", "dictionary"] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - for comp_id in self.config[key].keys(): - self.config[key][comp_id] = ["I", "should", "be", "a", "dict"] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - - def test_validate_config_wrong_format_connections(self): - self.config["connections"] = ["I", "should", "be", "a", "dictionary"] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - for comp_id in self.config["connections"].keys(): - self.config["connections"][comp_id] = {"I": "should", - "be": "a list"} - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - - def test_validate_connections_data_models(self): - self.config["connections"]["mod1"] = ["src1"] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - - def test_validate_connections_wrong_dest(self): - wrong_destinations = { - "src1": ["src1", "src2", "agg1", "sml1", - "vot1", "sin1", "sin2"], - "src2": ["src1", "src2", "agg1", "sml1", - "vot1", "sin1", "sin2"], - "ing1": ["src1", "src2", "ing1", "sml1", "vot1", "ldp1"], - "agg1": ["src1", "src2", "ing1", "agg1", - "vot1", "ldp1"], - "sml1": ["src1", "src2", "ing1", "sml1", - "ldp1"], - "vot1": ["src1", "src2", "ing1", "agg1", "sml1", "vot1"], - "sin1": ["src1", "src2", "ing1", "agg1", "sml1", - "vot1", "sin1", "sin2", "ldp1"], - "sin2": ["src1", "src2", "ing1", "agg1", "sml1", - "vot1", "sin1", "sin2", "ldp1"], - "ldp1": ["src1", "src2", "ing1", "agg1", "sml1", - "vot1", "ldp1"] - } - for from_id in wrong_destinations.keys(): - for dst_id in wrong_destinations[from_id]: - self.config["connections"][from_id] = [dst_id] - logger.debug("checking wrong connection: " + - from_id + " --> " + dst_id) - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - - def test_validate_connections_inexisteng_source(self): - self.config["connections"]["inex"] = ["sin2"] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() - - def test_validate_connections_inexisteng_dest(self): - self.config["connections"]["src1"] = ["inex"] - self.assertRaises(voluptuous.Invalid, - validation.validate_config, self.config) - self.config = self.get_config() diff --git a/test/util/test_math.py b/test/util/test_math.py deleted file mode 100644 index 774aa9d..0000000 --- a/test/util/test_math.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.util import math -from test.util_for_testing import MonanasTestCase - - -class MathTest(MonanasTestCase): - - def setUp(self): - super(MathTest, self).setUp() - - def tearDown(self): - super(MathTest, self).tearDown() - - def test_interp1(self): - table = [(0, 1.), (1, 1.), (2, 3)] - self.assertAlmostEqual(math.interpolate_1d(table, 0), 1.) - self.assertAlmostEqual(math.interpolate_1d(table, 0.), 1.) - self.assertAlmostEqual(math.interpolate_1d(table, 1.), 1.) - self.assertAlmostEqual(math.interpolate_1d(table, 0.5), 1.) - self.assertAlmostEqual(math.interpolate_1d(table, 0.7), 1.) - self.assertAlmostEqual(math.interpolate_1d(table, 2.), 3.) - self.assertAlmostEqual(math.interpolate_1d(table, 1.5), 2.) diff --git a/test/util_for_testing.py b/test/util_for_testing.py deleted file mode 100644 index db11692..0000000 --- a/test/util_for_testing.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import logging.config -import os -import unittest - - -class MonanasTestCase(unittest.TestCase): - - def setup_logging(self): - current_dir = os.path.dirname(__file__) - logging_config_file = os.path.join(current_dir, - "./resources/logging.json") - with open(logging_config_file, "rt") as f: - config = json.load(f) - logging.config.dictConfig(config) - - def setUp(self): - unittest.TestCase.setUp(self) - self.setup_logging() - - def tearDown(self): - unittest.TestCase.tearDown(self) - - -def gen_metric(name, value, timestamp, host): - return { - "metric": { - "dimensions": {"hostname": host}, - "timestamp": timestamp, - "name": name, - "value": value - }, - "meta": {}, - } diff --git a/test/voter/__init__.py b/test/voter/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/voter/test_base_voter.py b/test/voter/test_base_voter.py deleted file mode 100644 index 8b20707..0000000 --- a/test/voter/test_base_voter.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.voter import base -from test.util_for_testing import MonanasTestCase - - -class BaseVoterTest(MonanasTestCase): - - def setUp(self): - super(BaseVoterTest, self).setUp() - self.vot = VoterBasicChild("fake_id", "fake_config") - - def tearDown(self): - super(BaseVoterTest, self).tearDown() - - def test_suggest_structure_no_smls_or_structures(self): - self.vot.suggest_structure("who", "struct") - - -class VoterBasicChild(base.BaseVoter): - - def __init__(self, _id, _config): - self.elect_cnt = 0 - super(VoterBasicChild, self).__init__(_id, _config) - - def elect_structure(self, structures): - self.elect_cnt += 1 - - @staticmethod - def validate_config(_config): - pass - - @staticmethod - def get_default_config(): - return {"module": VoterBasicChild.__name__} - - @staticmethod - def get_params(): - return [] diff --git a/test/voter/test_pick_index.py b/test/voter/test_pick_index.py deleted file mode 100644 index 8806a9c..0000000 --- a/test/voter/test_pick_index.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_analytics.voter import pick_index -from test.util_for_testing import MonanasTestCase - - -class TestPickIndexVoter(MonanasTestCase): - - def setUp(self): - super(TestPickIndexVoter, self).setUp() - - def tearDown(self): - super(TestPickIndexVoter, self).tearDown() - - def test_get_default_config(self): - default_config = pick_index.PickIndexVoter.get_default_config() - pick_index.PickIndexVoter.validate_config(default_config) - self.assertEqual("PickIndexVoter", default_config["module"]) diff --git a/tools/test-setup.sh b/tools/test-setup.sh deleted file mode 100755 index 3e2fec8..0000000 --- a/tools/test-setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -xe - -# This script will be run by OpenStack CI before unit tests are run, -# it sets up the test system as needed. -# Developers should setup their test systems in a similar way. - -HOME=${HOME:-/home/jenkins} -SPARK_DIR=$HOME/spark -SPARK_VERSION=${SPARK_VERSION:-2.4.4} -SPARK_TARBALL_NAME=spark-$SPARK_VERSION.tgz -SPARK_URL=http://archive.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK_TARBALL_NAME - -mkdir -p $SPARK_DIR -curl $SPARK_URL -o $SPARK_DIR/$SPARK_TARBALL_NAME -tar -xzf $SPARK_DIR/$SPARK_TARBALL_NAME -C $SPARK_DIR diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 7c48b4a..0000000 --- a/tox.ini +++ /dev/null @@ -1,51 +0,0 @@ -[tox] -minversion = 2.0 -skipsdist = True -envlist = py37,pep8,cover - -[testenv] -basepython = python3 -usedevelop = True -setenv = - PYTHONUNBUFFERED=1 - VIRTUAL_ENV={envdir} - DISCOVER_DIRECTORY=tests - PYTHONPATH={homedir}/spark/spark-2.4.4/python:{homedir}/spark/spark-2.4.4/python/lib/py4j-0.10.7-src.zip: -install_command = pip install -U {opts} {packages} -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -whitelist_externals = bash - find -commands = - find ./ -type f -name "*.py[c|o]" -delete - nosetests -w test - -[testenv:cover] -commands = nosetests -w test --cover-package monasca_analytics -setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_COVERAGE=1 - NOSE_COVER_BRANCHES=1 - NOSE_COVER_HTML=1 - NOSE_COVER_HTML_DIR={toxinidir}/cover - PYTHONPATH={homedir}/spark/spark-2.4.4/python:{homedir}/spark/spark-2.4.4/python/lib/py4j-0.10.7-src.zip: - -[testenv:pep8] -commands = - flake8 monasca_analytics test - -[testenv:genconfig] - -[testenv:docs] - -[testenv:venv] -install_command = pip install -U {opts} {packages} -commands = {posargs} - -[flake8] -ignore = F821,H201,H404,H405,E305 -max-complexity = 50 -builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build -show-source = True - -[hacking]