Browse Source

seeding the project

Change-Id: I4235f6e837c85a062fda3fad188cd9339f15243d
Tong Li 3 years ago
parent
commit
163c121232
100 changed files with 11978 additions and 0 deletions
  1. 7
    0
      .coveragerc
  2. 58
    0
      .gitignore
  3. 8
    0
      .testr.conf
  4. 12
    0
      AUTHORS
  5. 7
    0
      CHANGELOG
  6. 13
    0
      CONTRIBUTING.md
  7. 201
    0
      LICENSE
  8. 136
    0
      README.md
  9. 40
    0
      _60_monitoring.py
  10. 2
    0
      babel.cfg
  11. 56
    0
      etc/alarms-persister.conf
  12. 17
    0
      etc/dashboard.ini
  13. 20
    0
      etc/default-mapping.json
  14. 62
    0
      etc/kiloeyes-notification-engine.conf
  15. 74
    0
      etc/kiloeyes-threshold-engine.conf
  16. 83
    0
      etc/kiloeyes.conf
  17. 25
    0
      etc/kiloeyes.ini
  18. 57
    0
      etc/metrics-persister.conf
  19. 0
    0
      kiloeyes/__init__.py
  20. 0
    0
      kiloeyes/api/__init__.py
  21. 138
    0
      kiloeyes/api/monasca_api_v2.py
  22. 71
    0
      kiloeyes/api/server.py
  23. 0
    0
      kiloeyes/common/__init__.py
  24. 91
    0
      kiloeyes/common/alarm_expr_calculator.py
  25. 320
    0
      kiloeyes/common/alarm_expr_parser.py
  26. 76
    0
      kiloeyes/common/alarm_expr_validator.py
  27. 87
    0
      kiloeyes/common/email_sender.py
  28. 142
    0
      kiloeyes/common/es_conn.py
  29. 212
    0
      kiloeyes/common/kafka_conn.py
  30. 18
    0
      kiloeyes/common/namespace.py
  31. 115
    0
      kiloeyes/common/resource_api.py
  32. 27
    0
      kiloeyes/dispatcher/__init__.py
  33. 79
    0
      kiloeyes/dispatcher/sample_dispatcher.py
  34. 0
    0
      kiloeyes/microservice/__init__.py
  35. 109
    0
      kiloeyes/microservice/es_persister.py
  36. 39
    0
      kiloeyes/microservice/fixed_strategy.py
  37. 58
    0
      kiloeyes/microservice/metrics_fixer.py
  38. 96
    0
      kiloeyes/microservice/notification_engine.py
  39. 132
    0
      kiloeyes/microservice/notification_processor.py
  40. 62
    0
      kiloeyes/microservice/server.py
  41. 346
    0
      kiloeyes/microservice/threshold_engine.py
  42. 328
    0
      kiloeyes/microservice/threshold_processor.py
  43. 110
    0
      kiloeyes/microservice/timed_strategy.py
  44. 0
    0
      kiloeyes/middleware/__init__.py
  45. 49
    0
      kiloeyes/middleware/inspector.py
  46. 45
    0
      kiloeyes/middleware/login.py
  47. 123
    0
      kiloeyes/middleware/metric_validator.py
  48. 0
    0
      kiloeyes/openstack/__init__.py
  49. 17
    0
      kiloeyes/openstack/common/__init__.py
  50. 145
    0
      kiloeyes/openstack/common/eventlet_backdoor.py
  51. 113
    0
      kiloeyes/openstack/common/excutils.py
  52. 146
    0
      kiloeyes/openstack/common/fileutils.py
  53. 0
    0
      kiloeyes/openstack/common/fixture/__init__.py
  54. 85
    0
      kiloeyes/openstack/common/fixture/config.py
  55. 51
    0
      kiloeyes/openstack/common/fixture/lockutils.py
  56. 34
    0
      kiloeyes/openstack/common/fixture/logging.py
  57. 62
    0
      kiloeyes/openstack/common/fixture/mockpatch.py
  58. 43
    0
      kiloeyes/openstack/common/fixture/moxstubout.py
  59. 479
    0
      kiloeyes/openstack/common/gettextutils.py
  60. 73
    0
      kiloeyes/openstack/common/importutils.py
  61. 202
    0
      kiloeyes/openstack/common/jsonutils.py
  62. 45
    0
      kiloeyes/openstack/common/local.py
  63. 322
    0
      kiloeyes/openstack/common/lockutils.py
  64. 713
    0
      kiloeyes/openstack/common/log.py
  65. 147
    0
      kiloeyes/openstack/common/loopingcall.py
  66. 512
    0
      kiloeyes/openstack/common/service.py
  67. 311
    0
      kiloeyes/openstack/common/strutils.py
  68. 106
    0
      kiloeyes/openstack/common/systemd.py
  69. 147
    0
      kiloeyes/openstack/common/threadgroup.py
  70. 210
    0
      kiloeyes/openstack/common/timeutils.py
  71. 39
    0
      kiloeyes/service.py
  72. 85
    0
      kiloeyes/tests/__init__.py
  73. 0
    0
      kiloeyes/tests/common/__init__.py
  74. 73
    0
      kiloeyes/tests/common/test_alarm_expr_calculator.py
  75. 201
    0
      kiloeyes/tests/common/test_alarm_expr_parser.py
  76. 78
    0
      kiloeyes/tests/common/test_alarm_expr_validator.py
  77. 164
    0
      kiloeyes/tests/common/test_case_alarm_expr_validator.json
  78. 96
    0
      kiloeyes/tests/common/test_email_sender.py
  79. 59
    0
      kiloeyes/tests/common/test_es_conn.py
  80. 0
    0
      kiloeyes/tests/microservice/__init__.py
  81. 828
    0
      kiloeyes/tests/microservice/test_case_threshold_processor.json
  82. 59
    0
      kiloeyes/tests/microservice/test_metrics_fixer.py
  83. 108
    0
      kiloeyes/tests/microservice/test_notification_processor.py
  84. 147
    0
      kiloeyes/tests/microservice/test_strategy.py
  85. 212
    0
      kiloeyes/tests/microservice/test_threshold_engine.py
  86. 249
    0
      kiloeyes/tests/microservice/test_threshold_processor.py
  87. 103
    0
      kiloeyes/tests/setup_metrics.py
  88. 0
    0
      kiloeyes/tests/v2/__init__.py
  89. 0
    0
      kiloeyes/tests/v2/elasticsearch/__init__.py
  90. 438
    0
      kiloeyes/tests/v2/elasticsearch/test_alarmdefinitions.py
  91. 82
    0
      kiloeyes/tests/v2/elasticsearch/test_alarmdefinitions_data
  92. 281
    0
      kiloeyes/tests/v2/elasticsearch/test_alarms.py
  93. 116
    0
      kiloeyes/tests/v2/elasticsearch/test_alarms_data
  94. 212
    0
      kiloeyes/tests/v2/elasticsearch/test_get_alarms_data
  95. 343
    0
      kiloeyes/tests/v2/elasticsearch/test_metrics.py
  96. 240
    0
      kiloeyes/tests/v2/elasticsearch/test_notificationmethods.py
  97. 51
    0
      kiloeyes/tests/v2/elasticsearch/test_versions.py
  98. 0
    0
      kiloeyes/v2/__init__.py
  99. 0
    0
      kiloeyes/v2/elasticsearch/__init__.py
  100. 0
    0
      kiloeyes/v2/elasticsearch/alarmdefinitions.py

+ 7
- 0
.coveragerc View File

@@ -0,0 +1,7 @@
1
+[run]
2
+branch = True
3
+source = kiloeyes
4
+omit = kiloeyes/tests/*,kiloeyes/openstack/*
5
+
6
+[report]
7
+ignore-errors = True

+ 58
- 0
.gitignore View File

@@ -0,0 +1,58 @@
1
+# Byte-compiled / optimized / DLL files
2
+__pycache__/
3
+*.py[cod]
4
+
5
+# C extensions
6
+*.so
7
+
8
+# Distribution / packaging
9
+.Python
10
+.testrepository/
11
+.venv/
12
+env/
13
+bin/
14
+build/
15
+develop-eggs/
16
+dist/
17
+eggs/
18
+lib/
19
+lib64/
20
+parts/
21
+sdist/
22
+var/
23
+covhtml/
24
+cover/
25
+*.egg-info/
26
+.installed.cfg
27
+*.egg
28
+
29
+# Installer logs
30
+pip-log.txt
31
+pip-delete-this-directory.txt
32
+
33
+# Unit test / coverage reports
34
+htmlcov/
35
+.tox/
36
+.coverage
37
+.cache
38
+nosetests.xml
39
+coverage.xml
40
+
41
+# Translations
42
+*.mo
43
+
44
+# Mr Developer
45
+.mr.developer.cfg
46
+.project
47
+.pydevproject
48
+
49
+# Rope
50
+.ropeproject
51
+
52
+# Django stuff:
53
+*.log
54
+*.pot
55
+
56
+# Sphinx documentation
57
+docs/_build/
58
+

+ 8
- 0
.testr.conf View File

@@ -0,0 +1,8 @@
1
+[DEFAULT]
2
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
3
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
4
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
5
+             ${PYTHON:-python} -m subunit.run discover kiloeyes $LISTOPT $IDOPTION
6
+
7
+test_id_option=--load-list $IDFILE
8
+test_list_option=--list

+ 12
- 0
AUTHORS View File

@@ -0,0 +1,12 @@
1
+Maintainer
2
+----------
3
+OpenStack Foundation
4
+IRC: #openstack-kiloeyes on irc.freenode.net
5
+
6
+Original Authors
7
+----------------
8
+Tong Li (litong01@us.ibm.com)
9
+
10
+Contributors
11
+------------
12
+Tong Li (litong01@us.ibm.com)

+ 7
- 0
CHANGELOG View File

@@ -0,0 +1,7 @@
1
+kiloeyes (1.0)
2
+
3
+    * Initial project setup
4
+      Choose framework of wsgiref, pasteDeploy, falcon.
5
+      The server will be wsgiref server like any other OpenStack server
6
+      Use PasteDeploy to allow WSGI pipelines
7
+      Use Falcon framework to implement ReSTful API services

+ 13
- 0
CONTRIBUTING.md View File

@@ -0,0 +1,13 @@
1
+If you would like to contribute to the development of OpenStack,
2
+you must follow the steps in the "If you're a developer, start here"
3
+section of this page: [http://wiki.openstack.org/HowToContribute](http://wiki.openstack.org/HowToContribute#If_you.27re_a_developer.2C_start_here:)
4
+
5
+Once those steps have been completed, changes to OpenStack
6
+should be submitted for review via the Gerrit tool, following
7
+the workflow documented at [http://wiki.openstack.org/GerritWorkflow](http://wiki.openstack.org/GerritWorkflow).
8
+
9
+Gerrit is the review system used in the OpenStack projects.  We're sorry, but
10
+we won't be able to respond to pull requests submitted through GitHub.
11
+
12
+Bugs should be filed [on Launchpad](https://bugs.launchpad.net/kiloeyes),
13
+not in GitHub's issue tracker.

+ 201
- 0
LICENSE View File

@@ -0,0 +1,201 @@
1
+Apache License
2
+                           Version 2.0, January 2004
3
+                        http://www.apache.org/licenses/
4
+
5
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+   1. Definitions.
8
+
9
+      "License" shall mean the terms and conditions for use, reproduction,
10
+      and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+      "Licensor" shall mean the copyright owner or entity authorized by
13
+      the copyright owner that is granting the License.
14
+
15
+      "Legal Entity" shall mean the union of the acting entity and all
16
+      other entities that control, are controlled by, or are under common
17
+      control with that entity. For the purposes of this definition,
18
+      "control" means (i) the power, direct or indirect, to cause the
19
+      direction or management of such entity, whether by contract or
20
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+      outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+      "You" (or "Your") shall mean an individual or Legal Entity
24
+      exercising permissions granted by this License.
25
+
26
+      "Source" form shall mean the preferred form for making modifications,
27
+      including but not limited to software source code, documentation
28
+      source, and configuration files.
29
+
30
+      "Object" form shall mean any form resulting from mechanical
31
+      transformation or translation of a Source form, including but
32
+      not limited to compiled object code, generated documentation,
33
+      and conversions to other media types.
34
+
35
+      "Work" shall mean the work of authorship, whether in Source or
36
+      Object form, made available under the License, as indicated by a
37
+      copyright notice that is included in or attached to the work
38
+      (an example is provided in the Appendix below).
39
+
40
+      "Derivative Works" shall mean any work, whether in Source or Object
41
+      form, that is based on (or derived from) the Work and for which the
42
+      editorial revisions, annotations, elaborations, or other modifications
43
+      represent, as a whole, an original work of authorship. For the purposes
44
+      of this License, Derivative Works shall not include works that remain
45
+      separable from, or merely link (or bind by name) to the interfaces of,
46
+      the Work and Derivative Works thereof.
47
+
48
+      "Contribution" shall mean any work of authorship, including
49
+      the original version of the Work and any modifications or additions
50
+      to that Work or Derivative Works thereof, that is intentionally
51
+      submitted to Licensor for inclusion in the Work by the copyright owner
52
+      or by an individual or Legal Entity authorized to submit on behalf of
53
+      the copyright owner. For the purposes of this definition, "submitted"
54
+      means any form of electronic, verbal, or written communication sent
55
+      to the Licensor or its representatives, including but not limited to
56
+      communication on electronic mailing lists, source code control systems,
57
+      and issue tracking systems that are managed by, or on behalf of, the
58
+      Licensor for the purpose of discussing and improving the Work, but
59
+      excluding communication that is conspicuously marked or otherwise
60
+      designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+      "Contributor" shall mean Licensor and any individual or Legal Entity
63
+      on behalf of whom a Contribution has been received by Licensor and
64
+      subsequently incorporated within the Work.
65
+
66
+   2. Grant of Copyright License. Subject to the terms and conditions of
67
+      this License, each Contributor hereby grants to You a perpetual,
68
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+      copyright license to reproduce, prepare Derivative Works of,
70
+      publicly display, publicly perform, sublicense, and distribute the
71
+      Work and such Derivative Works in Source or Object form.
72
+
73
+   3. Grant of Patent License. Subject to the terms and conditions of
74
+      this License, each Contributor hereby grants to You a perpetual,
75
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+      (except as stated in this section) patent license to make, have made,
77
+      use, offer to sell, sell, import, and otherwise transfer the Work,
78
+      where such license applies only to those patent claims licensable
79
+      by such Contributor that are necessarily infringed by their
80
+      Contribution(s) alone or by combination of their Contribution(s)
81
+      with the Work to which such Contribution(s) was submitted. If You
82
+      institute patent litigation against any entity (including a
83
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+      or a Contribution incorporated within the Work constitutes direct
85
+      or contributory patent infringement, then any patent licenses
86
+      granted to You under this License for that Work shall terminate
87
+      as of the date such litigation is filed.
88
+
89
+   4. Redistribution. You may reproduce and distribute copies of the
90
+      Work or Derivative Works thereof in any medium, with or without
91
+      modifications, and in Source or Object form, provided that You
92
+      meet the following conditions:
93
+
94
+      (a) You must give any other recipients of the Work or
95
+          Derivative Works a copy of this License; and
96
+
97
+      (b) You must cause any modified files to carry prominent notices
98
+          stating that You changed the files; and
99
+
100
+      (c) You must retain, in the Source form of any Derivative Works
101
+          that You distribute, all copyright, patent, trademark, and
102
+          attribution notices from the Source form of the Work,
103
+          excluding those notices that do not pertain to any part of
104
+          the Derivative Works; and
105
+
106
+      (d) If the Work includes a "NOTICE" text file as part of its
107
+          distribution, then any Derivative Works that You distribute must
108
+          include a readable copy of the attribution notices contained
109
+          within such NOTICE file, excluding those notices that do not
110
+          pertain to any part of the Derivative Works, in at least one
111
+          of the following places: within a NOTICE text file distributed
112
+          as part of the Derivative Works; within the Source form or
113
+          documentation, if provided along with the Derivative Works; or,
114
+          within a display generated by the Derivative Works, if and
115
+          wherever such third-party notices normally appear. The contents
116
+          of the NOTICE file are for informational purposes only and
117
+          do not modify the License. You may add Your own attribution
118
+          notices within Derivative Works that You distribute, alongside
119
+          or as an addendum to the NOTICE text from the Work, provided
120
+          that such additional attribution notices cannot be construed
121
+          as modifying the License.
122
+
123
+      You may add Your own copyright statement to Your modifications and
124
+      may provide additional or different license terms and conditions
125
+      for use, reproduction, or distribution of Your modifications, or
126
+      for any such Derivative Works as a whole, provided Your use,
127
+      reproduction, and distribution of the Work otherwise complies with
128
+      the conditions stated in this License.
129
+
130
+   5. Submission of Contributions. Unless You explicitly state otherwise,
131
+      any Contribution intentionally submitted for inclusion in the Work
132
+      by You to the Licensor shall be under the terms and conditions of
133
+      this License, without any additional terms or conditions.
134
+      Notwithstanding the above, nothing herein shall supersede or modify
135
+      the terms of any separate license agreement you may have executed
136
+      with Licensor regarding such Contributions.
137
+
138
+   6. Trademarks. This License does not grant permission to use the trade
139
+      names, trademarks, service marks, or product names of the Licensor,
140
+      except as required for reasonable and customary use in describing the
141
+      origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+   7. Disclaimer of Warranty. Unless required by applicable law or
144
+      agreed to in writing, Licensor provides the Work (and each
145
+      Contributor provides its Contributions) on an "AS IS" BASIS,
146
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+      implied, including, without limitation, any warranties or conditions
148
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+      PARTICULAR PURPOSE. You are solely responsible for determining the
150
+      appropriateness of using or redistributing the Work and assume any
151
+      risks associated with Your exercise of permissions under this License.
152
+
153
+   8. Limitation of Liability. In no event and under no legal theory,
154
+      whether in tort (including negligence), contract, or otherwise,
155
+      unless required by applicable law (such as deliberate and grossly
156
+      negligent acts) or agreed to in writing, shall any Contributor be
157
+      liable to You for damages, including any direct, indirect, special,
158
+      incidental, or consequential damages of any character arising as a
159
+      result of this License or out of the use or inability to use the
160
+      Work (including but not limited to damages for loss of goodwill,
161
+      work stoppage, computer failure or malfunction, or any and all
162
+      other commercial damages or losses), even if such Contributor
163
+      has been advised of the possibility of such damages.
164
+
165
+   9. Accepting Warranty or Additional Liability. While redistributing
166
+      the Work or Derivative Works thereof, You may choose to offer,
167
+      and charge a fee for, acceptance of support, warranty, indemnity,
168
+      or other liability obligations and/or rights consistent with this
169
+      License. However, in accepting such obligations, You may act only
170
+      on Your own behalf and on Your sole responsibility, not on behalf
171
+      of any other Contributor, and only if You agree to indemnify,
172
+      defend, and hold each Contributor harmless for any liability
173
+      incurred by, or claims asserted against, such Contributor by reason
174
+      of your accepting any such warranty or additional liability.
175
+
176
+   END OF TERMS AND CONDITIONS
177
+
178
+   APPENDIX: How to apply the Apache License to your work.
179
+
180
+      To apply the Apache License to your work, attach the following
181
+      boilerplate notice, with the fields enclosed by brackets "{}"
182
+      replaced with your own identifying information. (Don't include
183
+      the brackets!)  The text should be enclosed in the appropriate
184
+      comment syntax for the file format. We also recommend that a
185
+      file or class name and description of purpose be included on the
186
+      same "printed page" as the copyright notice for easier
187
+      identification within third-party archives.
188
+
189
+   Copyright {yyyy} {name of copyright owner}
190
+
191
+   Licensed under the Apache License, Version 2.0 (the "License");
192
+   you may not use this file except in compliance with the License.
193
+   You may obtain a copy of the License at
194
+
195
+       http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+   Unless required by applicable law or agreed to in writing, software
198
+   distributed under the License is distributed on an "AS IS" BASIS,
199
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+   See the License for the specific language governing permissions and
201
+   limitations under the License.

+ 136
- 0
README.md View File

@@ -0,0 +1,136 @@
1
+Kiloeyes
2
+=======
3
+
4
+Kiloeyes is a monitoring software allowing you to collect data from any compute
5
+systems.
6
+
7
+Install Prerequisites
8
+=====================
9
+
10
+Kiloeyes python implementation install process installs Kiloeyes and most of its
11
+dependencies automatically. However some components cannot be installed automatically
12
+by python setup tools, they will have to be installed manually. These components are
13
+python setup tools, python-dev, python-pip and gunicorn. Follow the steps below to
14
+install dependencies:
15
+
16
+The typical process of installing setup tools is to download the tar.gz file
17
+then tar -xvf and run python setup.py install, you can also reference this page:
18
+
19
+    https://pypi.python.org/pypi/setuptools
20
+
21
+To install python-dev and pip, run the following command:
22
+
23
+    sudo apt-get install python-dev python-pip
24
+
25
+To install gunicorn, run the following command:
26
+
27
+    sudo pip install gunicorn==19.1.0
28
+    
29
+Kiloeyes depends on Kafka and ElasticSearch, both requires Java. If you do not
30
+already have Java, Kafka and ElasticSearch running, you will have to install
31
+them. Please refer to respective document on how to install Java, Kafka and
32
+ElasticSearch::
33
+
34
+    http://www.java.com
35
+    http://kafka.apache.org/documentation.html#introduction
36
+    https://www.elastic.co/products/elasticsearch
37
+
38
+Install Kiloeyes
39
+===============
40
+Get the source code::
41
+
42
+    git clone https://github.com/openstack/kiloeyes.git
43
+
44
+Go to the root directory of the project and run the following command:
45
+
46
+    sudo python setup.py install
47
+
48
+If Kiloeyes installs successfully, you can then make changes to the following
49
+two files to reflect your system settings, such as Kafka server locations::
50
+
51
+    /etc/kiloeyes/kiloeyes.ini
52
+    /etc/kiloeyes/kiloeyes.conf
53
+
54
+Once the configurations are modified to match your environment, you can start
55
+up various services by following these instructions.
56
+
57
+To start the API server, run the following command:
58
+
59
+    Running the server in foreground mode
60
+    gunicorn -k eventlet --worker-connections=2000 --backlog=1000
61
+             --paste /etc/kiloeyes/kiloeyes.ini
62
+
63
+    Running the server as daemons
64
+    gunicorn -k eventlet --worker-connections=2000 --backlog=1000
65
+             --paste /etc/kiloeyes/kiloeyes.ini -D
66
+
67
+To start a Kiloeyes micro service servers, run the following command:
68
+
69
+    kiloeyes-service --config-file /etc/kiloeyes/xxxx.conf
70
+
71
+    where xxxx.conf should be a micro service specific configuration file.
72
+    Currently the following services are available:
73
+
74
+    Metrics persister service:
75
+    kiloeyes-service --config-file /etc/kiloeyes/metrics-persister.conf
76
+
77
+    Alarm persister service:
78
+    kiloeyes-service --config-file /etc/kiloeyes/alarms-persister.conf
79
+
80
+    Notification service:
81
+    kiloeyes-service --config-file /etc/kiloeyes/kiloeyes-notification-engine.conf
82
+
83
+    Threshold service:
84
+    kiloeyes-service --config-file /etc/kiloeyes/kiloeyes-threshold-engine.conf
85
+
86
+In the future, there might be other services such as threshold engine,
87
+anomaly detection, alarms etc. All these services should be able to take
88
+a specific configuration file to be launched. Here are the examples:
89
+
90
+    kiloeyes-service --config-file /etc/kiloeyes/kiloeyes-anomaly.conf
91
+
92
+If you are developing kiloeyes, and would like to run all the services in one
93
+screen and use default configurations, then you can simply run the following
94
+command from kiloeyes root direction:
95
+
96
+    screen -c kiloeyes
97
+
98
+
99
+Kiloeyes Integration with OpenStack Horizon
100
+==========================================
101
+To integrate with Horizon, two projects (monasca-ui and python-monascaclient)
102
+have to be installed. The steps on how to install these two projects can be
103
+found here::
104
+
105
+    https://github.com/stackforge/monasca-ui
106
+    https://github.com/stackforge/python-monascaclient
107
+
108
+Once both projects are installed, some configurations are needed:
109
+
110
+    Copy _60_monitoring.py to Horizon openstack_dashboard/local/enabled directory
111
+
112
+    Run the following command to create service and endpoint
113
+
114
+    setup_horizon.sh
115
+
116
+
117
+Kiloeyes Development
118
+===================
119
+To check if the code follows python coding style, run the following command
120
+from the root directory of this project
121
+
122
+    ./run_tests.sh -p
123
+
124
+To run all the unit test cases, run the following command from the root
125
+directory of this project
126
+
127
+    ./run_tests.sh
128
+
129
+To see the unit test case coverage, run the following command from the root
130
+directory of the project
131
+
132
+    ./run_tests.sh -c
133
+
134
+If the command runs successfully, then set of files will be created in the root
135
+directory named covhtml. Open up the index.html from a browser to see the summary
136
+of the unit test coverage and the details.

+ 40
- 0
_60_monitoring.py View File

@@ -0,0 +1,40 @@
1
+#!/usr/bin/env python
2
+#
3
+# Copyright 2013 IBM Corp
4
+##
5
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
6
+# not use this file except in compliance with the License. You may obtain
7
+# a copy of the License at
8
+#
9
+#      http://www.apache.org/licenses/LICENSE-2.0
10
+#
11
+# Unless required by applicable law or agreed to in writing, software
12
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14
+# License for the specific language governing permissions and limitations
15
+# under the License.
16
+# The name of the dashboard to be added to HORIZON['dashboards']. Required.
17
+DASHBOARD = "monitoring"
18
+
19
+# A list of applications to be added to INSTALLED_APPS.
20
+ADD_INSTALLED_APPS = ['monitoring']
21
+
22
+# A list of angular modules to be added as dependencies to horizon app.
23
+ADD_ANGULAR_MODULES = ['monitoringApp']
24
+
25
+# A list of javascript files to be included for all pages
26
+ADD_JS_FILES = ['monitoring/js/app.js',
27
+                'monitoring/js/controllers.js',
28
+                'monitoring/js/ng-tags-input.js']
29
+
30
+from monascaclient import exc
31
+# A dictionary of exception classes to be added to HORIZON['exceptions'].
32
+ADD_EXCEPTIONS = {
33
+    'recoverable': (exc.HTTPUnProcessable, exc.HTTPConflict,
34
+                    exc.HTTPException),
35
+    'not_found': (exc.HTTPNotFound,),
36
+    'unauthorized': (exc.HTTPUnauthorized,),
37
+}
38
+
39
+# If set to True, this dashboard will not be added to the settings.
40
+DISABLED = False

+ 2
- 0
babel.cfg View File

@@ -0,0 +1,2 @@
1
+[python: **.py]
2
+

+ 56
- 0
etc/alarms-persister.conf View File

@@ -0,0 +1,56 @@
1
+[DEFAULT]
2
+#logging, make sure that the user under whom the server runs has permission
3
+#to write to the directory.
4
+log_file=alarm_persister.log
5
+log_dir=/var/log/kiloeyes/
6
+log_level=DEBUG
7
+default_log_levels = kiloeyes=DEBUG
8
+
9
+service = es_persister
10
+threads = 3
11
+
12
+[es_persister]
13
+topic = alarms
14
+doc_type = alarms
15
+index_strategy = timed
16
+index_prefix = data_
17
+
18
+[timed_strategy]
19
+time_unit = m
20
+frequency = 1
21
+start_date = 2015-01-01
22
+
23
+[kafka_opts]
24
+#The endpoint to the kafka server, you can have multiple servers listed here
25
+#for example:
26
+#uri = 10.100.41.114:9092,10.100.41.115:9092,10.100.41.116:9092
27
+uri = 192.168.1.191:9092
28
+
29
+#consumer group name
30
+group = datapoints_group
31
+
32
+#how many times to try when error occurs
33
+max_retry = 1
34
+
35
+#wait time between tries when kafka goes down
36
+wait_time = 1
37
+
38
+#use synchronized or asynchronized connection to kafka
39
+async = False
40
+
41
+#send messages in bulk or send messages one by one.
42
+compact = False
43
+
44
+#How many partitions this connection should listen messages on, this
45
+#parameter is for reading from kafka. If listens on multiple partitions,
46
+#For example, if the client should listen on partitions 1 and 3, then the
47
+#configuration should look like the following:
48
+#   partitions = 1
49
+#   partitions = 3
50
+#default to listen on partition 0.
51
+partitions = 0
52
+
53
+[es_conn]
54
+uri = http://192.168.1.191:9200
55
+id_field = id
56
+drop_data = False

+ 17
- 0
etc/dashboard.ini View File

@@ -0,0 +1,17 @@
1
+[DEFAULT]
2
+monasca_root = /opt/kiloeyes
3
+
4
+[composite:main]
5
+use = egg:Paste#urlmap
6
+/ = home
7
+
8
+[app:home]
9
+use = egg:Paste#static
10
+document_root = %(monasca_root)s/ui
11
+
12
+[server:main]
13
+use = egg:gunicorn#main
14
+host = 0.0.0.0
15
+port = 9400
16
+workers = 3
17
+proc_name = kiloeyes-ui

+ 20
- 0
etc/default-mapping.json View File

@@ -0,0 +1,20 @@
1
+{"_default_":
2
+    {"dynamic_templates": 
3
+        [{"date_template":
4
+            {"match": "timestamp", 
5
+                "match_mapping_type": "date",
6
+                "mapping": 
7
+                    {"type": "date"}
8
+            }
9
+         },
10
+         {"string_template":
11
+            {"match": "*", 
12
+                "match_mapping_type": "string",
13
+                "mapping": 
14
+                    {"type": "string",
15
+                     "index": "not_analyzed"
16
+                    }
17
+            }
18
+         }]
19
+    }
20
+}

+ 62
- 0
etc/kiloeyes-notification-engine.conf View File

@@ -0,0 +1,62 @@
1
+[DEFAULT]
2
+#logging, make sure that the user under whom the server runs has permission
3
+#to write to the directory.
4
+log_file=notification.log
5
+log_dir=/var/log/kiloeyes/
6
+log_level=DEBUG
7
+default_log_levels = kiloeyes=DEBUG
8
+
9
+service = notification_engine
10
+threads = 3
11
+
12
+[notificationengine]
13
+topic = alarms
14
+doc_type = notificationmethods
15
+index_strategy = fixed
16
+index_prefix = admin
17
+processor = notification_processor
18
+
19
+[fixed_strategy]
20
+index_name =
21
+
22
+[mailsender]
23
+username = kiloeyes.notification@gmail.com
24
+password = notification
25
+smtp_host = smtp.gmail.com
26
+port = 25
27
+use_tls = true
28
+
29
+[kafka_opts]
30
+#The endpoint to the kafka server, you can have multiple servers listed here
31
+#for example:
32
+#uri = 10.100.41.114:9092,10.100.41.115:9092,10.100.41.116:9092
33
+uri = 192.168.1.191:9092
34
+
35
+#consumer group name
36
+group = datapoints_group
37
+
38
+#how many times to try when error occurs
39
+max_retry = 1
40
+
41
+#wait time between tries when kafka goes down
42
+wait_time = 1
43
+
44
+#use synchronized or asynchronized connection to kafka
45
+async = False
46
+
47
+#send messages in bulk or send messages one by one.
48
+compact = False
49
+
50
+#How many partitions this connection should listen messages on, this
51
+#parameter is for reading from kafka. If listens on multiple partitions,
52
+#For example, if the client should listen on partitions 1 and 3, then the
53
+#configuration should look like the following:
54
+#   partitions = 1
55
+#   partitions = 3
56
+#default to listen on partition 0.
57
+partitions = 0
58
+
59
+[es_conn]
60
+uri = http://192.168.1.191:9200
61
+time_id =
62
+drop_data = False

+ 74
- 0
etc/kiloeyes-threshold-engine.conf View File

@@ -0,0 +1,74 @@
1
+[DEFAULT]
2
+#logging, make sure that the user under whom the server runs has permission
3
+#to write to the directory.
4
+log_file=threshold.log
5
+log_dir=/var/log/kiloeyes/
6
+log_level=DEBUG
7
+default_log_levels = kiloeyes=DEBUG
8
+
9
+service = threshold_engine
10
+threads = 3
11
+
12
+[thresholdengine]
13
+metrics_topic = metrics
14
+alarm_topic = alarms
15
+processor = threshold_processor
16
+check_alarm_interval = 60
17
+
18
+[alarmdefinitions]
19
+doc_type = alarmdefinitions
20
+index_strategy = fixed
21
+index_prefix = admin
22
+size = 10000
23
+
24
+#query param includes name and dimensions
25
+#GET /v2.0/alarm-definitions?name=CPU percent greater than 10&dimensions=hostname:devstack,os=linux
26
+
27
+#the alarm definition name filter
28
+#for example, name = cpu, only start thresh engine with alarm defs who are about cpu
29
+#name =, set no filter
30
+name =
31
+
32
+#dimension key/value pairs filter
33
+#for example, dimensions = hostname:devstack,os=linux,
34
+#only start engine with alarm defs whose have the related dimensions
35
+dimensions =
36
+
37
+#the time interval to retrieve the latest alarm definitions
38
+check_alarm_def_interval = 120
39
+
40
+[kafka_opts]
41
+#The endpoint to the kafka server, you can have multiple servers listed here
42
+#for example:
43
+#uri = 10.100.41.114:9092,10.100.41.115:9092,10.100.41.116:9092
44
+
45
+uri = 192.168.1.191:9092
46
+
47
+#consumer group name
48
+group = datapoints_group
49
+
50
+#how many times to try when error occurs
51
+max_retry = 1
52
+
53
+#wait time between tries when kafka goes down
54
+wait_time = 1
55
+
56
+#use synchronized or asynchronized connection to kafka
57
+async = False
58
+
59
+#send messages in bulk or send messages one by one.
60
+compact = False
61
+
62
+#How many partitions this connection should listen messages on, this
63
+#parameter is for reading from kafka. If listens on multiple partitions,
64
+#For example, if the client should listen on partitions 1 and 3, then the
65
+#configuration should look like the following:
66
+#   partitions = 1
67
+#   partitions = 3
68
+#default to listen on partition 0.
69
+partitions = 0
70
+
71
+[es_conn]
72
+uri = http://192.168.1.191:9200
73
+time_id = timestamp
74
+drop_data = False

+ 83
- 0
etc/kiloeyes.conf View File

@@ -0,0 +1,83 @@
1
+[DEFAULT]
2
+#logging, make sure that the user under whom the server runs has permission
3
+#to write to the directory.
4
+log_file=api.log
5
+log_dir=/var/log/kiloeyes/
6
+log_level=DEBUG
7
+default_log_levels = kiloeyes=DEBUG
8
+
9
+dispatcher = metrics
10
+dispatcher = versions
11
+dispatcher = alarmdefinitions
12
+dispatcher = notificationmethods
13
+dispatcher = alarms
14
+
15
+[metrics]
16
+topic = metrics
17
+doc_type = metrics
18
+index_strategy = timed
19
+index_prefix = data_
20
+size = 10000
21
+
22
+[alarmdefinitions]
23
+doc_type = alarmdefinitions
24
+index_strategy = fixed
25
+index_prefix = admin
26
+size = 10000
27
+
28
+[notificationmethods]
29
+doc_type = notificationmethods
30
+index_strategy = fixed
31
+index_prefix = admin
32
+size = 10000
33
+
34
+[alarms]
35
+topic = alarms
36
+doc_type = alarms
37
+index_strategy = timed
38
+index_prefix = data_
39
+size = 10000
40
+
41
+[fixed_strategy]
42
+index_name = 
43
+
44
+[timed_strategy]
45
+time_unit = m
46
+frequency = 1
47
+start_date = 2015-01-01
48
+
49
+[kafka_opts]
50
+#The endpoint to the kafka server, you can have multiple servers listed here
51
+#for example:
52
+#uri = 10.100.41.114:9092,10.100.41.115:9092,10.100.41.116:9092
53
+
54
+uri = 192.168.1.191:9092
55
+
56
+#consumer group name
57
+group = datapoints_group
58
+
59
+#how many times to try when error occurs
60
+max_retry = 1
61
+
62
+#wait time between tries when kafka goes down
63
+wait_time = 1
64
+
65
+#use synchronized or asynchronized connection to kafka
66
+async = False
67
+
68
+#send messages in bulk or send messages one by one.
69
+compact = False
70
+
71
+#How many partitions this connection should listen messages on, this
72
+#parameter is for reading from kafka. If listens on multiple partitions,
73
+#For example, if the client should listen on partitions 1 and 3, then the
74
+#configuration should look like the following:
75
+#   partitions = 1
76
+#   partitions = 3
77
+#default to listen on partition 0.
78
+partitions = 0
79
+
80
+[es_conn]
81
+uri = http://192.168.1.191:9200
82
+time_id = timestamp
83
+drop_data = False

+ 25
- 0
etc/kiloeyes.ini View File

@@ -0,0 +1,25 @@
1
+[DEFAULT]
2
+name = kiloeyes
3
+
4
+[pipeline:main]
5
+# Add validator in the pipeline so the metrics messages can be validated.
6
+pipeline = api
7
+
8
+[app:api]
9
+paste.app_factory = kiloeyes.api.server:api_app
10
+
11
+[filter:login]
12
+use = egg: kiloeyes#login
13
+
14
+[filter:inspector]
15
+use = egg: kiloeyes#inspector
16
+
17
+[filter:validator]
18
+use = egg: kiloeyes#metric_validator
19
+
20
+[server:main]
21
+use = egg:gunicorn#main
22
+host = 0.0.0.0
23
+port = 9090
24
+workers = 1
25
+proc_name = kiloeyes-api

+ 57
- 0
etc/metrics-persister.conf View File

@@ -0,0 +1,57 @@
1
+[DEFAULT]
2
+#logging, make sure that the user under whom the server runs has permission
3
+#to write to the directory.
4
+log_file=metric_persister.log
5
+log_dir=/var/log/kiloeyes/
6
+log_level=DEBUG
7
+default_log_levels = kiloeyes=DEBUG
8
+
9
+service = es_persister
10
+threads = 3
11
+
12
+[es_persister]
13
+topic = metrics
14
+doc_type = metrics
15
+index_strategy = timed
16
+index_prefix = data_
17
+processor = metrics_msg_fixer
18
+
19
+[timed_strategy]
20
+time_unit = m
21
+frequency = 1
22
+start_date = 2015-01-01
23
+
24
+[kafka_opts]
25
+#The endpoint to the kafka server, you can have multiple servers listed here
26
+#for example:
27
+#uri = 10.100.41.114:9092,10.100.41.115:9092,10.100.41.116:9092
28
+uri = 192.168.1.191:9092
29
+
30
+#consumer group name
31
+group = datapoints_group
32
+
33
+#how many times to try when error occurs
34
+max_retry = 1
35
+
36
+#wait time between tries when kafka goes down
37
+wait_time = 1
38
+
39
+#use synchronized or asynchronized connection to kafka
40
+async = False
41
+
42
+#send messages in bulk or send messages one by one.
43
+compact = False
44
+
45
+#How many partitions this connection should listen messages on, this
46
+#parameter is for reading from kafka. If listens on multiple partitions,
47
+#For example, if the client should listen on partitions 1 and 3, then the
48
+#configuration should look like the following:
49
+#   partitions = 1
50
+#   partitions = 3
51
+#default to listen on partition 0.
52
+partitions = 0
53
+
54
+[es_conn]
55
+uri = http://192.168.1.191:9200
56
+time_id = timestamp
57
+drop_data = False

+ 0
- 0
kiloeyes/__init__.py View File


+ 0
- 0
kiloeyes/api/__init__.py View File


+ 138
- 0
kiloeyes/api/monasca_api_v2.py View File

@@ -0,0 +1,138 @@
1
+# Copyright 2013 IBM Corp
2
+##
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+from kiloeyes.common import resource_api
16
+from kiloeyes.openstack.common import log
17
+
18
+
19
+LOG = log.getLogger(__name__)
20
+
21
+
22
+class V2API(object):
23
+    def __init__(self, global_conf):
24
+        LOG.debug('initializing V2API!')
25
+        self.global_conf = global_conf
26
+
27
+    @resource_api.Restify('/', method='get')
28
+    def do_get_versions(self, req, res, version_id):
29
+        res.status = '501 Not Implemented'
30
+
31
+    @resource_api.Restify('/{version_id}', method='get')
32
+    def do_get_version_by_id(self, req, res, version_id):
33
+        res.status = '501 Not Implemented'
34
+
35
+    @resource_api.Restify('/v2.0/metrics/', method='post')
36
+    def do_post_metrics(self, req, res):
37
+        res.status = '501 Not Implemented'
38
+
39
+    # This is an extension to kiloeyes spec.
40
+    @resource_api.Restify('/v2.0/metrics/{id}', method='get')
41
+    def do_get_metrics_by_id(self, req, res, id):
42
+        res.status = '501 Not Implemented'
43
+
44
+    @resource_api.Restify('/v2.0/metrics/', method='get')
45
+    def do_get_metrics(self, req, res):
46
+        res.status = '501 Not Implemented'
47
+
48
+    @resource_api.Restify('/v2.0/metrics/measurements', method='get')
49
+    def do_get_measurements(self, req, res):
50
+        res.status = '501 Not Implemented'
51
+
52
+    @resource_api.Restify('/v2.0/metrics/names', method='get')
53
+    def do_get_metrics_names(self, req, res):
54
+        res.status = '501 Not Implemented'
55
+
56
+    @resource_api.Restify('/v2.0/metrics/statistics')
57
+    def do_get_statistics(self, req, res):
58
+        res.status = '501 Not Implemented'
59
+
60
+    # Notification-method APIs
61
+    @resource_api.Restify('/v2.0/notification-methods', method='post')
62
+    def do_post_notification_methods(self, req, res):
63
+        res.status = '501 Not Implemented'
64
+
65
+    @resource_api.Restify('/v2.0/notification-methods/', method='get')
66
+    def do_get_notification_methods(self, req, res):
67
+        res.status = '501 Not Implemented'
68
+
69
+    @resource_api.Restify('/v2.0/notification-methods/{id}', method='get')
70
+    def do_get_notification_method_by_id(self, req, res, id):
71
+        res.status = '501 Not Implemented'
72
+
73
+    @resource_api.Restify('/v2.0/notification-methods/{id}', method='put')
74
+    def do_put_notification_methods(self, req, res, id):
75
+        res.status = '501 Not Implemented'
76
+
77
+    @resource_api.Restify('/v2.0/notification-methods/{id}', method='delete')
78
+    def do_delete_notification_methods(self, req, res, id):
79
+        res.status = '501 Not Implemented'
80
+
81
+    # Alarm-definition APIs
82
+    @resource_api.Restify('/v2.0/alarm-definitions/', method='post')
83
+    def do_post_alarm_definitions(self, req, res):
84
+        res.status = '501 Not Implemented'
85
+
86
+    @resource_api.Restify('/v2.0/alarm-definitions/', method='get')
87
+    def do_get_alarm_definitions(self, req, res, id):
88
+        res.status = '501 Not Implemented'
89
+
90
+    @resource_api.Restify('/v2.0/alarm-definitions/{id}', method='get')
91
+    def do_get_alarm_definition_by_id(self, req, res, id):
92
+        res.status = '501 Not Implemented'
93
+
94
+    @resource_api.Restify('/v2.0/alarm-definitions/{id}', method='put')
95
+    def do_put_alarm_definition_by_id(self, req, res, id):
96
+        res.status = '501 Not Implemented'
97
+
98
+    @resource_api.Restify('/v2.0/alarm-definitions/{id}', method='patch')
99
+    def do_patch_alarm_definition_by_id(self, req, res, id):
100
+        res.status = '501 Not Implemented'
101
+
102
+    @resource_api.Restify('/v2.0/alarm-definitions/{id}', method='delete')
103
+    def do_delete_alarm_definition_by_id(self, req, res, id):
104
+        res.status = '501 Not Implemented'
105
+
106
+    # Alarm APIs
107
+    @resource_api.Restify('/v2.0/alarms/', method='get')
108
+    def do_get_alarms(self, req, res, id):
109
+        res.status = '501 Not Implemented'
110
+
111
+    @resource_api.Restify('/v2.0/alarms/state-history', method='get')
112
+    def do_get_alarms_state_history(self, req, res, id):
113
+        res.status = '501 Not Implemented'
114
+
115
+    @resource_api.Restify('/v2.0/alarms/{alarm_id}', method='get')
116
+    def do_get_alarm_by_id(self, req, res, id):
117
+        res.status = '501 Not Implemented'
118
+
119
+    @resource_api.Restify('/v2.0/alarms/{alarm_id}', method='put')
120
+    def do_put_alarms(self, req, res, id):
121
+        res.status = '501 Not Implemented'
122
+
123
+    @resource_api.Restify('/v2.0/alarms/{alarm_id}', method='patch')
124
+    def do_patch_alarms(self, req, res, id):
125
+        res.status = '501 Not Implemented'
126
+
127
+    @resource_api.Restify('/v2.0/alarms/{alarm_id}', method='delete')
128
+    def do_delete_alarms(self, req, res, id):
129
+        res.status = '501 Not Implemented'
130
+
131
+    # This is an extention to the API spec.
132
+    @resource_api.Restify('/v2.0/alarms', method='post')
133
+    def do_post_alarms(self, req, res):
134
+        res.status = '501 Not Implemented'
135
+
136
+    @resource_api.Restify('/v2.0/alarms/{alarm_id}/state-history')
137
+    def do_get_alarm_state_history(self, req, res, id):
138
+        res.status = '501 Not Implemented'

+ 71
- 0
kiloeyes/api/server.py View File

@@ -0,0 +1,71 @@
1
+# Copyright 2014 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+# http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import os
16
+from wsgiref import simple_server
17
+
18
+from oslo_config import cfg
19
+import paste.deploy
20
+from stevedore import named
21
+
22
+
23
+from kiloeyes.common import namespace
24
+from kiloeyes.common import resource_api
25
+from kiloeyes.openstack.common import log
26
+
27
+OPTS = [
28
+    cfg.MultiStrOpt('dispatcher',
29
+                    default=[],
30
+                    help='Dispatchers to process data.'),
31
+]
32
+cfg.CONF.register_opts(OPTS)
33
+
34
+LOG = log.getLogger(__name__)
35
+
36
+
37
+def api_app(conf):
38
+    cfg.CONF(args=[], project='kiloeyes')
39
+    log_levels = (cfg.CONF.default_log_levels)
40
+    cfg.set_defaults(log.log_opts, default_log_levels=log_levels)
41
+    log.setup('kiloeyes')
42
+
43
+    dispatcher_manager = named.NamedExtensionManager(
44
+        namespace=namespace.DISPATCHER_NS,
45
+        names=cfg.CONF.dispatcher,
46
+        invoke_on_load=True,
47
+        invoke_args=[cfg.CONF])
48
+
49
+    if not list(dispatcher_manager):
50
+        LOG.error('Failed to load any dispatchers for %s' %
51
+                  namespace.DISPATCHER_NS)
52
+        return None
53
+
54
+    # Create the application
55
+    app = resource_api.ResourceAPI()
56
+
57
+    # add each dispatcher to the application to serve requests offered by
58
+    # each dispatcher
59
+    for driver in dispatcher_manager:
60
+        app.add_route(None, driver.obj)
61
+
62
+    LOG.debug('Dispatcher drivers have been added to the routes!')
63
+    return app
64
+
65
+
66
+if __name__ == '__main__':
67
+    wsgi_app = (
68
+        paste.deploy.loadapp('config:etc/kiloeyes.ini',
69
+                             relative_to=os.getcwd()))
70
+    httpd = simple_server.make_server('127.0.0.1', 9000, wsgi_app)
71
+    httpd.serve_forever()

+ 0
- 0
kiloeyes/common/__init__.py View File


+ 91
- 0
kiloeyes/common/alarm_expr_calculator.py View File

@@ -0,0 +1,91 @@
1
+# Copyright 2015 Carnegie Mellon University
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+# http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+agg_ops = {'SUM': sum,
16
+           'AVG': lambda x: sum(x) / len(x),
17
+           'MAX': max,
18
+           'MIN': min,
19
+           'COUNT': len}
20
+
21
+
22
+comp_ops = {'GT': lambda x, y: x <= y,
23
+            'LT': lambda x, y: x >= y,
24
+            'LTE': lambda x, y: x > y,
25
+            'GTE': lambda x, y: x < y}
26
+
27
+STATE_OK = 'OK'
28
+STATE_ALARM = 'ALARM'
29
+STATE_UNDETERMINED = 'UNDETERMINED'
30
+
31
+
32
+def calc_value(func, data_list):
33
+    """Calc float values according to 5 functions."""
34
+
35
+    if (func not in agg_ops or
36
+            (len(data_list) == 0 and func != 'COUNT')):
37
+        return None
38
+    else:
39
+        return agg_ops[func](data_list)
40
+
41
+
42
+def compare_thresh(values, op, thresh):
43
+    """Check if value from metrics exceeds thresh.
44
+
45
+    Only the value in each period meet thresh, the alarm state can be 'ALARM'.
46
+
47
+    For example, the alarm definition defines 3 periods, values = [a,b,c].
48
+    If the value in any period doesn't meet thresh,
49
+    then alarm state must be 'OK';
50
+    If some values are None (means no metrics in that period)
51
+    but all other values meet thresh,
52
+    we still don't know if the alarm can be triggered,
53
+    so it's 'UNDETERMINED';
54
+    otherwise, the state can be 'ALARM'
55
+    """
56
+    for value in values:
57
+        if value is not None and comp_ops[op](value, thresh):
58
+            return STATE_OK
59
+
60
+    state = STATE_ALARM
61
+    for value in values:
62
+        if value is None:
63
+            state = STATE_UNDETERMINED
64
+    return state
65
+
66
+
67
+def calc_logic(logic_operator, subs):
68
+    """Calc overall state of an alarm expression.
69
+
70
+    'OK' means False;
71
+    'ALARM' means True;
72
+    'UNDETERMINED' means either True or False.
73
+    """
74
+    if logic_operator == 'AND':
75
+        state = 'ALARM'
76
+        for o in subs:
77
+            if o == 'OK':
78
+                return 'OK'
79
+            elif o == 'UNDETERMINED':
80
+                state = 'UNDETERMINED'
81
+        return state
82
+    elif logic_operator == 'OR':
83
+        state = 'OK'
84
+        for o in subs:
85
+            if o == 'ALARM':
86
+                return 'ALARM'
87
+            elif o == 'UNDETERMINED':
88
+                state = 'UNDETERMINED'
89
+        return state
90
+    else:
91
+        return 'UNDETERMINED'

+ 320
- 0
kiloeyes/common/alarm_expr_parser.py View File

@@ -0,0 +1,320 @@
1
+# -*- coding: utf-8 -*-
2
+# Copyright 2014 Hewlett-Packard
3
+# Copyright 2015 Carnegie Mellon University
4
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+# not use this file except in compliance with the License. You may obtain
6
+# a copy of the License at
7
+#
8
+# http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+# License for the specific language governing permissions and limitations
14
+# under the License.
15
+import itertools
16
+import pyparsing
17
+
18
+
19
+class SubExpr(object):
20
+    def __init__(self, tokens):
21
+
22
+        self._sub_expr = tokens
23
+        self._func = tokens.func
24
+        self._metric_name = tokens.metric_name
25
+        self._dimensions = tokens.dimensions.dimensions_list
26
+        self._operator = tokens.relational_op
27
+        self._threshold = tokens.threshold
28
+        self._period = tokens.period
29
+        self._periods = tokens.periods
30
+        self._id = None
31
+
32
+    @property
33
+    def sub_expr_str(self):
34
+        """Get the entire sub expression as a string with no spaces."""
35
+        return "".join(list(itertools.chain(*self._sub_expr)))
36
+
37
+    @property
38
+    def fmtd_sub_expr_str(self):
39
+        """Get the entire sub expressions as a string with spaces."""
40
+        result = "{}({}".format(self._func.encode('utf8'),
41
+                                self._metric_name.encode('utf8'))
42
+
43
+        if self._dimensions:
44
+            result += "{{{}}}".format(self._dimensions.encode('utf8'))
45
+
46
+        if self._period:
47
+            result += ", {}".format(self._period.encode('utf8'))
48
+
49
+        result += ")"
50
+
51
+        result += " {} {}".format(self._operator.encode('utf8'),
52
+                                  self._threshold.encode('utf8'))
53
+
54
+        if self._periods:
55
+            result += " times {}".format(self._periods.encode('utf8'))
56
+
57
+        return result.decode('utf8')
58
+
59
+    @property
60
+    def dimensions_str(self):
61
+        """Get all the dimensions as a single comma delimited string."""
62
+        return self._dimensions
63
+
64
+    @property
65
+    def operands_list(self):
66
+        """Get this sub expression as a list."""
67
+        return [self]
68
+
69
+    @property
70
+    def logic_operator(self):
71
+        return None
72
+
73
+    @property
74
+    def sub_expr_list(self):
75
+        return []
76
+
77
+    @property
78
+    def func(self):
79
+        """Get the function as it appears in the orig expression."""
80
+        return self._func
81
+
82
+    @property
83
+    def normalized_func(self):
84
+        """Get the function upper-cased."""
85
+        return self._func.upper()
86
+
87
+    @property
88
+    def metric_name(self):
89
+        """Get the metric name as it appears in the orig expression."""
90
+        return self._metric_name
91
+
92
+    @property
93
+    def normalized_metric_name(self):
94
+        """Get the metric name lower-cased."""
95
+        return self._metric_name.lower()
96
+
97
+    @property
98
+    def dimensions_as_list(self):
99
+        """Get the dimensions as a list."""
100
+        if self._dimensions:
101
+            return self._dimensions.split(",")
102
+        else:
103
+            return []
104
+
105
+    @property
106
+    def dimensions_as_dict(self):
107
+        """Get the dimensions as a dict."""
108
+        dimension_dict = {}
109
+        for di in self.dimensions_as_list:
110
+            temp = di.split("=")
111
+            dimension_dict[temp[0]] = temp[1]
112
+        return dimension_dict
113
+
114
+    @property
115
+    def operator(self):
116
+        """Get the operator."""
117
+        return self._operator
118
+
119
+    @property
120
+    def threshold(self):
121
+        """Get the threshold value."""
122
+        return self._threshold
123
+
124
+    @property
125
+    def period(self):
126
+        """Get the period. Default is 60 seconds."""
127
+        if self._period:
128
+            return self._period
129
+        else:
130
+            return u'60'
131
+
132
+    @property
133
+    def periods(self):
134
+        """Get the periods. Default is 1."""
135
+        if self._periods:
136
+            return self._periods
137
+        else:
138
+            return u'1'
139
+
140
+    @property
141
+    def normalized_operator(self):
142
+        """Get the operator as one of LT, GT, LTE, or GTE."""
143
+        if self._operator.lower() == "lt" or self._operator == "<":
144
+            return u"LT"
145
+        elif self._operator.lower() == "gt" or self._operator == ">":
146
+            return u"GT"
147
+        elif self._operator.lower() == "lte" or self._operator == "<=":
148
+            return u"LTE"
149
+        elif self._operator.lower() == "gte" or self._operator == ">=":
150
+            return u"GTE"
151
+
152
+    @property
153
+    def id(self):
154
+        """Get the id used to identify this sub expression in the repo."""
155
+        return self._id
156
+
157
+    @id.setter
158
+    def id(self, id):
159
+        """Set the d used to identify this sub expression in the repo."""
160
+        self._id = id
161
+
162
+
163
+class BinaryOp(object):
164
+    def __init__(self, tokens):
165
+        self.op = tokens[0][1]
166
+        self.operands = tokens[0][0::2]
167
+        if self.op == u'&&' or self.op == u'and':
168
+            self.op = u'AND'
169
+        if self.op == u'||' or self.op == u'or':
170
+            self.op = u'OR'
171
+
172
+    @property
173
+    def operands_list(self):
174
+        return ([sub_operand for operand in self.operands for sub_operand in
175
+                 operand.operands_list])
176
+
177
+    @property
178
+    def logic_operator(self):
179
+        return self.op
180
+
181
+    @property
182
+    def sub_expr_list(self):
183
+        if self.op:
184
+            return self.operands
185
+        else:
186
+            return []
187
+
188
+
189
+class AndSubExpr(BinaryOp):
190
+    """Expand later as needed."""
191
+    pass
192
+
193
+
194
+class OrSubExpr(BinaryOp):
195
+    """Expand later as needed."""
196
+    pass
197
+
198
+
199
+COMMA = pyparsing.Literal(",")
200
+LPAREN = pyparsing.Literal("(")
201
+RPAREN = pyparsing.Literal(")")
202
+EQUAL = pyparsing.Literal("=")
203
+LBRACE = pyparsing.Literal("{")
204
+RBRACE = pyparsing.Literal("}")
205
+
206
+# Initialize non-ascii unicode code points in the Basic Multilingual Plane.
207
+unicode_printables = u''.join(
208
+    unichr(c) for c in xrange(128, 65536) if not unichr(c).isspace())
209
+
210
+# Does not like comma. No Literals from above allowed.
211
+valid_identifier_chars = (
212
+    (unicode_printables + pyparsing.alphanums + ".-_#!$%&'*+/:;?@[\\]^`|~"))
213
+
214
+metric_name = (
215
+    pyparsing.Word(valid_identifier_chars, min=1, max=255)("metric_name"))
216
+dimension_name = pyparsing.Word(valid_identifier_chars, min=1, max=255)
217
+dimension_value = pyparsing.Word(valid_identifier_chars, min=1, max=255)
218
+
219
+integer_number = pyparsing.Word(pyparsing.nums)
220
+decimal_number = pyparsing.Word(pyparsing.nums + ".")
221
+
222
+max = pyparsing.CaselessLiteral("max")
223
+min = pyparsing.CaselessLiteral("min")
224
+avg = pyparsing.CaselessLiteral("avg")
225
+count = pyparsing.CaselessLiteral("count")
226
+sum = pyparsing.CaselessLiteral("sum")
227
+func = (max | min | avg | count | sum)("func")
228
+
229
+less_than_op = (
230
+    (pyparsing.CaselessLiteral("<") | pyparsing.CaselessLiteral("lt")))
231
+less_than_eq_op = (
232
+    (pyparsing.CaselessLiteral("<=") | pyparsing.CaselessLiteral("lte")))
233
+greater_than_op = (
234
+    (pyparsing.CaselessLiteral(">") | pyparsing.CaselessLiteral("gt")))
235
+greater_than_eq_op = (
236
+    (pyparsing.CaselessLiteral(">=") | pyparsing.CaselessLiteral("gte")))
237
+
238
+# Order is important. Put longer prefix first.
239
+relational_op = (
240
+    less_than_eq_op | less_than_op | greater_than_eq_op | greater_than_op)(
241
+    "relational_op")
242
+
243
+AND = pyparsing.CaselessLiteral("and") | pyparsing.CaselessLiteral("&&")
244
+OR = pyparsing.CaselessLiteral("or") | pyparsing.CaselessLiteral("||")
245
+logical_op = (AND | OR)("logical_op")
246
+
247
+times = pyparsing.CaselessLiteral("times")
248
+
249
+dimension = pyparsing.Group(dimension_name + EQUAL + dimension_value)
250
+
251
+# Cannot have any whitespace after the comma delimiter.
252
+dimension_list = pyparsing.Group(pyparsing.Optional(
253
+    LBRACE + pyparsing.delimitedList(dimension, delim=',', combine=True)(
254
+        "dimensions_list") + RBRACE))
255
+
256
+metric = metric_name + dimension_list("dimensions")
257
+period = integer_number("period")
258
+threshold = decimal_number("threshold")
259
+periods = integer_number("periods")
260
+
261
+expression = pyparsing.Forward()
262
+
263
+sub_expression = (func + LPAREN + metric + pyparsing.Optional(
264
+    COMMA + period) + RPAREN + relational_op + threshold + pyparsing.Optional(
265
+    times + periods) | LPAREN + expression + RPAREN)
266
+
267
+sub_expression.setParseAction(SubExpr)
268
+
269
+expression = (
270
+    pyparsing.operatorPrecedence(sub_expression,
271
+                                 [(AND, 2, pyparsing.opAssoc.LEFT, AndSubExpr),
272
+                                  (OR, 2, pyparsing.opAssoc.LEFT, OrSubExpr)]))
273
+
274
+
275
+class AlarmExprParser(object):
276
+    def __init__(self, expr):
277
+        self._expr = expr
278
+        try:
279
+            self.parseResult = (expression + pyparsing.stringEnd).parseString(
280
+                self._expr.replace(' ', ''))[0]
281
+        except Exception:
282
+            self.parseResult = None
283
+
284
+    @property
285
+    def parse_result(self):
286
+        return self.parseResult
287
+
288
+    @property
289
+    def sub_expr_list(self):
290
+        if self.parseResult:
291
+            return self.parseResult.operands_list
292
+        else:
293
+            return None
294
+
295
+    @property
296
+    def related_metrics(self):
297
+        """Get a list of all the metrics related with this expression."""
298
+        related_metrics = []
299
+        for expr in self.sub_expr_list:
300
+            related_metrics.append({
301
+                'name': expr.metric_name,
302
+                'dimensions': expr.dimensions_as_dict
303
+            })
304
+        return related_metrics
305
+
306
+    @property
307
+    def sub_alarm_expressions(self):
308
+        """Get a list of all the sub expr parsed information."""
309
+        sub_alarm_expr = {}
310
+        for expr in self.sub_expr_list:
311
+            sub_alarm_expr[expr.fmtd_sub_expr_str] = {
312
+                'function': expr.normalized_func,
313
+                'metric_name': expr.normalized_metric_name,
314
+                'dimensions': expr.dimensions_as_dict,
315
+                'operator': expr.normalized_operator,
316
+                'threshold': expr.threshold,
317
+                'period': expr.period,
318
+                'periods': expr.periods
319
+            }
320
+        return sub_alarm_expr

+ 76
- 0
kiloeyes/common/alarm_expr_validator.py View File

@@ -0,0 +1,76 @@
1
+# Copyright 2015 Carnegie Mellon University
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+# http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+
16
+import json
17
+from kiloeyes.common import alarm_expr_parser as parser
18
+
19
+
20
+key_set = ['expression',
21
+           'alarm_actions',
22
+           'ok_actions',
23
+           'undetermined_actions',
24
+           'match_by',
25
+           'name',
26
+           'description']
27
+
28
+
29
+def is_valid_alarm_definition(alarm_def_json):
30
+    alarm_definition = json.loads(alarm_def_json)
31
+    for key in key_set:
32
+        if key not in alarm_definition:
33
+            return False
34
+    expression = alarm_definition['expression']
35
+    alarm_parser = parser.AlarmExprParser(expression)
36
+    if not alarm_parser.parse_result:
37
+        return False
38
+    return True
39
+
40
+
41
+def is_valid_update_alarm_definition(ori_alarm_def_json, new_alarm_def_json):
42
+    # both should be valid alarm definition
43
+    if (not (is_valid_alarm_definition(ori_alarm_def_json)
44
+             and is_valid_alarm_definition(new_alarm_def_json))):
45
+        return False
46
+    ori_alarm_definition = json.loads(ori_alarm_def_json)
47
+    new_alarm_definition = json.loads(new_alarm_def_json)
48
+
49
+    # match_by should not change
50
+    if ori_alarm_definition['match_by'] != new_alarm_definition['match_by']:
51
+        return False
52
+
53
+    ori_expression = ori_alarm_definition['expression']
54
+    ori_alarm_parser = parser.AlarmExprParser(ori_expression)
55
+    ori_sub_expr_list = ori_alarm_parser.sub_expr_list
56
+    new_expression = new_alarm_definition['expression']
57
+    new_alarm_parser = parser.AlarmExprParser(new_expression)
58
+    new_sub_expr_list = new_alarm_parser.sub_expr_list
59
+
60
+    # should have same number of sub alarm exprs
61
+    l = len(ori_sub_expr_list)
62
+    if not new_sub_expr_list or l != len(new_sub_expr_list):
63
+        return False
64
+
65
+    for i in range(l):
66
+        sub_expr_ori = ori_sub_expr_list[i]
67
+        sub_expr_new = new_sub_expr_list[i]
68
+        # each metrics in alarm expr should remain the same
69
+        if (sub_expr_ori.normalized_metric_name
70
+                != sub_expr_new.normalized_metric_name):
71
+            return False
72
+        if (sub_expr_ori.dimensions_as_dict
73
+                != sub_expr_new.dimensions_as_dict):
74
+            return False
75
+
76
+    return True

+ 87
- 0
kiloeyes/common/email_sender.py View File

@@ -0,0 +1,87 @@
1
+# Copyright 2015 Carnegie Mellon University
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import email.mime.text
16
+from oslo_config import cfg
17
+import smtplib
18
+
19
+from kiloeyes.openstack.common import log
20
+
21
+MAILSENDER_OPTS = [
22
+    cfg.StrOpt('username',
23
+               default='kiloeyes.notification@gmail.com',
24
+               help='The email account user name.'),
25
+    cfg.StrOpt('password',
26
+               default='password',
27
+               help='The email account user password.'),
28
+    cfg.StrOpt('smtp_host', default='smtp.gmail.com',
29
+               help='The email service host.'),
30
+    cfg.IntOpt('port', default=25,
31
+               help='The email service port.'),
32
+    cfg.BoolOpt('use_tls', default=True,
33
+                help='Set to True if the service uses TLS.'),
34
+
35
+]
36
+
37
+cfg.CONF.register_opts(MAILSENDER_OPTS, group="mailsender")
38
+
39
+LOG = log.getLogger(__name__)
40
+
41
+
42
+class EmailSender(object):
43
+
44
+    def __init__(self):
45
+        self.username = cfg.CONF.mailsender.username
46
+        self.password = cfg.CONF.mailsender.password
47
+        self.smtp_host = cfg.CONF.mailsender.smtp_host
48
+        self.port = cfg.CONF.mailsender.port
49
+        self.use_tls = cfg.CONF.mailsender.use_tls
50
+        self.from_addr = self.username
51
+
52
+        self.smtp = smtplib.SMTP()
53
+
54
+        LOG.debug('connecting ...')
55
+
56
+        # connect
57
+        try:
58
+            self.smtp.connect(self.smtp_host, self.port)
59
+        except Exception:
60
+            LOG.debug('SMTP Connection error.')
61
+
62
+        if self.use_tls:
63
+            self.smtp.starttls()
64
+        # login with username & password
65
+        try:
66
+            LOG.debug('Login ...')
67
+            self.smtp.login(self.username, self.password)
68
+        except Exception:
69
+            LOG.debug('Login exception.')
70
+
71
+    def reset(self):
72
+        self.__init__()
73
+
74
+    def send_emails(self, to_addrs, subject, content):
75
+        # fill content with MIMEText's object
76
+        msg = email.mime.text.MIMEText(content)
77
+        msg['From'] = self.from_addr
78
+        msg['To'] = ';'.join(to_addrs)
79
+        msg['Subject'] = subject
80
+        try:
81
+            self.smtp.sendmail(self.from_addr, to_addrs, msg.as_string())
82
+            LOG.debug('Mail sent to: %s' % str(to_addrs))
83
+            return True
84
+        except Exception as e:
85
+            LOG.debug('Mail sent Exception: %s, reset the sender.' % str(e))
86
+            self.reset()
87
+            return False

+ 142
- 0
kiloeyes/common/es_conn.py View File

@@ -0,0 +1,142 @@
1
+# Copyright 2012-2013 eNovance <licensing@enovance.com>
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+from oslo_config import cfg
16
+import requests
17
+import ujson as json
18
+
19
+from kiloeyes.openstack.common import log
20
+
21
+
22
+ES_OPTS = [
23
+    cfg.StrOpt('uri',
24
+               help='Address to kafka server. For example: '
25
+               'uri=http://192.168.1.191:9200/'),
26
+    cfg.StrOpt('id_field',
27
+               default='',
28
+               help='The field name for _id.'),
29
+    cfg.BoolOpt('drop_data',
30
+                default=False,
31
+                help=('Specify if received data should be simply dropped. '
32
+                      'This parameter is only for testing purposes.')),
33
+]
34
+
35
+cfg.CONF.register_opts(ES_OPTS, group="es_conn")
36
+
37
+LOG = log.getLogger(__name__)
38
+
39
+
40
+class ESConnection(object):
41
+
42
+    def __init__(self, doc_type, index_stratey, index_prefix):
43
+        if not cfg.CONF.es_conn.uri:
44
+            raise Exception('ElasticSearch is not configured correctly! '
45
+                            'Use configuration file to specify ElasticSearch '
46
+                            'uri, for example: '
47
+                            'uri=192.168.1.191:9200')
48
+
49
+        self.uri = cfg.CONF.es_conn.uri
50
+        if self.uri.strip()[-1] != '/':
51
+            self.uri += '/'
52
+
53
+        self.doc_type = doc_type
54
+        self.index_strategy = index_stratey
55
+        self.index_prefix = index_prefix
56
+
57
+        self.id_field = cfg.CONF.es_conn.id_field
58
+        self.drop_data = cfg.CONF.es_conn.drop_data
59
+
60
+        self.search_path = '%s%s*/%s/_search' % (self.uri,
61
+                                                 self.index_prefix,
62
+                                                 self.doc_type)
63
+        LOG.debug('ElasticSearch Connection initialized successfully!')
64
+
65
+    def send_messages(self, msg):
66
+        LOG.debug('Prepare to send messages.')
67
+        if self.drop_data:
68
+            return
69
+        else:
70
+            # figure out id situation
71
+            _id = ''
72
+            if self.id_field:
73
+                obj = json.loads(msg)
74
+                _id = obj.get(self.id_field)
75
+                if not _id:
76
+                    LOG.error('Msg does not have required id field %s' %
77
+                              self.id_field)
78
+                    return 400
79
+            # index may change over the time, it has to be called for each
80
+            # request
81
+            index = self.index_strategy.get_index()
82
+            path = '%s%s%s/%s/%s' % (self.uri, self.index_prefix,
83
+                                     index, self.doc_type, _id)
84
+            res = requests.post(path, data=msg)
85
+            LOG.debug('Msg post target=%s' % path)
86
+            LOG.debug('Msg posted with response code: %s' % res.status_code)
87
+            return res.status_code
88
+
89
+    def get_messages(self, cond, q_string=""):
90
+        LOG.debug('Prepare to get messages.')
91
+        if cond:
92
+            data = json.dumps(cond)
93
+        else:
94
+            data = {}
95
+        return requests.post(self.search_path + "?" + q_string, data=data)
96
+
97
+    def get_message_by_id(self, id):
98
+        LOG.debug('Prepare to get messages by id.')
99
+        path = self.search_path + '?q=_id:' + id
100
+        LOG.debug('Search path:' + path)
101
+        res = requests.get(path)
102
+        LOG.debug('Msg get with response code: %s' % res.status_code)
103
+        return res
104
+
105
+    def post_messages(self, msg, id):
106
+        LOG.debug('Prepare to post messages.')
107
+        if self.drop_data:
108
+            return 204
109
+        else:
110
+            index = self.index_strategy.get_index()
111
+            path = '%s%s%s/%s/' % (self.uri, self.index_prefix,
112
+                                   index, self.doc_type)
113
+
114
+            res = requests.post(path + id, data=msg)
115
+            LOG.debug('Msg post with response code: %s' % res.status_code)
116
+            return res.status_code
117
+
118
+    def put_messages(self, msg, id):
119
+        LOG.debug('Prepare to put messages.')
120
+        if self.drop_data:
121
+            return 204
122
+        else:
123
+            index = self.index_strategy.get_index()
124
+            path = '%s%s%s/%s/' % (self.uri, self.index_prefix,
125
+                                   index, self.doc_type)
126
+
127
+            res = requests.put(path + id, data=msg)
128
+            LOG.debug('Msg put with response code: %s' % res.status_code)
129
+            return res.status_code
130
+
131
+    def del_messages(self, id):
132
+        LOG.debug('Prepare to delete messages.')
133
+        if self.drop_data:
134
+            return 204
135
+        else:
136
+            index = self.index_strategy.get_index()
137
+            path = '%s%s%s/%s/' % (self.uri, self.index_prefix,
138
+                                   index, self.doc_type)
139
+
140
+            res = requests.delete(path + id)
141
+            LOG.debug('Msg delete with response code: %s' % res.status_code)
142
+            return res.status_code

+ 212
- 0
kiloeyes/common/kafka_conn.py View File

@@ -0,0 +1,212 @@
1
+# Copyright 2013 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import time
16
+
17
+from kafka import client
18
+from kafka import common
19
+from kafka import consumer
20
+from kafka import producer
21
+from oslo_config import cfg
22
+from oslo_config import types
23
+
24
+try:
25
+    import ujson as json
26
+except ImportError:
27
+    import json
28
+
29
+from kiloeyes.openstack.common import log
30
+
31
+
32
+KAFKA_OPTS = [
33
+    cfg.StrOpt('uri', help='Address to kafka server. For example: '
34
+               'uri=192.168.1.191:9092'),
35
+    cfg.StrOpt('group', default='api',
36
+               help='The group name that this service belongs to.'),
37
+    cfg.IntOpt('wait_time', default=1,
38
+               help='The wait time when no messages on kafka queue.'),
39
+    cfg.IntOpt('ack_time', default=20,
40
+               help='The ack time back to kafka.'),
41
+    cfg.IntOpt('max_retry', default=3,
42
+               help='The number of retry when there is a connection error.'),
43
+    cfg.BoolOpt('auto_commit', default=True,
44
+                help='If automatically commmit when consume messages.'),
45
+    cfg.BoolOpt('async', default=True, help='The type of posting.'),
46
+    cfg.BoolOpt('compact', default=True,
47
+                help=('Specify if the message received should be parsed. '
48
+                      'If True, message will not be parsed, otherwise '
49
+                      'messages will be parsed.')),
50
+    cfg.MultiOpt('partitions', item_type=types.Integer(),
51
+                 default=[0],
52
+                 help='The sleep time when no messages on kafka queue.'),
53
+    cfg.BoolOpt('drop_data', default=False,
54
+                help=('Specify if received data should be simply dropped. '
55
+                      'This parameter is only for testing purposes.')),
56
+]
57
+
58
+cfg.CONF.register_opts(KAFKA_OPTS, group="kafka_opts")
59
+
60
+LOG = log.getLogger(__name__)
61
+
62
+
63
+class KafkaConnection(object):
64
+
65
+    def __init__(self, topic):
66
+        if not cfg.CONF.kafka_opts.uri:
67
+            raise Exception('Kafka is not configured correctly! '
68
+                            'Use configuration file to specify Kafka '
69
+                            'uri, for example: '
70
+                            'uri=192.168.1.191:9092')
71
+
72
+        self.uri = cfg.CONF.kafka_opts.uri
73
+        self.topic = topic
74
+        self.group = cfg.CONF.kafka_opts.group
75
+        self.wait_time = cfg.CONF.kafka_opts.wait_time
76
+        self.async = cfg.CONF.kafka_opts.async
77
+        self.ack_time = cfg.CONF.kafka_opts.ack_time
78
+        self.max_retry = cfg.CONF.kafka_opts.max_retry
79
+        self.auto_commit = cfg.CONF.kafka_opts.auto_commit
80
+        self.compact = cfg.CONF.kafka_opts.compact
81
+        self.partitions = cfg.CONF.kafka_opts.partitions
82
+        self.drop_data = cfg.CONF.kafka_opts.drop_data
83
+
84
+        self._client = None
85
+        self._consumer = None
86
+        self._producer = None
87
+
88
+        LOG.debug('Kafka Connection initialized successfully!')
89
+
90
+    def _init_client(self, wait_time=None):
91
+        for i in range(self.max_retry):
92
+            try:
93
+                # if there is a client instance, but _init_client is called
94
+                # again, most likely the connection has gone stale, close that
95
+                # connection and reconnect.
96
+                if self._client:
97
+                    self._client.close()
98
+
99
+                if not wait_time:
100
+                    wait_time = self.wait_time
101
+                time.sleep(wait_time)
102
+
103
+                self._client = client.KafkaClient(self.uri)
104
+
105
+                # when a client is re-initialized, existing consumer should be
106
+                # reset as well.
107
+                self._consumer = None
108
+                self._producer = None
109
+                LOG.debug("Successfully connected to Kafka server at topic: "
110
+                          "\"%s\" partitions %s" % (self.topic,
111
+                                                    self.partitions))
112
+                break
113
+            except common.KafkaUnavailableError:
114
+                LOG.error('Kafka server at %s is down.' % self.uri)
115
+            except common.LeaderNotAvailableError:
116
+                LOG.error('Kafka at %s has no leader available.' % self.uri)
117
+            except Exception:
118
+                LOG.error('Kafka at %s initialization failed.' % self.uri)
119
+            # Wait a bit and try again to get a client
120
+            time.sleep(self.wait_time)
121
+
122
+    def _init_consumer(self):
123
+        try:
124
+            if not self._client:
125
+                self._init_client()
126
+            self._consumer = consumer.SimpleConsumer(
127
+                self._client, self.group, self.topic,
128
+                auto_commit=self.auto_commit,
129
+                partitions=self.partitions)
130
+            LOG.debug('Consumer was created successfully.')
131
+        except Exception:
132
+            self._consumer = None
133
+            LOG.exception('Kafka (%s) consumer can not be created.' %
134
+                          self.uri)
135
+
136
+    def _init_producer(self):
137
+        try:
138
+            if not self._client:
139
+                self._init_client()
140
+            self._producer = producer.SimpleProducer(
141
+                self._client, async=self.async, ack_timeout=self.ack_time)
142
+            LOG.debug('Producer was created successfully.')
143
+        except Exception:
144
+            self._producer = None
145
+            LOG.exception('Kafka (%s) producer can not be created.' %
146
+                          self.uri)
147
+
148
+    def commit(self):
149
+        if self._consumer and self.auto_commit:
150
+            self._consumer.commit()
151
+
152
+    def close(self):
153
+        if self._client:
154
+            self._consumer = None
155
+            self._producer = None
156
+            self._client.close()
157
+
158
+    def get_messages(self):
159
+        try:
160
+            if not self._consumer:
161
+                self._init_consumer()
162
+
163
+            for msg in self._consumer:
164
+                if msg.message:
165
+                    LOG.debug(msg.message.value)
166
+                    yield msg
167
+        except common.OffsetOutOfRangeError:
168
+            self._consumer.seek(0, 0)
169
+            LOG.error('Seems consumer has been down for a long time.')
170
+            yield None
171
+        except Exception as ex:
172
+            LOG.exception(ex)
173
+            self._consumer = None
174
+            yield None
175
+
176
+    def send_messages(self, messages):
177
+        LOG.debug('Prepare to send messages.')
178
+        if not messages or self.drop_data:
179
+            return 204
180
+
181
+        code = 400
182
+        try:
183
+            if not self._producer:
184
+                self._init_producer()
185
+
186
+            LOG.debug('Start sending messages to kafka.')
187
+            if self.compact:
188
+                self._producer.send_messages(self.topic, messages)
189
+            else:
190
+                data = json.loads(messages)
191
+                LOG.debug('Msg parsed successfully.')
192
+                if isinstance(data, list):
193
+                    for item in data:
194
+                        self._producer.send_messages(
195
+                            self.topic, json.dumps(item))
196
+                else:
197
+                    self._producer.send_messages(self.topic, messages)
198
+            LOG.debug('Message posted successfully.')
199
+            code = 204
200
+        except (common.KafkaUnavailableError,
201
+                common.LeaderNotAvailableError):
202
+            self._client = None
203
+            code = 503
204
+            LOG.exception('Error occurred while posting data to Kafka.')
205
+        except ValueError:
206
+            code = 406
207
+            LOG.exception('Message %s is not valid json.' % messages)
208
+        except Exception:
209
+            code = 500
210
+            LOG.exception('Unknown error.')
211
+
212
+        return code

+ 18
- 0
kiloeyes/common/namespace.py View File

@@ -0,0 +1,18 @@
1
+# Copyright 2013 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+DISPATCHER_NS = 'kiloeyes.dispatcher'
16
+PROCESSOR_NS = 'kiloeyes.message.processor'
17
+STRATEGY_NS = 'kiloeyes.index.strategy'
18
+MICROSERVICE_NS = 'kiloeyes.microservice'

+ 115
- 0
kiloeyes/common/resource_api.py View File

@@ -0,0 +1,115 @@
1
+# Copyright 2013 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import falcon
16
+from falcon import api_helpers
17
+
18
+from kiloeyes.openstack.common import log
19
+
20
+
21
+RESOURCE_METHOD_FLAG = 'fab05a04-b861-4651-bd0c-9cb3eb9a6088'
22
+
23
+LOG = log.getLogger(__name__)
24
+
25
+
26
+class Restify(object):
27
+    def __init__(self, path='', method='GET'):
28
+        if not path:
29
+            raise Exception('Path has to be specified.')
30
+
31
+        if method.upper() not in falcon.HTTP_METHODS:
32
+            raise Exception('Invalid request method.')
33
+        self.path = path
34
+        self.method = method.upper()
35
+
36
+    def __call__(self, func):
37
+        setattr(func, RESOURCE_METHOD_FLAG, self)
38
+        return func
39
+
40
+
41
+class ResourceAPI(falcon.API):
42
+
43
+    def add_route(self, uri_template, resource):
44
+        """Associates uri patterns with resource methods.
45
+
46
+        A resource is an instance of a class that defines various methods
47
+        to handle http requests.
48
+
49
+        Use this class to create applications which serve a standard
50
+        compliant ReSTful API. For example, you may have an API which manage
51
+        monitoring data, there can be multiple implementations of the API
52
+        using different technologies. One can use Mongodb, the other can use
53
+        Cassandra. To make the configuration of the application very easy,
54
+        each implementation provides a class with set of methods decorated
55
+        by class Restify, the application can simply using single entry
56
+        configuration to load different implementations.
57
+
58
+        For example::
59
+
60
+            class ExampleResource(object):
61
+                @Restify(path='/path1/', method='post')
62
+                def func1(self, req, res):
63
+                    pass
64
+
65
+                @Restify(path='/path2/{id}/key/', method='get')
66
+                def func2(self, req, res, id):
67
+                    pass
68
+
69
+                def func3(self, req, res, id):
70
+                    pass
71
+
72
+        With the above class, the following code will add the class method
73
+        func1, func2 to handle post and get requests respectively, method
74
+        func3 won't be added to the routes.::
75
+
76
+            app.add_route(None, ExampleResource())
77
+
78
+        Args:
79
+            uri_template (url pattern): the url pattern which a client will
80
+                post a request against. If none, ResourceAPI will
81
+                automatically look up the decorated methods.
82
+            resource (instance): Object which represents an HTTP/REST
83
+                "resource". Falcon will pass requests to various decorated
84
+                methods to handle http requests.
85
+        """
86
+        if not resource:
87
+            raise Exception('Not a valid resource')
88
+
89
+        path_maps = {}
90
+        try:
91
+            if uri_template:
92
+                super(ResourceAPI, self).add_route(uri_template, resource)
93
+            else:
94
+                for attr in dir(resource):
95
+                    method = getattr(resource, attr)
96
+                    if callable(method) and hasattr(method,
97
+                                                    RESOURCE_METHOD_FLAG):
98
+                        flag = getattr(method, RESOURCE_METHOD_FLAG)
99
+                        map = path_maps.get(flag.path)
100
+                        if not map:
101
+                            uri_fields, template = (
102
+                                api_helpers.compile_uri_template(flag.path))
103
+                            map = (template, {})
104
+                            path_maps[flag.path] = map
105
+
106
+                        new_method = api_helpers._wrap_with_hooks(
107
+                            self._before, self._after, method)
108
+                        map[1][flag.method] = new_method
109
+
110
+                for item in path_maps:
111
+                    self._routes.insert(0, (path_maps[item][0],
112
+                                            path_maps[item][1]))
113
+        except Exception:
114
+            LOG.exception('Error occurred while adding the resource')
115
+        LOG.debug(self._routes)

+ 27
- 0
kiloeyes/dispatcher/__init__.py View File

@@ -0,0 +1,27 @@
1
+# Copyright 2013 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import abc
16
+import six
17
+
18
+
19
+@six.add_metaclass(abc.ABCMeta)
20
+class Base(object):
21
+
22
+    def __init__(self, conf):
23
+        self.conf = conf
24
+
25
+    @abc.abstractmethod
26
+    def define_routes(self, app):
27
+        """Post metric data interface."""

+ 79
- 0
kiloeyes/dispatcher/sample_dispatcher.py View File

@@ -0,0 +1,79 @@
1
+# Copyright 2013 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import falcon
16
+from oslo_config import cfg
17
+
18
+from kiloeyes.api import monasca_api_v2
19
+from kiloeyes.common import resource_api
20
+from kiloeyes.openstack.common import log
21
+
22
+OPTS = [
23
+    cfg.MultiStrOpt('id',
24
+                    default=['sample'],
25
+                    help='Multiple String configuration.'),
26
+    cfg.StrOpt('prefix',
27
+               default='monasca_',
28
+               help='String configuration sample.'),
29
+]
30
+cfg.CONF.register_opts(OPTS, group='sample_dispatcher')
31
+
32
+
33
+LOG = log.getLogger(__name__)
34
+
35
+
36
+class SampleDispatcher(monasca_api_v2.V2API):
37
+    """kiloeyes dispatcher sample class
38
+
39
+    This class shows how to develop a dispatcher and how the configuration
40
+    parameters should be defined and how these configuration parameters
41
+    should be set in kiloeyes.conf file.
42
+
43
+    This class uses configuration parameters appear in sample_dispatcher
44
+    section such as the following:
45
+
46
+    [sample_dispatcher]
47
+    id = 101
48
+    id = 105
49
+    id = 180
50
+    prefix = sample__
51
+
52
+    If the above section appears in file kiloeyes.conf, these values will be
53
+    loaded to cfg.CONF after the dispatcher gets loaded. The cfg.CONF should
54
+    have the following values:
55
+
56
+    cfg.CONF.sample_dispatcher.id = [101, 105, 180]
57
+    cfg.CONF.sample_dispatcher.prefix = "sample__"
58
+    """
59
+    def __init__(self, global_conf):
60
+        LOG.debug('initializing SampleDispatcher!')
61
+        super(SampleDispatcher, self).__init__(global_conf)
62
+
63
+        LOG.debug('SampleDispatcher conf entries: prefix')
64
+        LOG.debug(global_conf.sample_dispatcher.prefix)
65
+        LOG.debug('SampleDispatcher conf entries: id')
66
+        LOG.debug(global_conf.sample_dispatcher.id)
67
+
68
+    @resource_api.Restify('/v2.0/datapoints/', method='post')
69
+    def do_post_metrics(self, req, res):
70
+        LOG.debug('Getting the call at endpoint datapoints.')
71
+        msg = req.stream.read()
72
+        LOG.debug('The msg:', msg)
73
+        res.status = getattr(falcon, 'HTTP_201')
74
+
75
+    @resource_api.Restify('/v2.0/demopoints/', method='get')
76
+    def do_get_metrics(self, req, res):
77
+        LOG.debug('Getting the call at endpoint demopoints.')
78
+        res.body = 'demo response'
79
+        res.status = getattr(falcon, 'HTTP_200')

+ 0
- 0
kiloeyes/microservice/__init__.py View File


+ 109
- 0
kiloeyes/microservice/es_persister.py View File

@@ -0,0 +1,109 @@
1
+#
2
+# Copyright 2013 IBM Corp
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+# not use this file except in compliance with the License. You may obtain
6
+# a copy of the License at
7
+#
8
+#      http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+# License for the specific language governing permissions and limitations
14
+# under the License.
15
+
16
+from oslo_config import cfg
17
+from stevedore import driver
18
+
19
+from kiloeyes.common import es_conn
20
+from kiloeyes.common import kafka_conn
21
+from kiloeyes.common import namespace
22
+from kiloeyes.openstack.common import log
23
+from kiloeyes.openstack.common import service as os_service
24
+
25
+OPTS = [
26
+    cfg.StrOpt('topic', default='metrics',
27
+               help=('The topic that messages will be retrieved from.'
28
+                     'This also will be used as a doc type when saved '
29
+                     'to ElasticSearch.')),
30
+    cfg.StrOpt('doc_type', default='',
31
+               help=('The document type which defines what document '
32
+                     'type the messages will be save into. If not '
33
+                     'specified, then the topic will be used.')),
34
+    cfg.StrOpt('index_strategy', default='fixed',
35
+               help='The index strategy used to create index name.'),
36
+    cfg.StrOpt('index_prefix', default='data_',
37
+               help='The index prefix where metrics were saved to.'),
38
+    cfg.StrOpt('processor', default='',
39
+               help=('The message processer to load to process the message.'
40
+                     'If the message does not need to be process anyway,'
41
+                     'leave the default')),
42
+]
43
+
44
+cfg.CONF.register_opts(OPTS, group="es_persister")
45
+
46
+LOG = log.getLogger(__name__)
47
+
48
+
49
+class ESPersister(os_service.Service):
50
+
51
+    def __init__(self, threads=1000):
52
+        super(ESPersister, self).__init__(threads)
53
+        self._kafka_conn = kafka_conn.KafkaConnection(
54
+            cfg.CONF.es_persister.topic)
55
+
56
+        # load index strategy
57
+        if cfg.CONF.es_persister.index_strategy:
58
+            self.index_strategy = driver.DriverManager(
59
+                namespace.STRATEGY_NS,
60
+                cfg.CONF.es_persister.index_strategy,
61
+                invoke_on_load=True,
62
+                invoke_kwds={}).driver
63
+            LOG.debug(dir(self.index_strategy))
64
+        else:
65
+            self.index_strategy = None
66
+
67
+        self.index_prefix = cfg.CONF.es_persister.index_prefix
68
+        # Use doc_type if it is defined.
69
+        if cfg.CONF.es_persister.doc_type:
70
+            self.doc_type = cfg.CONF.es_persister.doc_type
71
+        else:
72
+            self.doc_type = cfg.CONF.es_persister.topic
73
+
74
+        # create connection to ElasticSearch
75
+        self._es_conn = es_conn.ESConnection(
76
+            self.doc_type, self.index_strategy, self.index_prefix)
77
+
78
+        # load message processor
79
+        if cfg.CONF.es_persister.processor:
80
+            self.msg_processor = driver.DriverManager(
81
+                namespace.PROCESSOR_NS,
82
+                cfg.CONF.es_persister.processor,
83
+                invoke_on_load=True,
84
+                invoke_kwds={}).driver
85
+            LOG.debug(dir(self.msg_processor))
86
+        else:
87
+            self.msg_processor = None
88
+
89
+    def start(self):
90
+        while True:
91
+            try:
92
+                for msg in self._kafka_conn.get_messages():
93
+                    if msg and msg.message:
94
+                        LOG.debug(msg.message.value)
95
+                        if self.msg_processor:
96
+                            value = self.msg_processor.process_msg(
97
+                                msg.message.value)
98
+                        else:
99
+                            value = msg.message.value
100
+                        if value:
101
+                            self._es_conn.send_messages(value)
102
+                # if autocommit is set, this will be a no-op call.
103
+                self._kafka_conn.commit()
104
+            except Exception:
105
+                LOG.exception('Error occurred while handling kafka messages.')
106
+
107
+    def stop(self):
108
+        self._kafka_conn.close()
109
+        super(ESPersister, self).stop()

+ 39
- 0
kiloeyes/microservice/fixed_strategy.py View File

@@ -0,0 +1,39 @@
1
+#
2
+# Copyright 2012-2013 eNovance <licensing@enovance.com>
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+# not use this file except in compliance with the License. You may obtain
6
+# a copy of the License at
7
+#
8
+#      http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+# License for the specific language governing permissions and limitations
14
+# under the License.
15
+
16
+from oslo_config import cfg
17
+
18
+from kiloeyes.openstack.common import log
19
+
20
+LOG = log.getLogger(__name__)
21
+
22
+OPTS = [
23
+    cfg.StrOpt('index_name',
24
+               default='',
25
+               help='The pre-configured index name.'),
26
+]
27
+
28
+cfg.CONF.register_opts(OPTS, group="fixed_strategy")
29
+
30
+
31
+class FixedStrategy(object):
32
+    """This strategy returns an empty string."""
33
+
34
+    def __init__(self):
35
+        self.index_name = cfg.CONF.fixed_strategy.index_name
36
+        LOG.debug('EmptyStrategy initialized successfully!')
37
+
38
+    def get_index(self):
39
+        return self.index_name

+ 58
- 0
kiloeyes/microservice/metrics_fixer.py View File

@@ -0,0 +1,58 @@
1
+# Copyright 2013 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+
16
+import hashlib
17
+import json
18
+import time
19
+
20
+from kiloeyes.openstack.common import log
21
+
22
+LOG = log.getLogger(__name__)
23
+
24
+
25
+class MetricsFixer(object):
26
+