openstack/monasca-persister
Code Issues Proposed changes

Merge "Reconcile example and mini-mon configs"

Browse Source
This commit is contained in:
Jenkins 2015-10-16 02:32:50 +00:00 committed by Gerrit Code Review
commit aeae3262bf
3 changed files with 59 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@ -3,11 +3,11 @@ monasca-persister
The Monitoring Persister consumes metrics and alarm state transitions from the Message Queue and stores them in the Metrics and Alarms database.
Although the Persister isn't primarly a Web service it uses DropWizard, https://dropwizard.github.io/dropwizard/, which provides a nice Web appliction framework to expose an http endpoint that provides an interface through which metrics about the Persister can be queried as well as health status.
Although the Persister isn't primarly a Web service it uses DropWizard, https://dropwizard.github.io/dropwizard/, which provides a nice Web appliction framework to expose an http endpoint that provides an interface through which metrics about the Persister can be queried as well as health status.
The basic design of the Persister is to have one Kafka consumer publish to a Disruptor, https://github.com/LMAX-Exchange/disruptor, that has output processors. The output processors use prepared batch statements to write to the Metrics and Alarms database.
The number of output processors/threads in the Persister can be specified to scale to more messages. To horizontally scale and provide fault-tolerance any number of Persisters can be started as consumers from the the Message Queue.
The number of output processors/threads in the Persister can be specified to scale to more messages. To horizontally scale and provide fault-tolerance any number of Persisters can be started as consumers from the the Message Queue.
# Build
@ -17,10 +17,16 @@ Requires monasca-common from https://github.com/stackforge/monasca-common. Downl
mvn clean package
```
# Configuration
A sample configuration file is available in java/src/deb/etc/persister-config.yml-sample.
A second configuration file is provided in java/src/main/resources/persister-config.yml for use with the [vagrant "mini-mon" development environment](https://github.com/stackforge/monasca-vagrant/).
# TODO
* Purge metrics on shutdown
* Add more robust offset management in Kafka. Currently, the offset is advanced as each message is read. If the Persister stops after the metric has been read and prior to it being committed to the Metrics and Alarms database, the metric will be lost.
* Add more robust offset management in Kafka. Currently, the offset is advanced as each message is read. If the Persister stops after the metric has been read and prior to it being committed to the Metrics and Alarms database, the metric will be lost.
* Add better handling of SQL exceptions.
* Complete health check.
* Specify and document the names of the metrics that are available for monitoring of the Persister.
@ -35,7 +41,7 @@ you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or

47
java/src/deb/etc/persister-config.yml-sample
View File

@ -4,25 +4,25 @@ alarmHistoryConfiguration:
batchSize: 100
numThreads: 2
maxBatchTime: 15
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
topic: alarm-state-transitions
groupId: 1_alarm-state-transitions
consumerId: mini-mon
clientId : 1
clientId: 1
metricConfiguration:
batchSize: 10000
numThreads: 4
maxBatchTime: 15
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
topic: metrics
groupId: 1_metrics
consumerId: mini-mon
clientId : 1
clientId: 1
#Kafka settings.
kafkaConfig:
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
zookeeperConnect: localhost:2181
socketTimeoutMs: 30000
socketReceiveBufferBytes : 65536
@ -43,15 +43,17 @@ verticaMetricRepoConfig:
maxCacheSize: 2000000
databaseConfiguration:
# vertica | influxdb
# databaseType can be (vertica | influxdb)
databaseType: influxdb
# Uncomment if databaseType is influxdb
influxDbConfiguration:
# Retention policy may be left blank to indicate default policy.
retentionPolicy:
# Used only if version is V9.
maxHttpConnections: 100
name: mon
replicationFactor: 1
url: http://localhost:8086
user: mon_persister
password: password
@ -63,24 +65,28 @@ influxDbConfiguration:
# user: dbadmin
# password: password
# properties:
# ssl: false
# ssl: false
# # the maximum amount of time to wait on an empty pool before throwing an exception
# maxWaitForConnection: 1s
#
# # the SQL query to run when validating a connection's liveness
# validationQuery: "/* MyService Health Check */ SELECT 1"
#
# # the minimum number of connections to keep open
# minSize: 8
#
# # the maximum number of connections to keep open
# maxSize: 41
#
# # whether or not idle connections should be validated
# checkConnectionWhileIdle: false
#
# # the maximum lifetime of an idle connection
# maxConnectionAge: 1 minute
metrics:
frequency: 1 second
# Logging settings.
logging:
@ -90,23 +96,38 @@ logging:
# Logger-specific levels.
loggers:
# Sets the level for 'com.example.app' to DEBUG.
com.example.app: DEBUG
# com.hpcloud: debug
# com.hpcloud.mon.persister.repository: DEBUG
monasca: DEBUG
appenders:
# Uncomment to enable logging to the console:
#- type: console
# threshold: DEBUG
# timeZone: UTC
# target: stdout
- type: file
threshold: INFO
archive: true
# The file to which current statements will be logged.
currentLogFilename: /var/log/monasca/persister/monasca-persister.log
# When the log file rotates, the archived log will be renamed to this and gzipped. The
# %d is replaced with the previous day (yyyy-MM-dd). Custom rolling windows can be created
# by passing a SimpleDateFormat-compatible format as an argument: "%d{yyyy-MM-dd-hh}".
archivedLogFilenamePattern: /var/log/monasca/persister/monasca-persister.log-%d.log.gz
# The number of archived files to keep.
archivedFileCount: 5
# The timezone used to format dates. HINT: USE THE DEFAULT, UTC.
timeZone: UTC
# Uncomment to approximately match the default log format of the python
# Openstack components. %pid is unavoidably formatted with [brackets],
# which are hard-coded in dropwizard's logging module.
# See http://logback.qos.ch/manual/layouts.html#conversionWord for details of the format string
# logFormat: "%app%pid: %d{YYYY-MM-dd HH:mm:ss.SSS} %pid %level %logger [-] [%thread] %msg %ex{1}"
# Set the persister ports to 8090/8091 to avoid conflict with the api
server:
applicationConnectors:

39
java/src/main/resources/persister-config.yml
View File

@ -4,7 +4,7 @@ alarmHistoryConfiguration:
batchSize: 100
numThreads: 1
maxBatchTime: 15
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
topic: alarm-state-transitions
groupId: persister_alarms
consumerId: 1
@ -14,7 +14,7 @@ metricConfiguration:
batchSize: 1000
numThreads: 2
maxBatchTime: 30
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
topic: metrics
groupId: persister_metrics
consumerId: 1
@ -22,8 +22,7 @@ metricConfiguration:
#Kafka settings.
kafkaConfig:
#zookeeperConnect: localhost:2181
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
zookeeperConnect: 192.168.10.4:2181
socketTimeoutMs: 30000
socketReceiveBufferBytes : 65536
@ -44,10 +43,10 @@ verticaMetricRepoConfig:
maxCacheSize: 2000000
databaseConfiguration:
# databaseType can be (vertica | influxdb)
# databaseType can be (vertica | influxdb)
databaseType: influxdb
# databaseType: vertica
# Uncomment if databaseType is influxdb
influxDbConfiguration:
# Retention policy may be left blank to indicate default policy.
retentionPolicy:
@ -56,19 +55,14 @@ influxDbConfiguration:
name: mon
replicationFactor: 1
url: http://192.168.10.4:8086
# url: http://10.10.10.2:8086
user: root
password: root
# Uncomment if databaseType is vertica
#dataSourceFactory:
# driverClass: com.vertica.jdbc.Driver
## url: jdbc:vertica://mini-mon:5433/monasca
# # my Vertica DB in the cloud
# #url: jdbc:vertica://15.185.94.245:5433/som
# url: jdbc:vertica://192.168.10.4:5433/mon
## user: persister
# #user: mon_persister
# user: dbadmin
# url: jdbc:vertica://192.168.10.4:5433/monasca
# user: monasca_persister
# password: password
# properties:
# ssl: false
@ -98,17 +92,14 @@ logging:
# The default level of all loggers. Can be OFF, ERROR, WARN, INFO,
# DEBUG, TRACE, or ALL.
level: info
level: INFO
# Logger-specific levels.
loggers:
# Sets the level for 'com.example.app' to DEBUG.
com.example.app: DEBUG
monasca: DEBUG
# monasca.persister.repository: DEBUG
appenders:
- type: console
threshold: DEBUG
timeZone: UTC
@ -117,11 +108,6 @@ logging:
- type: file
threshold: DEBUG
archive: true
currentLogFilename: ./logs/monasca-persister.log
archivedLogFilenamePattern: ./logs/monasca-persister-%d.log.gz
archivedFileCount: 5
- type: file
# The file to which current statements will be logged.
currentLogFilename: ./logs/monasca-persister.log
@ -136,3 +122,8 @@ logging:
# The timezone used to format dates. HINT: USE THE DEFAULT, UTC.
timeZone: UTC
# Uncomment to approximately match the default log format of the python
# Openstack components. %pid is unavoidably formatted with [brackets],
# which are hard-coded in dropwizard's logging module.
# See http://logback.qos.ch/manual/layouts.html#conversionWord for details of the format string
# logFormat: "%app%pid: %d{YYYY-MM-dd HH:mm:ss.SSS} %pid %level %logger [-] [%thread] %msg %ex{1}"