Merge "Reconcile example and mini-mon configs"
This commit is contained in:
commit
aeae3262bf
14
README.md
14
README.md
@ -3,11 +3,11 @@ monasca-persister
|
||||
|
||||
The Monitoring Persister consumes metrics and alarm state transitions from the Message Queue and stores them in the Metrics and Alarms database.
|
||||
|
||||
Although the Persister isn't primarly a Web service it uses DropWizard, https://dropwizard.github.io/dropwizard/, which provides a nice Web appliction framework to expose an http endpoint that provides an interface through which metrics about the Persister can be queried as well as health status.
|
||||
Although the Persister isn't primarly a Web service it uses DropWizard, https://dropwizard.github.io/dropwizard/, which provides a nice Web appliction framework to expose an http endpoint that provides an interface through which metrics about the Persister can be queried as well as health status.
|
||||
|
||||
The basic design of the Persister is to have one Kafka consumer publish to a Disruptor, https://github.com/LMAX-Exchange/disruptor, that has output processors. The output processors use prepared batch statements to write to the Metrics and Alarms database.
|
||||
|
||||
The number of output processors/threads in the Persister can be specified to scale to more messages. To horizontally scale and provide fault-tolerance any number of Persisters can be started as consumers from the the Message Queue.
|
||||
The number of output processors/threads in the Persister can be specified to scale to more messages. To horizontally scale and provide fault-tolerance any number of Persisters can be started as consumers from the the Message Queue.
|
||||
|
||||
# Build
|
||||
|
||||
@ -17,10 +17,16 @@ Requires monasca-common from https://github.com/stackforge/monasca-common. Downl
|
||||
mvn clean package
|
||||
```
|
||||
|
||||
# Configuration
|
||||
|
||||
A sample configuration file is available in java/src/deb/etc/persister-config.yml-sample.
|
||||
|
||||
A second configuration file is provided in java/src/main/resources/persister-config.yml for use with the [vagrant "mini-mon" development environment](https://github.com/stackforge/monasca-vagrant/).
|
||||
|
||||
# TODO
|
||||
|
||||
* Purge metrics on shutdown
|
||||
* Add more robust offset management in Kafka. Currently, the offset is advanced as each message is read. If the Persister stops after the metric has been read and prior to it being committed to the Metrics and Alarms database, the metric will be lost.
|
||||
* Add more robust offset management in Kafka. Currently, the offset is advanced as each message is read. If the Persister stops after the metric has been read and prior to it being committed to the Metrics and Alarms database, the metric will be lost.
|
||||
* Add better handling of SQL exceptions.
|
||||
* Complete health check.
|
||||
* Specify and document the names of the metrics that are available for monitoring of the Persister.
|
||||
@ -35,7 +41,7 @@ you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
|
@ -4,25 +4,25 @@ alarmHistoryConfiguration:
|
||||
batchSize: 100
|
||||
numThreads: 2
|
||||
maxBatchTime: 15
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
topic: alarm-state-transitions
|
||||
groupId: 1_alarm-state-transitions
|
||||
consumerId: mini-mon
|
||||
clientId : 1
|
||||
clientId: 1
|
||||
|
||||
metricConfiguration:
|
||||
batchSize: 10000
|
||||
numThreads: 4
|
||||
maxBatchTime: 15
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
topic: metrics
|
||||
groupId: 1_metrics
|
||||
consumerId: mini-mon
|
||||
clientId : 1
|
||||
clientId: 1
|
||||
|
||||
#Kafka settings.
|
||||
kafkaConfig:
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
zookeeperConnect: localhost:2181
|
||||
socketTimeoutMs: 30000
|
||||
socketReceiveBufferBytes : 65536
|
||||
@ -43,15 +43,17 @@ verticaMetricRepoConfig:
|
||||
maxCacheSize: 2000000
|
||||
|
||||
databaseConfiguration:
|
||||
# vertica | influxdb
|
||||
# databaseType can be (vertica | influxdb)
|
||||
databaseType: influxdb
|
||||
|
||||
# Uncomment if databaseType is influxdb
|
||||
influxDbConfiguration:
|
||||
# Retention policy may be left blank to indicate default policy.
|
||||
retentionPolicy:
|
||||
# Used only if version is V9.
|
||||
maxHttpConnections: 100
|
||||
name: mon
|
||||
replicationFactor: 1
|
||||
url: http://localhost:8086
|
||||
user: mon_persister
|
||||
password: password
|
||||
@ -63,24 +65,28 @@ influxDbConfiguration:
|
||||
# user: dbadmin
|
||||
# password: password
|
||||
# properties:
|
||||
# ssl: false
|
||||
# ssl: false
|
||||
# # the maximum amount of time to wait on an empty pool before throwing an exception
|
||||
# maxWaitForConnection: 1s
|
||||
#
|
||||
# # the SQL query to run when validating a connection's liveness
|
||||
# validationQuery: "/* MyService Health Check */ SELECT 1"
|
||||
#
|
||||
# # the minimum number of connections to keep open
|
||||
# minSize: 8
|
||||
#
|
||||
# # the maximum number of connections to keep open
|
||||
# maxSize: 41
|
||||
#
|
||||
# # whether or not idle connections should be validated
|
||||
# checkConnectionWhileIdle: false
|
||||
#
|
||||
# # the maximum lifetime of an idle connection
|
||||
# maxConnectionAge: 1 minute
|
||||
|
||||
metrics:
|
||||
frequency: 1 second
|
||||
|
||||
|
||||
# Logging settings.
|
||||
logging:
|
||||
|
||||
@ -90,23 +96,38 @@ logging:
|
||||
|
||||
# Logger-specific levels.
|
||||
loggers:
|
||||
|
||||
# Sets the level for 'com.example.app' to DEBUG.
|
||||
com.example.app: DEBUG
|
||||
# com.hpcloud: debug
|
||||
# com.hpcloud.mon.persister.repository: DEBUG
|
||||
monasca: DEBUG
|
||||
|
||||
appenders:
|
||||
# Uncomment to enable logging to the console:
|
||||
#- type: console
|
||||
# threshold: DEBUG
|
||||
# timeZone: UTC
|
||||
# target: stdout
|
||||
|
||||
- type: file
|
||||
threshold: INFO
|
||||
archive: true
|
||||
# The file to which current statements will be logged.
|
||||
currentLogFilename: /var/log/monasca/persister/monasca-persister.log
|
||||
|
||||
# When the log file rotates, the archived log will be renamed to this and gzipped. The
|
||||
# %d is replaced with the previous day (yyyy-MM-dd). Custom rolling windows can be created
|
||||
# by passing a SimpleDateFormat-compatible format as an argument: "%d{yyyy-MM-dd-hh}".
|
||||
archivedLogFilenamePattern: /var/log/monasca/persister/monasca-persister.log-%d.log.gz
|
||||
|
||||
# The number of archived files to keep.
|
||||
archivedFileCount: 5
|
||||
|
||||
# The timezone used to format dates. HINT: USE THE DEFAULT, UTC.
|
||||
timeZone: UTC
|
||||
|
||||
# Uncomment to approximately match the default log format of the python
|
||||
# Openstack components. %pid is unavoidably formatted with [brackets],
|
||||
# which are hard-coded in dropwizard's logging module.
|
||||
# See http://logback.qos.ch/manual/layouts.html#conversionWord for details of the format string
|
||||
# logFormat: "%app%pid: %d{YYYY-MM-dd HH:mm:ss.SSS} %pid %level %logger [-] [%thread] %msg %ex{1}"
|
||||
|
||||
# Set the persister ports to 8090/8091 to avoid conflict with the api
|
||||
server:
|
||||
applicationConnectors:
|
||||
|
@ -4,7 +4,7 @@ alarmHistoryConfiguration:
|
||||
batchSize: 100
|
||||
numThreads: 1
|
||||
maxBatchTime: 15
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
topic: alarm-state-transitions
|
||||
groupId: persister_alarms
|
||||
consumerId: 1
|
||||
@ -14,7 +14,7 @@ metricConfiguration:
|
||||
batchSize: 1000
|
||||
numThreads: 2
|
||||
maxBatchTime: 30
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
topic: metrics
|
||||
groupId: persister_metrics
|
||||
consumerId: 1
|
||||
@ -22,8 +22,7 @@ metricConfiguration:
|
||||
|
||||
#Kafka settings.
|
||||
kafkaConfig:
|
||||
#zookeeperConnect: localhost:2181
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
zookeeperConnect: 192.168.10.4:2181
|
||||
socketTimeoutMs: 30000
|
||||
socketReceiveBufferBytes : 65536
|
||||
@ -44,10 +43,10 @@ verticaMetricRepoConfig:
|
||||
maxCacheSize: 2000000
|
||||
|
||||
databaseConfiguration:
|
||||
# databaseType can be (vertica | influxdb)
|
||||
# databaseType can be (vertica | influxdb)
|
||||
databaseType: influxdb
|
||||
# databaseType: vertica
|
||||
|
||||
# Uncomment if databaseType is influxdb
|
||||
influxDbConfiguration:
|
||||
# Retention policy may be left blank to indicate default policy.
|
||||
retentionPolicy:
|
||||
@ -56,19 +55,14 @@ influxDbConfiguration:
|
||||
name: mon
|
||||
replicationFactor: 1
|
||||
url: http://192.168.10.4:8086
|
||||
# url: http://10.10.10.2:8086
|
||||
user: root
|
||||
password: root
|
||||
|
||||
# Uncomment if databaseType is vertica
|
||||
#dataSourceFactory:
|
||||
# driverClass: com.vertica.jdbc.Driver
|
||||
## url: jdbc:vertica://mini-mon:5433/monasca
|
||||
# # my Vertica DB in the cloud
|
||||
# #url: jdbc:vertica://15.185.94.245:5433/som
|
||||
# url: jdbc:vertica://192.168.10.4:5433/mon
|
||||
## user: persister
|
||||
# #user: mon_persister
|
||||
# user: dbadmin
|
||||
# url: jdbc:vertica://192.168.10.4:5433/monasca
|
||||
# user: monasca_persister
|
||||
# password: password
|
||||
# properties:
|
||||
# ssl: false
|
||||
@ -98,17 +92,14 @@ logging:
|
||||
|
||||
# The default level of all loggers. Can be OFF, ERROR, WARN, INFO,
|
||||
# DEBUG, TRACE, or ALL.
|
||||
level: info
|
||||
level: INFO
|
||||
|
||||
# Logger-specific levels.
|
||||
loggers:
|
||||
|
||||
# Sets the level for 'com.example.app' to DEBUG.
|
||||
com.example.app: DEBUG
|
||||
monasca: DEBUG
|
||||
# monasca.persister.repository: DEBUG
|
||||
|
||||
appenders:
|
||||
|
||||
- type: console
|
||||
threshold: DEBUG
|
||||
timeZone: UTC
|
||||
@ -117,11 +108,6 @@ logging:
|
||||
- type: file
|
||||
threshold: DEBUG
|
||||
archive: true
|
||||
currentLogFilename: ./logs/monasca-persister.log
|
||||
archivedLogFilenamePattern: ./logs/monasca-persister-%d.log.gz
|
||||
archivedFileCount: 5
|
||||
|
||||
- type: file
|
||||
# The file to which current statements will be logged.
|
||||
currentLogFilename: ./logs/monasca-persister.log
|
||||
|
||||
@ -136,3 +122,8 @@ logging:
|
||||
# The timezone used to format dates. HINT: USE THE DEFAULT, UTC.
|
||||
timeZone: UTC
|
||||
|
||||
# Uncomment to approximately match the default log format of the python
|
||||
# Openstack components. %pid is unavoidably formatted with [brackets],
|
||||
# which are hard-coded in dropwizard's logging module.
|
||||
# See http://logback.qos.ch/manual/layouts.html#conversionWord for details of the format string
|
||||
# logFormat: "%app%pid: %d{YYYY-MM-dd HH:mm:ss.SSS} %pid %level %logger [-] [%thread] %msg %ex{1}"
|
||||
|
Loading…
Reference in New Issue
Block a user