From cb99a66ee1664636fa181a615ead44e23b4b22d0 Mon Sep 17 00:00:00 2001 From: Ben Motz Date: Wed, 30 Sep 2015 18:16:56 +0100 Subject: [PATCH] Reconcile example and mini-mon configs Bring contents of the example config and the mini-mon config into line, add a note on log formatting, and a note in the README about the sample configurations. Change-Id: I47794b600d44a87f58825d1e11099c272f06d7aa --- README.md | 14 ++++-- java/src/deb/etc/persister-config.yml-sample | 47 ++++++++++++++------ java/src/main/resources/persister-config.yml | 39 +++++++--------- 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 1acf9c4e..ca7f6a38 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ monasca-persister The Monitoring Persister consumes metrics and alarm state transitions from the Message Queue and stores them in the Metrics and Alarms database. -Although the Persister isn't primarly a Web service it uses DropWizard, https://dropwizard.github.io/dropwizard/, which provides a nice Web appliction framework to expose an http endpoint that provides an interface through which metrics about the Persister can be queried as well as health status. +Although the Persister isn't primarly a Web service it uses DropWizard, https://dropwizard.github.io/dropwizard/, which provides a nice Web appliction framework to expose an http endpoint that provides an interface through which metrics about the Persister can be queried as well as health status. The basic design of the Persister is to have one Kafka consumer publish to a Disruptor, https://github.com/LMAX-Exchange/disruptor, that has output processors. The output processors use prepared batch statements to write to the Metrics and Alarms database. -The number of output processors/threads in the Persister can be specified to scale to more messages. To horizontally scale and provide fault-tolerance any number of Persisters can be started as consumers from the the Message Queue. +The number of output processors/threads in the Persister can be specified to scale to more messages. To horizontally scale and provide fault-tolerance any number of Persisters can be started as consumers from the the Message Queue. # Build @@ -17,10 +17,16 @@ Requires monasca-common from https://github.com/stackforge/monasca-common. Downl mvn clean package ``` +# Configuration + +A sample configuration file is available in java/src/deb/etc/persister-config.yml-sample. + +A second configuration file is provided in java/src/main/resources/persister-config.yml for use with the [vagrant "mini-mon" development environment](https://github.com/stackforge/monasca-vagrant/). + # TODO * Purge metrics on shutdown -* Add more robust offset management in Kafka. Currently, the offset is advanced as each message is read. If the Persister stops after the metric has been read and prior to it being committed to the Metrics and Alarms database, the metric will be lost. +* Add more robust offset management in Kafka. Currently, the offset is advanced as each message is read. If the Persister stops after the metric has been read and prior to it being committed to the Metrics and Alarms database, the metric will be lost. * Add better handling of SQL exceptions. * Complete health check. * Specify and document the names of the metrics that are available for monitoring of the Persister. @@ -35,7 +41,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or diff --git a/java/src/deb/etc/persister-config.yml-sample b/java/src/deb/etc/persister-config.yml-sample index 47a6ed24..6c924d3e 100755 --- a/java/src/deb/etc/persister-config.yml-sample +++ b/java/src/deb/etc/persister-config.yml-sample @@ -4,25 +4,25 @@ alarmHistoryConfiguration: batchSize: 100 numThreads: 2 maxBatchTime: 15 -# See http://kafka.apache.org/documentation.html#api for semantics and defaults. + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. topic: alarm-state-transitions groupId: 1_alarm-state-transitions consumerId: mini-mon - clientId : 1 + clientId: 1 metricConfiguration: batchSize: 10000 numThreads: 4 maxBatchTime: 15 -# See http://kafka.apache.org/documentation.html#api for semantics and defaults. + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. topic: metrics groupId: 1_metrics consumerId: mini-mon - clientId : 1 + clientId: 1 #Kafka settings. kafkaConfig: -# See http://kafka.apache.org/documentation.html#api for semantics and defaults. + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. zookeeperConnect: localhost:2181 socketTimeoutMs: 30000 socketReceiveBufferBytes : 65536 @@ -43,15 +43,17 @@ verticaMetricRepoConfig: maxCacheSize: 2000000 databaseConfiguration: -# vertica | influxdb + # databaseType can be (vertica | influxdb) databaseType: influxdb # Uncomment if databaseType is influxdb influxDbConfiguration: # Retention policy may be left blank to indicate default policy. retentionPolicy: + # Used only if version is V9. maxHttpConnections: 100 name: mon + replicationFactor: 1 url: http://localhost:8086 user: mon_persister password: password @@ -63,24 +65,28 @@ influxDbConfiguration: # user: dbadmin # password: password # properties: -# ssl: false +# ssl: false # # the maximum amount of time to wait on an empty pool before throwing an exception # maxWaitForConnection: 1s +# # # the SQL query to run when validating a connection's liveness # validationQuery: "/* MyService Health Check */ SELECT 1" +# # # the minimum number of connections to keep open # minSize: 8 +# # # the maximum number of connections to keep open # maxSize: 41 +# # # whether or not idle connections should be validated # checkConnectionWhileIdle: false +# # # the maximum lifetime of an idle connection # maxConnectionAge: 1 minute metrics: frequency: 1 second - # Logging settings. logging: @@ -90,23 +96,38 @@ logging: # Logger-specific levels. loggers: - - # Sets the level for 'com.example.app' to DEBUG. - com.example.app: DEBUG - # com.hpcloud: debug - # com.hpcloud.mon.persister.repository: DEBUG + monasca: DEBUG appenders: + # Uncomment to enable logging to the console: + #- type: console + # threshold: DEBUG + # timeZone: UTC + # target: stdout - type: file threshold: INFO archive: true + # The file to which current statements will be logged. currentLogFilename: /var/log/monasca/persister/monasca-persister.log + + # When the log file rotates, the archived log will be renamed to this and gzipped. The + # %d is replaced with the previous day (yyyy-MM-dd). Custom rolling windows can be created + # by passing a SimpleDateFormat-compatible format as an argument: "%d{yyyy-MM-dd-hh}". archivedLogFilenamePattern: /var/log/monasca/persister/monasca-persister.log-%d.log.gz + + # The number of archived files to keep. archivedFileCount: 5 + # The timezone used to format dates. HINT: USE THE DEFAULT, UTC. timeZone: UTC + # Uncomment to approximately match the default log format of the python + # Openstack components. %pid is unavoidably formatted with [brackets], + # which are hard-coded in dropwizard's logging module. + # See http://logback.qos.ch/manual/layouts.html#conversionWord for details of the format string + # logFormat: "%app%pid: %d{YYYY-MM-dd HH:mm:ss.SSS} %pid %level %logger [-] [%thread] %msg %ex{1}" + # Set the persister ports to 8090/8091 to avoid conflict with the api server: applicationConnectors: diff --git a/java/src/main/resources/persister-config.yml b/java/src/main/resources/persister-config.yml index 0f644605..072e00d5 100644 --- a/java/src/main/resources/persister-config.yml +++ b/java/src/main/resources/persister-config.yml @@ -4,7 +4,7 @@ alarmHistoryConfiguration: batchSize: 100 numThreads: 1 maxBatchTime: 15 -# See http://kafka.apache.org/documentation.html#api for semantics and defaults. + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. topic: alarm-state-transitions groupId: persister_alarms consumerId: 1 @@ -14,7 +14,7 @@ metricConfiguration: batchSize: 1000 numThreads: 2 maxBatchTime: 30 -# See http://kafka.apache.org/documentation.html#api for semantics and defaults. + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. topic: metrics groupId: persister_metrics consumerId: 1 @@ -22,8 +22,7 @@ metricConfiguration: #Kafka settings. kafkaConfig: - #zookeeperConnect: localhost:2181 -# See http://kafka.apache.org/documentation.html#api for semantics and defaults. + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. zookeeperConnect: 192.168.10.4:2181 socketTimeoutMs: 30000 socketReceiveBufferBytes : 65536 @@ -44,10 +43,10 @@ verticaMetricRepoConfig: maxCacheSize: 2000000 databaseConfiguration: -# databaseType can be (vertica | influxdb) + # databaseType can be (vertica | influxdb) databaseType: influxdb -# databaseType: vertica +# Uncomment if databaseType is influxdb influxDbConfiguration: # Retention policy may be left blank to indicate default policy. retentionPolicy: @@ -56,19 +55,14 @@ influxDbConfiguration: name: mon replicationFactor: 1 url: http://192.168.10.4:8086 -# url: http://10.10.10.2:8086 user: root password: root +# Uncomment if databaseType is vertica #dataSourceFactory: # driverClass: com.vertica.jdbc.Driver -## url: jdbc:vertica://mini-mon:5433/monasca -# # my Vertica DB in the cloud -# #url: jdbc:vertica://15.185.94.245:5433/som -# url: jdbc:vertica://192.168.10.4:5433/mon -## user: persister -# #user: mon_persister -# user: dbadmin +# url: jdbc:vertica://192.168.10.4:5433/monasca +# user: monasca_persister # password: password # properties: # ssl: false @@ -98,17 +92,14 @@ logging: # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, # DEBUG, TRACE, or ALL. - level: info + level: INFO # Logger-specific levels. loggers: - - # Sets the level for 'com.example.app' to DEBUG. - com.example.app: DEBUG monasca: DEBUG -# monasca.persister.repository: DEBUG appenders: + - type: console threshold: DEBUG timeZone: UTC @@ -117,11 +108,6 @@ logging: - type: file threshold: DEBUG archive: true - currentLogFilename: ./logs/monasca-persister.log - archivedLogFilenamePattern: ./logs/monasca-persister-%d.log.gz - archivedFileCount: 5 - - - type: file # The file to which current statements will be logged. currentLogFilename: ./logs/monasca-persister.log @@ -136,3 +122,8 @@ logging: # The timezone used to format dates. HINT: USE THE DEFAULT, UTC. timeZone: UTC + # Uncomment to approximately match the default log format of the python + # Openstack components. %pid is unavoidably formatted with [brackets], + # which are hard-coded in dropwizard's logging module. + # See http://logback.qos.ch/manual/layouts.html#conversionWord for details of the format string + # logFormat: "%app%pid: %d{YYYY-MM-dd HH:mm:ss.SSS} %pid %level %logger [-] [%thread] %msg %ex{1}"