openstack/monasca-persister
Code Issues Proposed changes

Create deb.

Browse Source
This commit is contained in:
Deklan Dieterly 2014-02-26 10:28:31 -07:00
parent 50cc1e57af
commit 28d73219e2
3 changed files with 151 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
pom.xml
View File

@ -15,6 +15,20 @@
</repository> </repository>
</repositories> </repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<artifactNamedVersion>${project.name}-${project.version}-${timestamp}-${buildNumber}
</artifactNamedVersion>
</properties>
<!--Needed for buildnumber-maven-plugin-->
<scm>
<connection>scm:git:git@keg.dev.uswest.hpcloud.net:som-persister</connection>
<developerConnection>scm:git:git@keg.dev.uswest.hpcloud.net:som-persister
</developerConnection>
</scm>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.kafka</groupId> <groupId>org.apache.kafka</groupId>
@ -75,6 +89,16 @@
<artifactId>vertica-jdbc</artifactId> <artifactId>vertica-jdbc</artifactId>
<version>6.1.0</version> <version>6.1.0</version>
</dependency> </dependency>
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.vafer</groupId>
<artifactId>jdeb</artifactId>
<version>1.0.1</version>
</dependency>
</dependencies> </dependencies>
@ -96,6 +120,7 @@
<artifactId>maven-shade-plugin</artifactId> <artifactId>maven-shade-plugin</artifactId>
<version>2.2</version> <version>2.2</version>
<configuration> <configuration>
<finalName>${artifactNamedVersion}</finalName>
<createDependencyReducedPom>true</createDependencyReducedPom> <createDependencyReducedPom>true</createDependencyReducedPom>
<filters> <filters>
<filter> <filter>
@ -150,10 +175,27 @@
</filesets> </filesets>
</configuration> </configuration>
</plugin> </plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.2</version>
<executions>
<execution>
<phase>validate</phase>
<goals>
<goal>create</goal>
</goals>
</execution>
</executions>
<configuration>
<doCheck>false</doCheck>
<shortRevisionLength>6</shortRevisionLength>
</configuration>
</plugin>
<plugin> <plugin>
<artifactId>jdeb</artifactId> <artifactId>jdeb</artifactId>
<groupId>org.vafer</groupId> <groupId>org.vafer</groupId>
<version>1.0</version> <version>1.0.1</version>
<executions> <executions>
<execution> <execution>
<phase>package</phase> <phase>package</phase>
@ -161,11 +203,12 @@
<goal>jdeb</goal> <goal>jdeb</goal>
</goals> </goals>
<configuration> <configuration>
<deb>${project.basedir}/../debs/binaries/${artifactNamedVersion}.deb</deb> <deb>${project.basedir}/debs/binaries/${artifactNamedVersion}.deb</deb>
<dataSet> <dataSet>
<data> <data>
<type>file</type> <type>file</type>
<src>${project.build.directory}/${artifactNamedVersion}.jar</src> <src>${project.build.directory}/${artifactNamedVersion}.jar
</src>
<dst>/opt/mon/mon-persister.jar</dst> <dst>/opt/mon/mon-persister.jar</dst>
</data> </data>
<data> <data>
@ -175,7 +218,9 @@
</data> </data>
<data> <data>
<type>file</type> <type>file</type>
<src>${project.basedir}/src/deb/etc/persister-config.yml-sample</src> <src>
${project.basedir}/src/deb/etc/persister-config.yml-sample
</src>
<dst>/etc/mon/persister-config.yml-sample</dst> <dst>/etc/mon/persister-config.yml-sample</dst>
</data> </data>
</dataSet> </dataSet>
@ -186,10 +231,5 @@
</plugins> </plugins>
</build> </build>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
</project> </project>

405
src/deb/etc/persister-config.yml-sample
View File

@ -1,169 +1,46 @@
#Application Config name: mon-persister
metricsPersister: #Kafka settings.
enabled: true kafkaConfiguration:
csvConsumerEnabled: true # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
udpConsumerEnabled: true topic: test
numThreads: 2
groupId: 1
zookeeperConnect: localhost:2181
consumerId: 1
socketTimeoutMs: 30000
socketReceiveBufferBytes : 65536
fetchMessageMaxBytes: 1048576
autoCommitEnable: true
autoCommitIntervalMs: 60000
queuedMaxMessageChunks: 10
rebalanceMaxRetries: 4
fetchMinBytes: 1
fetchWaitMaxMs: 100
rebalanceBackoffMs: 2000
refreshLeaderBackoffMs: 200
autoOffsetReset: largest
consumerTimeoutMs: -1
clientId : 1
zookeeperSessionTimeoutMs : 6000
zookeeperConnectionTimeoutMs : 6000
zookeeperSyncTimeMs: 2000
disruptorConfig: disruptorConfiguration:
claimBufferSize: 262144 bufferSize: 1048576
numBatches: 8 numProcessors: 2
numOutputProcessors: 4
repositoryType: "vertica"
schemaName: Metrics
enableJournaling: false
enableAmqpOutputProcessor: false
udpConsumerConfig: verticaOutputProcessorConfiguration:
useMulticast: false batchSize: 10
port: 25826
ipAddress: 0.0.0.0
datagramPacketBufferSize: 4096
numDatagramPacketTranslatorProcessors: 4
csvConsumerConfig: databaseConfiguration:
inputFileDir: "/var/som/csv/toProcess"
processedFileDir : "/var/som/csv/processed"
somMaasConfig:
rabbitExchange: "metrics.rnd"
rabbit:
hosts: [15.185.167.30,15.185.102.80,15.185.224.94]
port: 5672
username: som
password: password
vhost: /
consumerThreadPoolSize: 1
maxWaitForConnection: 1s
connectRetryMilliseconds: 2000
filterType: "plugin"
filterMatch: "libvirt,bock"
amqpMetricsMsgBufferSize: 65536
numAmqpPublishers: 1
csPersister:
enabled: true
amqpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 512
numBatches: 1
numOutputProcessors: 1
repositoryType: "vertica"
schemaName: CS
enableJournaling: false
enableAmqpOutputProcessor: false
amqpConsumerConfig:
exchangeName: "cs_entities"
queueName: "control-services-sync"
routingKey: "#"
declareQueue: false
prefetchCount: 8192
connection:
hosts:
- som-rabbit.rndd.aw1.hpcloud.net
port: 5672
username: cspersister
password: <password>
vhost: /
consumerThreadPoolSize: 8
nagiosPersister:
enabled: true
amqpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 512
numBatches: 4
numOutputProcessors: 1
repositoryType: "vertica"
schemaName: Nagios
enableJournaling: false
enableAmqpOutputProcessor: false
amqpConsumerConfig:
exchangeName: "icinga_events"
queueName: "som-nagios-consumer"
routingKey: "#"
declareQueue: false
prefetchCount: 1024
connection:
hosts:
- som-rabbit.rndd.aw1.hpcloud.net
port: 5672
username: nagiospersister
password: <password>
vhost: /
consumerThreadPoolSize: 8
salesforcePersister:
enabled: true
httpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 512
numBatches: 1
numOutputProcessors: 0
repositoryType: "vertica"
schemaName: SalesForce
enableJournaling: false
enableAmqpOutputProcessor: false
httpConsumerConfig:
username: hpcs-mon-som@hp.com
password: <password>
securityToken: <token>
endPoint: https://salesforce.com/services/Soap/c/22.0/0DFV00000008OKG
setTrace: false
proxy: web-proxy.dev.uswest.hpcloud.net
port: 8080
apiVersion: 28.0
customMetricsPersister:
enabled: true
amqpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 512
numBatches : 1
numOutputProcessors: 16
repositoryType: http
schemaName: CustomMetrics
enableJournaling: false
enableAmqpOutputProcessor: false
amqpConsumerConfig:
exchangeName: "custom_metrics"
queueName: "som-custommetrics-consumer"
routingKey: "#"
declareQueue: true
prefetchCount: 512
connection:
hosts:
- som-rabbit.rndd.aw1.hpcloud.net
port: 5672
username: cmpersister
password: <password>
vhost: /custom_metrics
consumerThreadPoolSize: 8
httpProducerConfig:
endPoints:
- https://mon-aw1rdd1-somapi0001.rndd.aw1.hpcloud.net/v1/metrics/custom
- https://mon-aw1rdd1-somapi0002.rndd.aw1.hpcloud.net/v1/metrics/custom
maxConnections: 64
# Platform config
database:
driverClass: com.vertica.jdbc.Driver driverClass: com.vertica.jdbc.Driver
url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som # url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som
url: jdbc:vertica://15.185.94.245:5433/som
user: persister user: persister
password: <password> password: password
properties: properties:
ssl: true ssl: false
# the maximum amount of time to wait on an empty pool before throwing an exception # the maximum amount of time to wait on an empty pool before throwing an exception
maxWaitForConnection: 1s maxWaitForConnection: 1s
@ -174,8 +51,8 @@ database:
minSize: 8 minSize: 8
# the maximum number of connections to keep open # the maximum number of connections to keep open
# If you add a DB connection, this value will
# need to increase
maxSize: 41 maxSize: 41
# whether or not idle connections should be validated # whether or not idle connections should be validated
@ -187,108 +64,63 @@ database:
# the maximum lifetime of an idle connection # the maximum lifetime of an idle connection
closeConnectionIfIdleFor: 1 minute closeConnectionIfIdleFor: 1 minute
# HTTP-specific options. # Logging settings.
http: logging:
# The port on which the HTTP server listens for service requests. # The default level of all loggers. Can be OFF, ERROR, WARN, INFO,
port: 8080 # DEBUG, TRACE, or ALL.
level: DEBUG
# The port on which the HTTP server listens for administrative requests. # Logger-specific levels.
adminPort: 8081 loggers:
# Maximum number of threads. # Sets the level for 'com.example.app' to DEBUG.
maxThreads: 100 com.example.app: DEBUG
# Minimum number of thread to keep alive.
minThreads: 10
# The type of connector to use. Other valid values are "nonblocking" or "legacy". In general, the
# blocking connector should be used for low-latency services with short request durations. The
# nonblocking connector should be used for services with long request durations or which
# specifically take advantage of Jetty's continuation support.
# If you need SSL support, you can either choose from "nonblocking+ssl" or "legacy+ssl".
connectorType: blocking
# The maximum amount of time a connection is allowed to be idle before being closed.
maxIdleTime: 1s
# The number of threads dedicated to accepting connections. If omitted, this defaults to the
# number of logical CPUs on the current machine.
acceptorThreads: 3
# The offset of the acceptor threads' priorities. Can be [-5...5], with -5 dropping the acceptor
# threads to the lowest possible priority and with 5 raising them to the highest priority.
acceptorThreadPriorityOffset: 0
# The number of unaccepted requests to keep in the accept queue before refusing connections. If
# set to -1 or omitted, the system default is used.
acceptQueueSize: 100
# The maximum number of buffers to keep in memory.
maxBufferCount: 1024
# The initial buffer size for reading requests.
requestBufferSize: 32KB
# The initial buffer size for reading request headers.
requestHeaderBufferSize: 6KB
# The initial buffer size for writing responses.
responseBufferSize: 32KB
# The initial buffer size for writing response headers.
responseHeaderBufferSize: 6KB
# Enables SO_REUSEADDR on the server socket.
reuseAddress: true
# Enables SO_LINGER on the server socket with the specified linger time.
soLingerTime: 1s
# The number of open connections at which the server transitions to a "low-resources" mode.
lowResourcesConnectionThreshold: 25000
# When in low-resources mode, the maximum amount of time a connection is allowed to be idle before
# being closed. Overrides maxIdleTime.
lowResourcesMaxIdleTime: 5s
# If non-zero, the server will allow worker threads to finish processing requests after the server
# socket has been closed for the given amount of time.
shutdownGracePeriod: 2s
# If true, the HTTP server will prefer X-Forwarded headers over their non-forwarded equivalents.
useForwardedHeaders: true
# If true, forces the HTTP connector to use off-heap, direct buffers.
useDirectBuffers: true
# The hostname of the interface to which the HTTP server socket wil be found. If omitted, the
# socket will listen on all interfaces.
# bindHost: app1.example.com
# Trust store for SSL trusted Certificate Authorities
# ssl:
# trustStore: /etc/ssl/hpcs-keystore.jks
# trustStorePassword: <%= @keystore_pass %>
# HTTP request log settings
requestLog:
# Settings for logging to stdout. # Settings for logging to stdout.
console: console:
# If true, addToBatch log statements to stdout.
enabled: false
# Settings for logging to a file. Possibly to move to this and off syslog if we end up with man http requests # If true, write log statements to stdout.
enabled: true
# Do not display log statements below this threshold to stdout.
threshold: ALL
# The time zone in which dates should be displayed.
timeZone: UTC
# A custom Logback format string.
logFormat: null
# Settings for logging to a file.
file: file:
# If true, addToBatch log statements to a file.
enabled: false # If true, write log statements to a file.
enabled: true
# Do not write log statements below this threshold to the file.
threshold: ALL
# The time zone in which dates should be displayed.
timeZone: UTC
# A custom Logback format string.
logFormat: null
# The file to which statements will be logged. # The file to which statements will be logged.
currentLogFilename: ./logs/som-metrics-persister-requests.log #
# If enabled is true, this must be specified.
currentLogFilename: ./logs/mon-persister.log
# When the log file rolls over, the file will be archived to example-2012-03-15.log.gz, # If true, log files are rotated and archived.
# example.log will be truncated, and new statements written to it. archive: true
archivedLogFilenamePattern: ./logs/som-metrics-persister-requests-%d.log.gz
# When the log file rolls over, the file will be archived to
# app-2012-03-15.log.gz, example.log will be truncated,
# and new statements written to it.
#
# If archive is true, this must be specified.
archivedLogFilenamePattern: ./logs/mon-persister-%d.log.gz
# The maximum number of log files to archive. # The maximum number of log files to archive.
archivedFileCount: 5 archivedFileCount: 5
@ -296,63 +128,26 @@ http:
# Settings for logging to syslog. # Settings for logging to syslog.
syslog: syslog:
# If true, addToBatch log statements to syslog. # If true, write log statements to syslog.
enabled: true enabled: false
# The hostname of the syslog server to which statements will be sent. # The hostname of the syslog server to which statements will be
# N.B.: If this is the local host, the local syslog instance will need to be configured to # sent.
# listen on an inet socket, not just a Unix socket. #
# N.B.: If this is the local host, the local syslog instance
# will need to be configured to listen on an inet socket, not just
# a Unix socket.
host: localhost host: localhost
# The syslog facility to which statements will be sent. # The syslog facility to which statements will be sent.
#
# Can be one of: {AUTH, AUTHPRIV, DAEMON, CRON, FTP, LPR, KERN,
# MAIL, NEWS, SYSLOG, USER, UUCP, LOCAL0, LOCAL1, LOCAL2, LOCAL3,
# LOCAL4, LOCAL5, LOCAL6, LOCAL7}.
facility: local0 facility: local0
# Logging settings. # The time zone in which dates should be displayed.
logging: timeZone: UTC
# The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. # A custom Logback format string.
level: INFO logFormat: null
# Logger-specific levels.
loggers:
com.hpcloud: INFO
com.hpcloud.som: INFO
com.yammer.metrics.reporting.JmxReporter: ERROR
# Settings for logging to a file.
file:
# If true, addToBatch log statements to a file.
enabled: true
# Do not addToBatch log statements below this threshold to the file.
threshold: ALL
# The file to which statements will be logged.
currentLogFilename: /var/log/som/som-persister.log
# When the log file rolls over, the file will be archived to example-2012-03-15.log.gz,
# example.log will be truncated, and new statements written to it.
archivedLogFilenamePattern: /var/log/som/som-persister%d.log.gz
# The maximum number of log files to archive.
archivedFileCount: 10
# Settings for logging to stdout.
console:
# If true, addToBatch log statements to stdout.
enabled: false
# Do not display log statements below this threshold to stdout.
threshold: ALL
# Settings for logging to syslog.
syslog:
# If true, addToBatch log statements to syslog.
enabled: false
# The hostname of the syslog server to which statements will be sent.
# N.B.: If this is the local host, the local syslog instance will need to be configured to
# listen on an inet socket, not just a Unix socket.
host: localhost
# The syslog facility to which statements will be sent.
facility: local0

2
src/deb/init/mon-persister.conf
View File

@ -8,4 +8,4 @@ respawn
setgid persister setgid persister
setuid persister setuid persister
exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.mon.persister.PersisterApp --config /etc/mon/persister-config.yml exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.MonPersisterService --config /etc/mon/persister-config.yml