From 50cc1e57af680737dc6bee5e1c5b116032ed1808 Mon Sep 17 00:00:00 2001 From: Tim Kuhlman Date: Tue, 25 Feb 2014 14:45:00 -0700 Subject: [PATCH] Initial import of jdeb I expect that the config does not work at this point. --- pom.xml | 46 ++- src/deb/control/control | 9 + src/deb/control/postinst | 15 + src/deb/control/postrm | 9 + src/deb/control/prerm | 9 + src/deb/etc/persister-config.yml-sample | 358 ++++++++++++++++++++++++ src/deb/init/mon-persister.conf | 11 + 7 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 src/deb/control/control create mode 100755 src/deb/control/postinst create mode 100644 src/deb/control/postrm create mode 100644 src/deb/control/prerm create mode 100755 src/deb/etc/persister-config.yml-sample create mode 100644 src/deb/init/mon-persister.conf diff --git a/pom.xml b/pom.xml index 88938b51..1684b412 100644 --- a/pom.xml +++ b/pom.xml @@ -139,6 +139,50 @@ + + maven-clean-plugin + 2.5 + + + + ../debs + + + + + + jdeb + org.vafer + 1.0 + + + package + + jdeb + + + ${project.basedir}/../debs/binaries/${artifactNamedVersion}.deb + + + file + ${project.build.directory}/${artifactNamedVersion}.jar + /opt/mon/mon-persister.jar + + + file + ${project.basedir}/src/deb/init/mon-persister.conf + /etc/init/mon-persister.conf + + + file + ${project.basedir}/src/deb/etc/persister-config.yml-sample + /etc/mon/persister-config.yml-sample + + + + + + @@ -148,4 +192,4 @@ - \ No newline at end of file + diff --git a/src/deb/control/control b/src/deb/control/control new file mode 100644 index 00000000..171cbb26 --- /dev/null +++ b/src/deb/control/control @@ -0,0 +1,9 @@ +Package: [[name]] +Section: misc +Priority: optional +Architecture: all +Depends: openjdk-7-jre-headless | openjdk-7-jre +Version: [[version]]-[[timestamp]]-[[buildNumber]] +Maintainer: HPCloud Monitoring +Description: Mon-Persister + Pulls data from various sources and inserts into the SOM DB. diff --git a/src/deb/control/postinst b/src/deb/control/postinst new file mode 100755 index 00000000..090993c2 --- /dev/null +++ b/src/deb/control/postinst @@ -0,0 +1,15 @@ +#!/bin/sh + +case "$1" in + configure) + if ! [ `getent passwd persister` ]; then + adduser --system --group persister + fi + + #Restart does not pick up upstart job config changes stop/start is needed + stop mon-persister + start mon-persister + ;; +esac + +exit 0 diff --git a/src/deb/control/postrm b/src/deb/control/postrm new file mode 100644 index 00000000..78d5d28b --- /dev/null +++ b/src/deb/control/postrm @@ -0,0 +1,9 @@ +#!/bin/sh + +case "$1" in + remove|purge) + userdel persister + ;; +esac + +exit 0 diff --git a/src/deb/control/prerm b/src/deb/control/prerm new file mode 100644 index 00000000..aab6f389 --- /dev/null +++ b/src/deb/control/prerm @@ -0,0 +1,9 @@ +#!/bin/sh + +case "$1" in + remove) + stop mon-persister + ;; +esac + +exit 0 diff --git a/src/deb/etc/persister-config.yml-sample b/src/deb/etc/persister-config.yml-sample new file mode 100755 index 00000000..c76985a4 --- /dev/null +++ b/src/deb/etc/persister-config.yml-sample @@ -0,0 +1,358 @@ +#Application Config + +metricsPersister: + enabled: true + csvConsumerEnabled: true + udpConsumerEnabled: true + + disruptorConfig: + claimBufferSize: 262144 + numBatches: 8 + numOutputProcessors: 4 + repositoryType: "vertica" + schemaName: Metrics + enableJournaling: false + enableAmqpOutputProcessor: false + + udpConsumerConfig: + useMulticast: false + port: 25826 + ipAddress: 0.0.0.0 + datagramPacketBufferSize: 4096 + numDatagramPacketTranslatorProcessors: 4 + + csvConsumerConfig: + inputFileDir: "/var/som/csv/toProcess" + processedFileDir : "/var/som/csv/processed" + + somMaasConfig: + rabbitExchange: "metrics.rnd" + rabbit: + hosts: [15.185.167.30,15.185.102.80,15.185.224.94] + port: 5672 + username: som + password: password + vhost: / + consumerThreadPoolSize: 1 + maxWaitForConnection: 1s + connectRetryMilliseconds: 2000 + filterType: "plugin" + filterMatch: "libvirt,bock" + amqpMetricsMsgBufferSize: 65536 + numAmqpPublishers: 1 + +csPersister: + enabled: true + amqpConsumerEnabled: true + + disruptorConfig: + claimBufferSize: 512 + numBatches: 1 + numOutputProcessors: 1 + repositoryType: "vertica" + schemaName: CS + enableJournaling: false + enableAmqpOutputProcessor: false + + amqpConsumerConfig: + exchangeName: "cs_entities" + queueName: "control-services-sync" + routingKey: "#" + declareQueue: false + prefetchCount: 8192 + connection: + hosts: + - som-rabbit.rndd.aw1.hpcloud.net + port: 5672 + username: cspersister + password: + vhost: / + consumerThreadPoolSize: 8 + +nagiosPersister: + enabled: true + amqpConsumerEnabled: true + + disruptorConfig: + claimBufferSize: 512 + numBatches: 4 + numOutputProcessors: 1 + repositoryType: "vertica" + schemaName: Nagios + enableJournaling: false + enableAmqpOutputProcessor: false + + amqpConsumerConfig: + exchangeName: "icinga_events" + queueName: "som-nagios-consumer" + routingKey: "#" + declareQueue: false + prefetchCount: 1024 + + connection: + hosts: + - som-rabbit.rndd.aw1.hpcloud.net + port: 5672 + username: nagiospersister + password: + vhost: / + consumerThreadPoolSize: 8 + +salesforcePersister: + enabled: true + httpConsumerEnabled: true + + disruptorConfig: + claimBufferSize: 512 + numBatches: 1 + numOutputProcessors: 0 + repositoryType: "vertica" + schemaName: SalesForce + enableJournaling: false + enableAmqpOutputProcessor: false + + httpConsumerConfig: + username: hpcs-mon-som@hp.com + password: + securityToken: + endPoint: https://salesforce.com/services/Soap/c/22.0/0DFV00000008OKG + setTrace: false + proxy: web-proxy.dev.uswest.hpcloud.net + port: 8080 + apiVersion: 28.0 + +customMetricsPersister: + enabled: true + amqpConsumerEnabled: true + + disruptorConfig: + claimBufferSize: 512 + numBatches : 1 + numOutputProcessors: 16 + repositoryType: http + schemaName: CustomMetrics + enableJournaling: false + enableAmqpOutputProcessor: false + + amqpConsumerConfig: + exchangeName: "custom_metrics" + queueName: "som-custommetrics-consumer" + routingKey: "#" + declareQueue: true + prefetchCount: 512 + connection: + hosts: + - som-rabbit.rndd.aw1.hpcloud.net + port: 5672 + username: cmpersister + password: + vhost: /custom_metrics + consumerThreadPoolSize: 8 + + httpProducerConfig: + endPoints: + - https://mon-aw1rdd1-somapi0001.rndd.aw1.hpcloud.net/v1/metrics/custom + - https://mon-aw1rdd1-somapi0002.rndd.aw1.hpcloud.net/v1/metrics/custom + maxConnections: 64 + +# Platform config + +database: + driverClass: com.vertica.jdbc.Driver + url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som + user: persister + password: + properties: + ssl: true + # the maximum amount of time to wait on an empty pool before throwing an exception + maxWaitForConnection: 1s + + # the SQL query to run when validating a connection's liveness + validationQuery: "/* MyService Health Check */ SELECT 1" + + # the minimum number of connections to keep open + minSize: 8 + + # the maximum number of connections to keep open + # If you add a DB connection, this value will + # need to increase + maxSize: 41 + + # whether or not idle connections should be validated + checkConnectionWhileIdle: false + + # how long a connection must be held before it can be validated + checkConnectionHealthWhenIdleFor: 10s + + # the maximum lifetime of an idle connection + closeConnectionIfIdleFor: 1 minute + +# HTTP-specific options. +http: + + # The port on which the HTTP server listens for service requests. + port: 8080 + + # The port on which the HTTP server listens for administrative requests. + adminPort: 8081 + + # Maximum number of threads. + maxThreads: 100 + + # Minimum number of thread to keep alive. + minThreads: 10 + + # The type of connector to use. Other valid values are "nonblocking" or "legacy". In general, the + # blocking connector should be used for low-latency services with short request durations. The + # nonblocking connector should be used for services with long request durations or which + # specifically take advantage of Jetty's continuation support. + # If you need SSL support, you can either choose from "nonblocking+ssl" or "legacy+ssl". + connectorType: blocking + + # The maximum amount of time a connection is allowed to be idle before being closed. + maxIdleTime: 1s + + # The number of threads dedicated to accepting connections. If omitted, this defaults to the + # number of logical CPUs on the current machine. + acceptorThreads: 3 + + # The offset of the acceptor threads' priorities. Can be [-5...5], with -5 dropping the acceptor + # threads to the lowest possible priority and with 5 raising them to the highest priority. + acceptorThreadPriorityOffset: 0 + + # The number of unaccepted requests to keep in the accept queue before refusing connections. If + # set to -1 or omitted, the system default is used. + acceptQueueSize: 100 + + # The maximum number of buffers to keep in memory. + maxBufferCount: 1024 + + # The initial buffer size for reading requests. + requestBufferSize: 32KB + + # The initial buffer size for reading request headers. + requestHeaderBufferSize: 6KB + + # The initial buffer size for writing responses. + responseBufferSize: 32KB + + # The initial buffer size for writing response headers. + responseHeaderBufferSize: 6KB + + # Enables SO_REUSEADDR on the server socket. + reuseAddress: true + + # Enables SO_LINGER on the server socket with the specified linger time. + soLingerTime: 1s + + # The number of open connections at which the server transitions to a "low-resources" mode. + lowResourcesConnectionThreshold: 25000 + + # When in low-resources mode, the maximum amount of time a connection is allowed to be idle before + # being closed. Overrides maxIdleTime. + lowResourcesMaxIdleTime: 5s + + # If non-zero, the server will allow worker threads to finish processing requests after the server + # socket has been closed for the given amount of time. + shutdownGracePeriod: 2s + + # If true, the HTTP server will prefer X-Forwarded headers over their non-forwarded equivalents. + useForwardedHeaders: true + + # If true, forces the HTTP connector to use off-heap, direct buffers. + useDirectBuffers: true + + # The hostname of the interface to which the HTTP server socket wil be found. If omitted, the + # socket will listen on all interfaces. + # bindHost: app1.example.com + +# Trust store for SSL trusted Certificate Authorities +# ssl: +# trustStore: /etc/ssl/hpcs-keystore.jks +# trustStorePassword: <%= @keystore_pass %> + + # HTTP request log settings + requestLog: + # Settings for logging to stdout. + console: + # If true, addToBatch log statements to stdout. + enabled: false + + # Settings for logging to a file. Possibly to move to this and off syslog if we end up with man http requests + file: + # If true, addToBatch log statements to a file. + enabled: false + + # The file to which statements will be logged. + currentLogFilename: ./logs/som-metrics-persister-requests.log + + # When the log file rolls over, the file will be archived to example-2012-03-15.log.gz, + # example.log will be truncated, and new statements written to it. + archivedLogFilenamePattern: ./logs/som-metrics-persister-requests-%d.log.gz + + # The maximum number of log files to archive. + archivedFileCount: 5 + + # Settings for logging to syslog. + syslog: + + # If true, addToBatch log statements to syslog. + enabled: true + + # The hostname of the syslog server to which statements will be sent. + # N.B.: If this is the local host, the local syslog instance will need to be configured to + # listen on an inet socket, not just a Unix socket. + host: localhost + + # The syslog facility to which statements will be sent. + facility: local0 + +# Logging settings. +logging: + + # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. + level: INFO + + # Logger-specific levels. + loggers: + + com.hpcloud: INFO + com.hpcloud.som: INFO + com.yammer.metrics.reporting.JmxReporter: ERROR + +# Settings for logging to a file. + file: + # If true, addToBatch log statements to a file. + enabled: true + # Do not addToBatch log statements below this threshold to the file. + threshold: ALL + # The file to which statements will be logged. + currentLogFilename: /var/log/som/som-persister.log + # When the log file rolls over, the file will be archived to example-2012-03-15.log.gz, + # example.log will be truncated, and new statements written to it. + archivedLogFilenamePattern: /var/log/som/som-persister%d.log.gz + # The maximum number of log files to archive. + archivedFileCount: 10 + + # Settings for logging to stdout. + console: + + # If true, addToBatch log statements to stdout. + enabled: false + + # Do not display log statements below this threshold to stdout. + threshold: ALL + + # Settings for logging to syslog. + syslog: + + # If true, addToBatch log statements to syslog. + enabled: false + + # The hostname of the syslog server to which statements will be sent. + # N.B.: If this is the local host, the local syslog instance will need to be configured to + # listen on an inet socket, not just a Unix socket. + host: localhost + + # The syslog facility to which statements will be sent. + facility: local0 diff --git a/src/deb/init/mon-persister.conf b/src/deb/init/mon-persister.conf new file mode 100644 index 00000000..f25fbcd0 --- /dev/null +++ b/src/deb/init/mon-persister.conf @@ -0,0 +1,11 @@ +# Startup script for the mon-persister + +description "MON Persister java app" +start on runlevel [2345] + +console log +respawn + +setgid persister +setuid persister +exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.mon.persister.PersisterApp --config /etc/mon/persister-config.yml