Support deploying Monasca Thresh
Monasca Thresh is a Storm topology which generates alerts from metric streams according to alarms defined via the Monasca API. This change runs the thresholder in local mode, which means that the log output for the topology is directed to stdout and the topology is restarted if the container is restarted. A future change will improve the log collection and introduce a better way of the checking the topology is running for multi-node clusters. Change-Id: I063dca5eead15f3cec009df62f0fc5d857dd4bb0 Partially-Implements: blueprint monasca-roles
This commit is contained in:
parent
516b0dd34e
commit
b6cce3e3f3
@ -461,6 +461,9 @@ monasca
|
||||
[monasca-log-metrics:children]
|
||||
monasca
|
||||
|
||||
[monasca-thresh:children]
|
||||
monasca
|
||||
|
||||
# Storm
|
||||
[storm-worker:children]
|
||||
storm
|
||||
|
@ -470,6 +470,9 @@ monasca
|
||||
[monasca-log-metrics:children]
|
||||
monasca
|
||||
|
||||
[monasca-thresh:children]
|
||||
monasca
|
||||
|
||||
# Storm
|
||||
[storm-worker:children]
|
||||
storm
|
||||
|
@ -50,6 +50,17 @@ monasca_services:
|
||||
- "/etc/localtime:/etc/localtime:ro"
|
||||
- "kolla_logs:/var/log/kolla"
|
||||
dimensions: "{{ monasca_log_metrics_dimensions }}"
|
||||
monasca-thresh:
|
||||
container_name: monasca_thresh
|
||||
group: monasca-thresh
|
||||
enabled: true
|
||||
image: "{{ monasca_thresh_image_full }}"
|
||||
volumes:
|
||||
- "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro"
|
||||
- "/etc/localtime:/etc/localtime:ro"
|
||||
- "monasca_thresh:/var/lib/monasca-thresh/"
|
||||
- "kolla_logs:/var/log/kolla"
|
||||
dimensions: "{{ monasca_thresh_dimensions }}"
|
||||
|
||||
####################
|
||||
# Databases
|
||||
@ -69,10 +80,14 @@ monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansi
|
||||
monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
|
||||
monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
|
||||
monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}"
|
||||
monasca_storm_nimbus_servers: "{% for host in groups['storm-nimbus'] %}'{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}'{% if not loop.last %},{% endif %}{% endfor %}"
|
||||
|
||||
# Kafka topics used by Monasca services
|
||||
monasca_metrics_topic: "metrics"
|
||||
monasca_raw_logs_topic: "logs"
|
||||
monasca_transformed_logs_topic: "transformed-logs"
|
||||
monasca_events_topic: "events"
|
||||
monasca_alarm_state_transitions_topic: "alarm-state-transitions"
|
||||
|
||||
# Processing pipeline threads. In a large scale deployment you will likely
|
||||
# want to tune these with finer precision. For example, if you have a very
|
||||
@ -100,11 +115,16 @@ monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{
|
||||
monasca_logstash_tag: "{{ monasca_tag }}"
|
||||
monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}"
|
||||
|
||||
monasca_thresh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-thresh"
|
||||
monasca_thresh_tag: "{{ monasca_tag }}"
|
||||
monasca_thresh_image_full: "{{ monasca_thresh_image }}:{{ monasca_thresh_tag }}"
|
||||
|
||||
monasca_api_dimensions: "{{ default_container_dimensions }}"
|
||||
monasca_log_api_dimensions: "{{ default_container_dimensions }}"
|
||||
monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
|
||||
monasca_log_persister_dimensions: "{{ default_container_dimensions }}"
|
||||
monasca_log_metrics_dimensions: "{{ default_container_dimensions }}"
|
||||
monasca_thresh_dimensions: "{{ default_container_dimensions }}"
|
||||
|
||||
|
||||
####################
|
||||
|
@ -108,3 +108,25 @@
|
||||
- config_json.changed | bool
|
||||
or monasca_log_metrics_confs.changed | bool
|
||||
or monasca_log_metrics_container.changed | bool
|
||||
|
||||
- name: Restart monasca-thresh container
|
||||
vars:
|
||||
service_name: "monasca-thresh"
|
||||
service: "{{ monasca_services[service_name] }}"
|
||||
config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
|
||||
monasca_thresh_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
|
||||
kolla_docker:
|
||||
action: "recreate_or_restart_container"
|
||||
common_options: "{{ docker_common_options }}"
|
||||
name: "{{ service.container_name }}"
|
||||
image: "{{ service.image }}"
|
||||
volumes: "{{ service.volumes }}"
|
||||
dimensions: "{{ service.dimensions }}"
|
||||
when:
|
||||
- kolla_action != "config"
|
||||
- inventory_hostname in groups[service.group]
|
||||
- service.enabled | bool
|
||||
- config_json.changed | bool
|
||||
or monasca_thresh_confs.changed | bool
|
||||
or monasca_thresh_storm_conf.changed | bool
|
||||
or monasca_thresh_container.changed | bool
|
||||
|
@ -175,6 +175,47 @@
|
||||
notify:
|
||||
- Restart monasca-log-metrics container
|
||||
|
||||
- name: Copying over monasca-thresh config
|
||||
vars:
|
||||
service: "{{ monasca_services['monasca-thresh'] }}"
|
||||
# NOTE(dszumski): We can't use merge_yaml since it replaces empty values
|
||||
# with `null`. This breaks the thresholder config file parsing (which should
|
||||
# probably be more robust).
|
||||
template:
|
||||
src: "{{ item }}"
|
||||
dest: "{{ node_config_directory }}/monasca-thresh/thresh-config.yml"
|
||||
mode: "0660"
|
||||
become: true
|
||||
register: monasca_thresh_confs
|
||||
with_first_found:
|
||||
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/thresh-config.yml"
|
||||
- "{{ node_custom_config }}/monasca/thresh-config.yml"
|
||||
- "{{ role_path }}/templates/monasca-thresh/thresh-config.yml.j2"
|
||||
when:
|
||||
- inventory_hostname in groups[service['group']]
|
||||
- service.enabled | bool
|
||||
notify:
|
||||
- Restart monasca-thresh container
|
||||
|
||||
- name: Copying over monasca-thresh storm config
|
||||
vars:
|
||||
service: "{{ monasca_services['monasca-thresh'] }}"
|
||||
template:
|
||||
src: "{{ item }}"
|
||||
dest: "{{ node_config_directory }}/monasca-thresh/storm.yml"
|
||||
mode: "0660"
|
||||
become: true
|
||||
register: monasca_thresh_storm_conf
|
||||
with_first_found:
|
||||
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/storm.yml"
|
||||
- "{{ node_custom_config }}/monasca/storm.yml"
|
||||
- "{{ role_path }}/templates/monasca-thresh/storm.yml.j2"
|
||||
when:
|
||||
- inventory_hostname in groups[service['group']]
|
||||
- service.enabled | bool
|
||||
notify:
|
||||
- Restart monasca-thresh container
|
||||
|
||||
- name: Check monasca containers
|
||||
become: true
|
||||
kolla_docker:
|
||||
|
@ -8,7 +8,8 @@
|
||||
inventory_hostname in groups['monasca-log-api'] or
|
||||
inventory_hostname in groups['monasca-log-transformer'] or
|
||||
inventory_hostname in groups['monasca-log-persister'] or
|
||||
inventory_hostname in groups['monasca-log-metrics']
|
||||
inventory_hostname in groups['monasca-log-metrics'] or
|
||||
inventory_hostname in groups['monasca-thresh']
|
||||
|
||||
- include_tasks: bootstrap.yml
|
||||
when: inventory_hostname in groups['monasca-api']
|
||||
@ -21,4 +22,5 @@
|
||||
inventory_hostname in groups['monasca-log-api'] or
|
||||
inventory_hostname in groups['monasca-log-transformer'] or
|
||||
inventory_hostname in groups['monasca-log-persister'] or
|
||||
inventory_hostname in groups['monasca-log-metrics']
|
||||
inventory_hostname in groups['monasca-log-metrics'] or
|
||||
inventory_hostname in groups['monasca-thresh']
|
||||
|
@ -0,0 +1,29 @@
|
||||
{
|
||||
"command": "/opt/storm/bin/storm jar /monasca-thresh-source/monasca-thresh-*/thresh/target/monasca-thresh-*-SNAPSHOT-shaded.jar monasca.thresh.ThresholdingEngine /etc/monasca/thresh-config.yml monasca-thresh local",
|
||||
"config_files": [
|
||||
{
|
||||
"source": "{{ container_config_directory }}/thresh-config.yml",
|
||||
"dest": "/etc/monasca/thresh-config.yml",
|
||||
"owner": "monasca",
|
||||
"perm": "0600"
|
||||
},
|
||||
{
|
||||
"source": "/var/lib/kolla/config_files/storm.yml",
|
||||
"dest": "/opt/storm/conf/storm.yaml",
|
||||
"owner": "monasca",
|
||||
"perm": "0600"
|
||||
}
|
||||
],
|
||||
"permissions": [
|
||||
{
|
||||
"path": "/var/log/kolla/monasca",
|
||||
"owner": "monasca:kolla",
|
||||
"recurse": true
|
||||
},
|
||||
{
|
||||
"path": "/var/lib/monasca-thresh",
|
||||
"owner": "monasca:kolla",
|
||||
"recurse": true
|
||||
}
|
||||
]
|
||||
}
|
@ -0,0 +1,9 @@
|
||||
storm.local.dir: "/var/lib/monasca-thresh/data"
|
||||
storm.log.dir: "/var/log/kolla/storm"
|
||||
storm.workers.artifacts.dir: "/var/lib/monasca-thresh/worker-artifacts"
|
||||
nimbus.seeds: [{{ monasca_storm_nimbus_servers }}]
|
||||
storm.zookeeper.port: {{ zookeeper_client_port }}
|
||||
storm.zookeeper.servers:
|
||||
{% for host in groups['zookeeper'] %}
|
||||
- "{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}"
|
||||
{% endfor %}
|
@ -0,0 +1,132 @@
|
||||
#
|
||||
# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
|
||||
# Copyright 2017 Fujitsu LIMITED
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
metricSpoutThreads: 2
|
||||
metricSpoutTasks: 2
|
||||
|
||||
#statsdConfig:
|
||||
# host: "127.0.0.1"
|
||||
# port: %MONASCA_STATSD_PORT%
|
||||
# prefix: monasca.storm.
|
||||
# dimensions: !!map
|
||||
# service : monitoring
|
||||
# component : storm
|
||||
|
||||
|
||||
metricSpoutConfig:
|
||||
kafkaConsumerConfiguration:
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
topic: "{{ monasca_metrics_topic }}"
|
||||
numThreads: 1
|
||||
groupId: "thresh-metric"
|
||||
zookeeperConnect: "{{ monasca_zookeeper_servers }}"
|
||||
consumerId: 1
|
||||
socketTimeoutMs: 30000
|
||||
socketReceiveBufferBytes: 65536
|
||||
fetchMessageMaxBytes: 1048576
|
||||
autoCommitEnable: true
|
||||
autoCommitIntervalMs: 60000
|
||||
queuedMaxMessageChunks: 10
|
||||
rebalanceMaxRetries: 4
|
||||
fetchMinBytes: 1
|
||||
fetchWaitMaxMs: 100
|
||||
rebalanceBackoffMs: 2000
|
||||
refreshLeaderBackoffMs: 200
|
||||
autoOffsetReset: largest
|
||||
consumerTimeoutMs: -1
|
||||
clientId: 1
|
||||
zookeeperSessionTimeoutMs: 60000
|
||||
zookeeperConnectionTimeoutMs: 60000
|
||||
zookeeperSyncTimeMs: 2000
|
||||
|
||||
|
||||
eventSpoutConfig:
|
||||
kafkaConsumerConfiguration:
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
topic: "{{ monasca_events_topic }}"
|
||||
numThreads: 1
|
||||
groupId: "thresh-event"
|
||||
zookeeperConnect: "{{ monasca_zookeeper_servers }}"
|
||||
consumerId: 1
|
||||
socketTimeoutMs: 30000
|
||||
socketReceiveBufferBytes: 65536
|
||||
fetchMessageMaxBytes: 1048576
|
||||
autoCommitEnable: true
|
||||
autoCommitIntervalMs: 60000
|
||||
queuedMaxMessageChunks: 10
|
||||
rebalanceMaxRetries: 4
|
||||
fetchMinBytes: 1
|
||||
fetchWaitMaxMs: 100
|
||||
rebalanceBackoffMs: 2000
|
||||
refreshLeaderBackoffMs: 200
|
||||
autoOffsetReset: largest
|
||||
consumerTimeoutMs: -1
|
||||
clientId: 1
|
||||
zookeeperSessionTimeoutMs: 60000
|
||||
zookeeperConnectionTimeoutMs: 60000
|
||||
zookeeperSyncTimeMs: 2000
|
||||
|
||||
|
||||
kafkaProducerConfig:
|
||||
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
|
||||
topic: "{{ monasca_alarm_state_transitions_topic }}"
|
||||
metadataBrokerList: "{{ monasca_kafka_servers }}"
|
||||
serializerClass: kafka.serializer.StringEncoder
|
||||
partitionerClass:
|
||||
requestRequiredAcks: 1
|
||||
requestTimeoutMs: 10000
|
||||
producerType: sync
|
||||
keySerializerClass:
|
||||
compressionCodec: none
|
||||
compressedTopics:
|
||||
messageSendMaxRetries: 3
|
||||
retryBackoffMs: 100
|
||||
topicMetadataRefreshIntervalMs: 600000
|
||||
queueBufferingMaxMs: 5000
|
||||
queueBufferingMaxMessages: 10000
|
||||
queueEnqueueTimeoutMs: -1
|
||||
batchNumMessages: 200
|
||||
sendBufferBytes: 102400
|
||||
clientId: Threshold_Engine
|
||||
|
||||
|
||||
sporadicMetricNamespaces:
|
||||
- foo
|
||||
|
||||
database:
|
||||
driverClass: org.drizzle.jdbc.DrizzleDriver
|
||||
url: "jdbc:drizzle://{{ monasca_database_address }}/{{ monasca_database_name }}"
|
||||
user: "{{ monasca_database_user }}"
|
||||
password: "{{ monasca_database_password }}"
|
||||
properties:
|
||||
ssl: false
|
||||
# the maximum amount of time to wait on an empty pool before throwing an exception
|
||||
maxWaitForConnection: 1s
|
||||
# the SQL query to run when validating a connection's liveness TODO FIXME
|
||||
validationQuery: "/* MyService Health Check */ SELECT 1"
|
||||
# the minimum number of connections to keep open
|
||||
minSize: 8
|
||||
# the maximum number of connections to keep open
|
||||
maxSize: 41
|
||||
hibernateSupport: false
|
||||
# hibernate provider class
|
||||
providerClass: com.zaxxer.hikari.hibernate.HikariConnectionProvider
|
||||
databaseName: "{{ monasca_database_name }}"
|
||||
serverName: "{{ database_address }}"
|
||||
portNumber: "{{ database_port }}"
|
||||
# hibernate auto configuration parameter
|
||||
autoConfig: validate
|
@ -0,0 +1,5 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
Add support for deploying the Monasca thresh service, an Apache Storm
|
||||
topology for alerting.
|
Loading…
Reference in New Issue
Block a user