Support deploying Monasca Thresh

Monasca Thresh is a Storm topology which generates alerts from
metric streams according to alarms defined via the Monasca API.

This change runs the thresholder in local mode, which means that
the log output for the topology is directed to stdout and the
topology is restarted if the container is restarted. A future
change will improve the log collection and introduce a better
way of the checking the topology is running for multi-node
clusters.

Change-Id: I063dca5eead15f3cec009df62f0fc5d857dd4bb0
Partially-Implements: blueprint monasca-roles
This commit is contained in:
Doug Szumski 2018-06-13 16:26:06 +01:00
parent 516b0dd34e
commit b6cce3e3f3
10 changed files with 268 additions and 2 deletions

View File

@ -461,6 +461,9 @@ monasca
[monasca-log-metrics:children]
monasca
[monasca-thresh:children]
monasca
# Storm
[storm-worker:children]
storm

View File

@ -470,6 +470,9 @@ monasca
[monasca-log-metrics:children]
monasca
[monasca-thresh:children]
monasca
# Storm
[storm-worker:children]
storm

View File

@ -50,6 +50,17 @@ monasca_services:
- "/etc/localtime:/etc/localtime:ro"
- "kolla_logs:/var/log/kolla"
dimensions: "{{ monasca_log_metrics_dimensions }}"
monasca-thresh:
container_name: monasca_thresh
group: monasca-thresh
enabled: true
image: "{{ monasca_thresh_image_full }}"
volumes:
- "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "monasca_thresh:/var/lib/monasca-thresh/"
- "kolla_logs:/var/log/kolla"
dimensions: "{{ monasca_thresh_dimensions }}"
####################
# Databases
@ -69,10 +80,14 @@ monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansi
monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}"
monasca_storm_nimbus_servers: "{% for host in groups['storm-nimbus'] %}'{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}'{% if not loop.last %},{% endif %}{% endfor %}"
# Kafka topics used by Monasca services
monasca_metrics_topic: "metrics"
monasca_raw_logs_topic: "logs"
monasca_transformed_logs_topic: "transformed-logs"
monasca_events_topic: "events"
monasca_alarm_state_transitions_topic: "alarm-state-transitions"
# Processing pipeline threads. In a large scale deployment you will likely
# want to tune these with finer precision. For example, if you have a very
@ -100,11 +115,16 @@ monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{
monasca_logstash_tag: "{{ monasca_tag }}"
monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}"
monasca_thresh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-thresh"
monasca_thresh_tag: "{{ monasca_tag }}"
monasca_thresh_image_full: "{{ monasca_thresh_image }}:{{ monasca_thresh_tag }}"
monasca_api_dimensions: "{{ default_container_dimensions }}"
monasca_log_api_dimensions: "{{ default_container_dimensions }}"
monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
monasca_log_persister_dimensions: "{{ default_container_dimensions }}"
monasca_log_metrics_dimensions: "{{ default_container_dimensions }}"
monasca_thresh_dimensions: "{{ default_container_dimensions }}"
####################

View File

@ -108,3 +108,25 @@
- config_json.changed | bool
or monasca_log_metrics_confs.changed | bool
or monasca_log_metrics_container.changed | bool
- name: Restart monasca-thresh container
vars:
service_name: "monasca-thresh"
service: "{{ monasca_services[service_name] }}"
config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
monasca_thresh_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname in groups[service.group]
- service.enabled | bool
- config_json.changed | bool
or monasca_thresh_confs.changed | bool
or monasca_thresh_storm_conf.changed | bool
or monasca_thresh_container.changed | bool

View File

@ -175,6 +175,47 @@
notify:
- Restart monasca-log-metrics container
- name: Copying over monasca-thresh config
vars:
service: "{{ monasca_services['monasca-thresh'] }}"
# NOTE(dszumski): We can't use merge_yaml since it replaces empty values
# with `null`. This breaks the thresholder config file parsing (which should
# probably be more robust).
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/monasca-thresh/thresh-config.yml"
mode: "0660"
become: true
register: monasca_thresh_confs
with_first_found:
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/thresh-config.yml"
- "{{ node_custom_config }}/monasca/thresh-config.yml"
- "{{ role_path }}/templates/monasca-thresh/thresh-config.yml.j2"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-thresh container
- name: Copying over monasca-thresh storm config
vars:
service: "{{ monasca_services['monasca-thresh'] }}"
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/monasca-thresh/storm.yml"
mode: "0660"
become: true
register: monasca_thresh_storm_conf
with_first_found:
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/storm.yml"
- "{{ node_custom_config }}/monasca/storm.yml"
- "{{ role_path }}/templates/monasca-thresh/storm.yml.j2"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-thresh container
- name: Check monasca containers
become: true
kolla_docker:

View File

@ -8,7 +8,8 @@
inventory_hostname in groups['monasca-log-api'] or
inventory_hostname in groups['monasca-log-transformer'] or
inventory_hostname in groups['monasca-log-persister'] or
inventory_hostname in groups['monasca-log-metrics']
inventory_hostname in groups['monasca-log-metrics'] or
inventory_hostname in groups['monasca-thresh']
- include_tasks: bootstrap.yml
when: inventory_hostname in groups['monasca-api']
@ -21,4 +22,5 @@
inventory_hostname in groups['monasca-log-api'] or
inventory_hostname in groups['monasca-log-transformer'] or
inventory_hostname in groups['monasca-log-persister'] or
inventory_hostname in groups['monasca-log-metrics']
inventory_hostname in groups['monasca-log-metrics'] or
inventory_hostname in groups['monasca-thresh']

View File

@ -0,0 +1,29 @@
{
"command": "/opt/storm/bin/storm jar /monasca-thresh-source/monasca-thresh-*/thresh/target/monasca-thresh-*-SNAPSHOT-shaded.jar monasca.thresh.ThresholdingEngine /etc/monasca/thresh-config.yml monasca-thresh local",
"config_files": [
{
"source": "{{ container_config_directory }}/thresh-config.yml",
"dest": "/etc/monasca/thresh-config.yml",
"owner": "monasca",
"perm": "0600"
},
{
"source": "/var/lib/kolla/config_files/storm.yml",
"dest": "/opt/storm/conf/storm.yaml",
"owner": "monasca",
"perm": "0600"
}
],
"permissions": [
{
"path": "/var/log/kolla/monasca",
"owner": "monasca:kolla",
"recurse": true
},
{
"path": "/var/lib/monasca-thresh",
"owner": "monasca:kolla",
"recurse": true
}
]
}

View File

@ -0,0 +1,9 @@
storm.local.dir: "/var/lib/monasca-thresh/data"
storm.log.dir: "/var/log/kolla/storm"
storm.workers.artifacts.dir: "/var/lib/monasca-thresh/worker-artifacts"
nimbus.seeds: [{{ monasca_storm_nimbus_servers }}]
storm.zookeeper.port: {{ zookeeper_client_port }}
storm.zookeeper.servers:
{% for host in groups['zookeeper'] %}
- "{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}"
{% endfor %}

View File

@ -0,0 +1,132 @@
#
# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
# Copyright 2017 Fujitsu LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
metricSpoutThreads: 2
metricSpoutTasks: 2
#statsdConfig:
# host: "127.0.0.1"
# port: %MONASCA_STATSD_PORT%
# prefix: monasca.storm.
# dimensions: !!map
# service : monitoring
# component : storm
metricSpoutConfig:
kafkaConsumerConfiguration:
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
topic: "{{ monasca_metrics_topic }}"
numThreads: 1
groupId: "thresh-metric"
zookeeperConnect: "{{ monasca_zookeeper_servers }}"
consumerId: 1
socketTimeoutMs: 30000
socketReceiveBufferBytes: 65536
fetchMessageMaxBytes: 1048576
autoCommitEnable: true
autoCommitIntervalMs: 60000
queuedMaxMessageChunks: 10
rebalanceMaxRetries: 4
fetchMinBytes: 1
fetchWaitMaxMs: 100
rebalanceBackoffMs: 2000
refreshLeaderBackoffMs: 200
autoOffsetReset: largest
consumerTimeoutMs: -1
clientId: 1
zookeeperSessionTimeoutMs: 60000
zookeeperConnectionTimeoutMs: 60000
zookeeperSyncTimeMs: 2000
eventSpoutConfig:
kafkaConsumerConfiguration:
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
topic: "{{ monasca_events_topic }}"
numThreads: 1
groupId: "thresh-event"
zookeeperConnect: "{{ monasca_zookeeper_servers }}"
consumerId: 1
socketTimeoutMs: 30000
socketReceiveBufferBytes: 65536
fetchMessageMaxBytes: 1048576
autoCommitEnable: true
autoCommitIntervalMs: 60000
queuedMaxMessageChunks: 10
rebalanceMaxRetries: 4
fetchMinBytes: 1
fetchWaitMaxMs: 100
rebalanceBackoffMs: 2000
refreshLeaderBackoffMs: 200
autoOffsetReset: largest
consumerTimeoutMs: -1
clientId: 1
zookeeperSessionTimeoutMs: 60000
zookeeperConnectionTimeoutMs: 60000
zookeeperSyncTimeMs: 2000
kafkaProducerConfig:
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
topic: "{{ monasca_alarm_state_transitions_topic }}"
metadataBrokerList: "{{ monasca_kafka_servers }}"
serializerClass: kafka.serializer.StringEncoder
partitionerClass:
requestRequiredAcks: 1
requestTimeoutMs: 10000
producerType: sync
keySerializerClass:
compressionCodec: none
compressedTopics:
messageSendMaxRetries: 3
retryBackoffMs: 100
topicMetadataRefreshIntervalMs: 600000
queueBufferingMaxMs: 5000
queueBufferingMaxMessages: 10000
queueEnqueueTimeoutMs: -1
batchNumMessages: 200
sendBufferBytes: 102400
clientId: Threshold_Engine
sporadicMetricNamespaces:
- foo
database:
driverClass: org.drizzle.jdbc.DrizzleDriver
url: "jdbc:drizzle://{{ monasca_database_address }}/{{ monasca_database_name }}"
user: "{{ monasca_database_user }}"
password: "{{ monasca_database_password }}"
properties:
ssl: false
# the maximum amount of time to wait on an empty pool before throwing an exception
maxWaitForConnection: 1s
# the SQL query to run when validating a connection's liveness TODO FIXME
validationQuery: "/* MyService Health Check */ SELECT 1"
# the minimum number of connections to keep open
minSize: 8
# the maximum number of connections to keep open
maxSize: 41
hibernateSupport: false
# hibernate provider class
providerClass: com.zaxxer.hikari.hibernate.HikariConnectionProvider
databaseName: "{{ monasca_database_name }}"
serverName: "{{ database_address }}"
portNumber: "{{ database_port }}"
# hibernate auto configuration parameter
autoConfig: validate

View File

@ -0,0 +1,5 @@
---
features:
- |
Add support for deploying the Monasca thresh service, an Apache Storm
topology for alerting.