326 lines
15 KiB
YAML
326 lines
15 KiB
YAML
---
|
|
conf:
|
|
prometheus:
|
|
rules:
|
|
openstack:
|
|
groups:
|
|
- name: mariadb.rules
|
|
rules:
|
|
- alert: prom_exporter_mariadb_openstack_unavailable
|
|
expr: avg_over_time(up{job="mysql-exporter",kubernetes_namespace="openstack"}[5m]) == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: MariaDB exporter in {{ $labels.kubernetes_namespace }} is not collecting metrics or is not available for past 10 minutes
|
|
title: MariaDB exporter is not collecting metrics or is not available
|
|
- alert: prom_exporter_mariadb_osh_infra_unavailable
|
|
expr: avg_over_time(up{job="mysql-exporter",kubernetes_namespace="osh-infra"}[5m]) == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: MariaDB exporter in {{ $labels.kubernetes_namespace }} is not collecting metrics or is not available for past 10 minutes
|
|
title: MariaDB exporter is not collecting metrics or is not available
|
|
- alert: mariadb_table_lock_wait_high
|
|
expr: 100 * mysql_global_status_table_locks_waited/(mysql_global_status_table_locks_waited + mysql_global_status_table_locks_immediate) > 30
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'Mariadb has high table lock waits of {{ $value }} percentage'
|
|
summary: 'Mariadb table lock waits are high'
|
|
- alert: mariadb_node_not_ready
|
|
expr: mysql_global_status_wsrep_ready != 1
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: '{{$labels.job}} on {{$labels.instance}} is not ready.'
|
|
summary: 'Galera cluster node not ready'
|
|
- alert: mariadb_galera_node_out_of_sync
|
|
expr: mysql_global_status_wsrep_local_state != 4 AND mysql_global_variables_wsrep_desync == 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: '{{$labels.job}} on {{$labels.instance}} is not in sync ({{$value}} != 4)'
|
|
summary: 'Galera cluster node out of sync'
|
|
- alert: mariadb_innodb_replication_fallen_behind
|
|
expr: (mysql_global_variables_innodb_replication_delay > 30) AND on (instance) (predict_linear(mysql_global_variables_innodb_replication_delay[5m], 60*2) > 0)
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'The mysql innodb replication has fallen behind and is not recovering'
|
|
summary: 'MySQL innodb replication is lagging'
|
|
- name: openstack.rules
|
|
rules:
|
|
- alert: prom_exporter_openstack_unavailable
|
|
expr: avg_over_time(up{job="openstack-metrics"}[5m]) == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: Openstack exporter is not collecting metrics or is not available for past 10 minutes
|
|
title: Openstack exporter is not collecting metrics or is not available
|
|
- alert: os_glance_api_availability
|
|
expr: openstack_check_glance_api != 1
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Glance API is not available at {{$labels.url}} for more than 5 minutes'
|
|
summary: 'Glance API is not available at {{$labels.url}}'
|
|
- alert: os_nova_api_availability
|
|
expr: openstack_check_nova_api != 1
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Nova API is not available at {{$labels.url}} for more than 5 minutes'
|
|
summary: 'Nova API is not available at {{$labels.url}}'
|
|
- alert: os_keystone_api_availability
|
|
expr: openstack_check_keystone_api != 1
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Keystone API is not available at {{$labels.url}} for more than 5 minutes'
|
|
summary: 'Keystone API is not available at {{$labels.url}}'
|
|
- alert: os_neutron_api_availability
|
|
expr: openstack_check_neutron_api != 1
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Neutron API is not available at {{$labels.url}} for more than 5 minutes'
|
|
summary: 'Neutron API is not available at {{$labels.url}}'
|
|
- alert: os_neutron_metadata_agent_availability
|
|
expr: openstack_services_neutron_metadata_agent_down_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'One or more neutron metadata_agents are not available for more than 5 minutes'
|
|
summary: 'One or more neutron metadata_agents are not available'
|
|
- alert: os_neutron_openvswitch_agent_availability
|
|
expr: openstack_services_neutron_openvswitch_agent_down_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'One or more neutron openvswitch agents are not available for more than 5 minutes'
|
|
summary: 'One or more neutron openvswitch agents are not available'
|
|
- alert: os_neutron_dhcp_agent_availability
|
|
expr: openstack_services_neutron_dhcp_agent_down_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'One or more neutron dhcp agents are not available for more than 5 minutes'
|
|
summary: 'One or more neutron dhcp agents are not available'
|
|
- alert: os_neutron_l3_agent_availability
|
|
expr: openstack_services_neutron_l3_agent_down_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'One or more neutron L3 agents are not available for more than 5 minutes'
|
|
summary: 'One or more neutron L3 agents are not available'
|
|
- alert: os_swift_api_availability
|
|
expr: openstack_check_swift_api != 1
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Swift API is not available at {{$labels.url}} for more than 5 minutes'
|
|
summary: 'Swift API is not available at {{$labels.url}}'
|
|
- alert: os_cinder_api_availability
|
|
expr: openstack_check_cinder_api != 1
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Cinder API is not available at {{$labels.url}} for more than 5 minutes'
|
|
summary: 'Cinder API is not available at {{$labels.url}}'
|
|
- alert: os_cinder_scheduler_availability
|
|
expr: openstack_services_cinder_cinder_scheduler != 1
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Cinder scheduler is not available for more than 5 minutes'
|
|
summary: 'Cinder scheduler is not available'
|
|
- alert: os_heat_api_availability
|
|
expr: openstack_check_heat_api != 1
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Heat API is not available at {{$labels.url}} for more than 5 minutes'
|
|
summary: 'Heat API is not available at {{$labels.url}}'
|
|
- alert: os_nova_compute_disabled
|
|
expr: openstack_services_nova_compute_disabled_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'nova-compute is disabled on certain hosts for more than 5 minutes'
|
|
summary: 'Openstack compute service nova-compute is disabled on some hosts'
|
|
- alert: os_nova_conductor_disabled
|
|
expr: openstack_services_nova_conductor_disabled_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'nova-conductor is disabled on certain hosts for more than 5 minutes'
|
|
summary: 'Openstack compute service nova-conductor is disabled on some hosts'
|
|
- alert: os_nova_consoleauth_disabled
|
|
expr: openstack_services_nova_consoleauth_disabled_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'nova-consoleauth is disabled on certain hosts for more than 5 minutes'
|
|
summary: 'Openstack compute service nova-consoleauth is disabled on some hosts'
|
|
- alert: os_nova_scheduler_disabled
|
|
expr: openstack_services_nova_scheduler_disabled_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'nova-scheduler is disabled on certain hosts for more than 5 minutes'
|
|
summary: 'Openstack compute service nova-scheduler is disabled on some hosts'
|
|
- alert: os_nova_compute_down
|
|
expr: openstack_services_nova_compute_down_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'nova-compute is down on certain hosts for more than 5 minutes'
|
|
summary: 'Openstack compute service nova-compute is down on some hosts'
|
|
- alert: os_nova_conductor_down
|
|
expr: openstack_services_nova_conductor_down_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'nova-conductor is down on certain hosts for more than 5 minutes'
|
|
summary: 'Openstack compute service nova-conductor is down on some hosts'
|
|
- alert: os_nova_consoleauth_down
|
|
expr: openstack_services_nova_consoleauth_down_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'nova-consoleauth is down on certain hosts for more than 5 minutes'
|
|
summary: 'Openstack compute service nova-consoleauth is down on some hosts'
|
|
- alert: os_nova_scheduler_down
|
|
expr: openstack_services_nova_scheduler_down_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'nova-scheduler is down on certain hosts for more than 5 minutes'
|
|
summary: 'Openstack compute service nova-scheduler is down on some hosts'
|
|
- alert: os_vm_vcpu_usage_high
|
|
expr: openstack_total_used_vcpus * 100/(openstack_total_used_vcpus + openstack_total_free_vcpus) > 80
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Openstack VM vcpu usage is hight at {{$value}} percent'
|
|
summary: 'Openstack VM vcpu usage is high'
|
|
- alert: os_vm_ram_usage_high
|
|
expr: openstack_total_used_ram_MB * 100/(openstack_total_used_ram_MB + openstack_total_free_ram_MB) > 80
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Openstack VM RAM usage is hight at {{$value}} percent'
|
|
summary: 'Openstack VM RAM usage is high'
|
|
- alert: os_vm_disk_usage_high
|
|
expr: openstack_total_used_disk_GB * 100/ ( openstack_total_used_disk_GB + openstack_total_free_disk_GB ) > 80
|
|
for: 5m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'Openstack VM Disk usage is hight at {{$value}} percent'
|
|
summary: 'Openstack VM Disk usage is high'
|
|
- name: rabbitmq.rules
|
|
rules:
|
|
- alert: rabbitmq_network_pratitions_detected
|
|
expr: min(partitions) by(instance) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'RabbitMQ at {{ $labels.instance }} has {{ $value }} partitions'
|
|
summary: 'RabbitMQ Network partitions detected'
|
|
- alert: rabbitmq_down
|
|
expr: min(rabbitmq_up) by(instance) != 1
|
|
for: 10m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
description: 'RabbitMQ Server instance {{ $labels.instance }} is down'
|
|
summary: 'The RabbitMQ Server instance at {{ $labels.instance }} has been down the last 10 mins'
|
|
- alert: rabbitmq_file_descriptor_usage_high
|
|
expr: fd_used * 100 /fd_total > 80
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'RabbitMQ Server instance {{ $labels.instance }} has high file descriptor usage of {{ $value }} percent.'
|
|
summary: 'RabbitMQ file descriptors usage is high for last 10 mins'
|
|
- alert: rabbitmq_node_disk_free_alarm
|
|
expr: node_disk_free_alarm > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'RabbitMQ Server instance {{ $labels.instance }} has low disk free space available.'
|
|
summary: 'RabbitMQ disk space usage is high'
|
|
- alert: rabbitmq_node_memory_alarm
|
|
expr: node_mem_alarm > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'RabbitMQ Server instance {{ $labels.instance }} has low free memory.'
|
|
summary: 'RabbitMQ memory usage is high'
|
|
- alert: rabbitmq_less_than_3_nodes
|
|
expr: running < 3
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'RabbitMQ Server has less than 3 nodes running.'
|
|
summary: 'RabbitMQ server is at risk of loosing data'
|
|
- alert: rabbitmq_queue_messages_returned_high
|
|
expr: queue_messages_returned_total/queue_messages_published_total * 100 > 50
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'RabbitMQ Server is returing more than 50 percent of messages received.'
|
|
summary: 'RabbitMQ server is returning more than 50 percent of messages received.'
|
|
- alert: rabbitmq_consumers_low_utilization
|
|
expr: queue_consumer_utilisation < .4
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'RabbitMQ consumers message consumption speed is low'
|
|
summary: 'RabbitMQ consumers message consumption speed is low'
|
|
- alert: rabbitmq_high_message_load
|
|
expr: queue_messages_total > 17000 or increase(queue_messages_total[5m]) > 4000
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: 'RabbitMQ has high message load. Total Queue depth > 17000 or growth more than 4000 messages.'
|
|
summary: 'RabbitMQ has high message load'
|
|
...
|