---
conf:
  prometheus:
    rules:
      openstack:
        groups:
        - name: mariadb.rules
          rules:
          - alert: prom_exporter_mariadb_openstack_unavailable
            expr: avg_over_time(up{job="mysql-exporter",kubernetes_namespace="openstack"}[5m]) == 0
            for: 5m
            labels:
              severity: warning
            annotations:
              description: MariaDB exporter in  {{ $labels.kubernetes_namespace }} is not collecting metrics or is not available for past 10 minutes
              title: MariaDB exporter is not collecting metrics or is not available
          - alert: prom_exporter_mariadb_osh_infra_unavailable
            expr: avg_over_time(up{job="mysql-exporter",kubernetes_namespace="osh-infra"}[5m]) == 0
            for: 5m
            labels:
              severity: warning
            annotations:
              description: MariaDB exporter in  {{ $labels.kubernetes_namespace }} is not collecting metrics or is not available for past 10 minutes
              title: MariaDB exporter is not collecting metrics or is not available
          - alert: mariadb_table_lock_wait_high
            expr: 100 * mysql_global_status_table_locks_waited/(mysql_global_status_table_locks_waited + mysql_global_status_table_locks_immediate) > 30
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'Mariadb has high table lock waits of {{ $value }} percentage'
              summary: 'Mariadb table lock waits are high'
          - alert: mariadb_node_not_ready
            expr: mysql_global_status_wsrep_ready != 1
            for: 10m
            labels:
              severity: warning
            annotations:
              description: '{{$labels.job}} on {{$labels.instance}} is not ready.'
              summary: 'Galera cluster node not ready'
          - alert: mariadb_galera_node_out_of_sync
            expr: mysql_global_status_wsrep_local_state != 4 AND mysql_global_variables_wsrep_desync == 0
            for: 10m
            labels:
              severity: warning
            annotations:
              description: '{{$labels.job}} on {{$labels.instance}} is not in sync ({{$value}} != 4)'
              summary: 'Galera cluster node out of sync'
          - alert: mariadb_innodb_replication_fallen_behind
            expr: (mysql_global_variables_innodb_replication_delay > 30) AND on (instance) (predict_linear(mysql_global_variables_innodb_replication_delay[5m], 60*2) > 0)
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'The mysql innodb replication has fallen behind and is not recovering'
              summary: 'MySQL innodb replication is lagging'
        - name: openstack.rules
          rules:
          - alert: prom_exporter_openstack_unavailable
            expr: avg_over_time(up{job="openstack-metrics"}[5m]) == 0
            for: 5m
            labels:
              severity: warning
            annotations:
              description: Openstack exporter is not collecting metrics or is not available for past 10 minutes
              title: Openstack exporter is not collecting metrics or is not available
          - alert: os_glance_api_availability
            expr: openstack_check_glance_api != 1
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Glance API is not available at {{$labels.url}} for more than 5 minutes'
              summary: 'Glance API is not available at {{$labels.url}}'
          - alert: os_nova_api_availability
            expr: openstack_check_nova_api != 1
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Nova API is not available at {{$labels.url}} for more than 5 minutes'
              summary: 'Nova API is not available at {{$labels.url}}'
          - alert: os_keystone_api_availability
            expr: openstack_check_keystone_api != 1
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Keystone API is not available at {{$labels.url}} for more than 5 minutes'
              summary: 'Keystone API is not available at {{$labels.url}}'
          - alert: os_neutron_api_availability
            expr: openstack_check_neutron_api != 1
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Neutron API is not available at {{$labels.url}} for more than 5 minutes'
              summary: 'Neutron API is not available at {{$labels.url}}'
          - alert: os_neutron_metadata_agent_availability
            expr: openstack_services_neutron_metadata_agent_down_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'One or more neutron metadata_agents are not available for more than 5 minutes'
              summary: 'One or more neutron metadata_agents are not available'
          - alert: os_neutron_openvswitch_agent_availability
            expr: openstack_services_neutron_openvswitch_agent_down_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'One or more neutron openvswitch agents are not available for more than 5 minutes'
              summary: 'One or more neutron openvswitch agents are not available'
          - alert: os_neutron_dhcp_agent_availability
            expr: openstack_services_neutron_dhcp_agent_down_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'One or more neutron dhcp agents are not available for more than 5 minutes'
              summary: 'One or more neutron dhcp agents are not available'
          - alert: os_neutron_l3_agent_availability
            expr: openstack_services_neutron_l3_agent_down_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'One or more neutron L3 agents are not available for more than 5 minutes'
              summary: 'One or more neutron L3 agents are not available'
          - alert: os_swift_api_availability
            expr: openstack_check_swift_api != 1
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Swift API is not available at {{$labels.url}} for more than 5 minutes'
              summary: 'Swift API is not available at {{$labels.url}}'
          - alert: os_cinder_api_availability
            expr: openstack_check_cinder_api != 1
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Cinder API is not available at {{$labels.url}} for more than 5 minutes'
              summary: 'Cinder API is not available at {{$labels.url}}'
          - alert: os_cinder_scheduler_availability
            expr: openstack_services_cinder_cinder_scheduler != 1
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Cinder scheduler is not available for more than 5 minutes'
              summary: 'Cinder scheduler is not available'
          - alert: os_heat_api_availability
            expr: openstack_check_heat_api != 1
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Heat API is not available at {{$labels.url}} for more than 5 minutes'
              summary: 'Heat API is not available at {{$labels.url}}'
          - alert: os_nova_compute_disabled
            expr: openstack_services_nova_compute_disabled_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'nova-compute is disabled on certain hosts for more than 5 minutes'
              summary: 'Openstack compute service nova-compute is disabled on some hosts'
          - alert: os_nova_conductor_disabled
            expr: openstack_services_nova_conductor_disabled_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'nova-conductor is disabled on certain hosts for more than 5 minutes'
              summary: 'Openstack compute service nova-conductor is disabled on some hosts'
          - alert: os_nova_consoleauth_disabled
            expr: openstack_services_nova_consoleauth_disabled_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'nova-consoleauth is disabled on certain hosts for more than 5 minutes'
              summary: 'Openstack compute service nova-consoleauth is disabled on some hosts'
          - alert: os_nova_scheduler_disabled
            expr: openstack_services_nova_scheduler_disabled_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'nova-scheduler is disabled on certain hosts for more than 5 minutes'
              summary: 'Openstack compute service nova-scheduler is disabled on some hosts'
          - alert: os_nova_compute_down
            expr: openstack_services_nova_compute_down_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'nova-compute is down on certain hosts for more than 5 minutes'
              summary: 'Openstack compute service nova-compute is down on some hosts'
          - alert: os_nova_conductor_down
            expr: openstack_services_nova_conductor_down_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'nova-conductor is down on certain hosts for more than 5 minutes'
              summary: 'Openstack compute service nova-conductor is down on some hosts'
          - alert: os_nova_consoleauth_down
            expr: openstack_services_nova_consoleauth_down_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'nova-consoleauth is down on certain hosts for more than 5 minutes'
              summary: 'Openstack compute service nova-consoleauth is down on some hosts'
          - alert: os_nova_scheduler_down
            expr: openstack_services_nova_scheduler_down_total > 0
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'nova-scheduler is down on certain hosts for more than 5 minutes'
              summary: 'Openstack compute service nova-scheduler is down on some hosts'
          - alert: os_vm_vcpu_usage_high
            expr: openstack_total_used_vcpus * 100/(openstack_total_used_vcpus + openstack_total_free_vcpus) > 80
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Openstack VM vcpu usage is hight at {{$value}} percent'
              summary: 'Openstack VM vcpu usage is high'
          - alert: os_vm_ram_usage_high
            expr: openstack_total_used_ram_MB * 100/(openstack_total_used_ram_MB + openstack_total_free_ram_MB) > 80
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Openstack VM RAM usage is hight at {{$value}} percent'
              summary: 'Openstack VM RAM usage is high'
          - alert: os_vm_disk_usage_high
            expr: openstack_total_used_disk_GB * 100/ ( openstack_total_used_disk_GB + openstack_total_free_disk_GB ) > 80
            for: 5m
            labels:
              severity: page
            annotations:
              description: 'Openstack VM Disk usage is hight at {{$value}} percent'
              summary: 'Openstack VM Disk usage is high'
        - name: rabbitmq.rules
          rules:
          - alert: rabbitmq_network_pratitions_detected
            expr: min(partitions) by(instance) > 0
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'RabbitMQ at {{ $labels.instance }} has {{ $value }} partitions'
              summary: 'RabbitMQ Network partitions detected'
          - alert: rabbitmq_down
            expr: min(rabbitmq_up) by(instance) != 1
            for: 10m
            labels:
              severity: page
            annotations:
              description: 'RabbitMQ Server instance {{ $labels.instance }} is down'
              summary: 'The RabbitMQ Server instance at {{ $labels.instance }} has been down the last 10 mins'
          - alert: rabbitmq_file_descriptor_usage_high
            expr: fd_used * 100 /fd_total > 80
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'RabbitMQ Server instance {{ $labels.instance }} has high file descriptor usage of {{ $value }} percent.'
              summary: 'RabbitMQ file descriptors usage is high for last 10 mins'
          - alert: rabbitmq_node_disk_free_alarm
            expr: node_disk_free_alarm > 0
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'RabbitMQ Server instance {{ $labels.instance }} has low disk free space available.'
              summary: 'RabbitMQ disk space usage is high'
          - alert: rabbitmq_node_memory_alarm
            expr: node_mem_alarm > 0
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'RabbitMQ Server instance {{ $labels.instance }} has low free memory.'
              summary: 'RabbitMQ memory usage is high'
          - alert: rabbitmq_less_than_3_nodes
            expr: running < 3
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'RabbitMQ Server has less than 3 nodes running.'
              summary: 'RabbitMQ server is at risk of loosing data'
          - alert: rabbitmq_queue_messages_returned_high
            expr: queue_messages_returned_total/queue_messages_published_total * 100 > 50
            for: 5m
            labels:
              severity: warning
            annotations:
              description: 'RabbitMQ Server is returing more than 50 percent of messages received.'
              summary: 'RabbitMQ server is returning more than 50 percent of messages received.'
          - alert: rabbitmq_consumers_low_utilization
            expr: queue_consumer_utilisation < .4
            for: 5m
            labels:
              severity: warning
            annotations:
              description: 'RabbitMQ consumers message consumption speed is low'
              summary: 'RabbitMQ consumers message consumption speed is low'
          - alert: rabbitmq_high_message_load
            expr: queue_messages_total > 17000 or increase(queue_messages_total[5m]) > 4000
            for: 5m
            labels:
              severity: warning
            annotations:
              description: 'RabbitMQ has high message load. Total Queue depth > 17000 or growth more than 4000 messages.'
              summary: 'RabbitMQ has high message load'
...