Add auto discover for ceph prometheus metrics

Add to elk_metrics_6x the capability to auto discover the ceph mgr
prometheus plugin metric port.  Pull ceph metrics from prometheus when
the plugin is enabled.

Change-Id: I530a99f42e396ba7b2cd2c1b3d587f528ef84242
This commit is contained in:
Michael Vollman 2018-11-09 10:16:34 -05:00
parent bc180319ba
commit 3f70fc76d9
3 changed files with 48 additions and 5 deletions

View File

@ -18,3 +18,7 @@ elastic_metricbeat_rabbitmq_monitoring_hosts: '"localhost:15672"'
elastic_metricbeat_haproxy_monitoring_hosts: '"unix:///var/run/haproxy.stat"'
metricbeat_service_state: restarted
# Inventory group to configure metricbeat ceph monitoring
# via either ceph-restapi or mgr prometheus module
ceph_metricbeat_group: cinder_volume

View File

@ -86,7 +86,7 @@
{% endif %}
{{ ceph_stats }}
- name: Check for Ceph metric port
- name: Check for Ceph restapi metric port
wait_for:
port: "{{ item | regex_replace('^.+:', '') | int }}"
delay: 2
@ -103,6 +103,23 @@
when: not (item.failed | default(true))
with_items: "{{ ceph_restapi_port_check.results }}"
- name: Check for Ceph prometheus metric port
wait_for:
port: 9283
delay: 2
timeout: 5
host: "{{ item | regex_replace(':\\d+$', '') }}"
with_items: "{{ ceph_stats_hosts }}"
when: ceph.stat.exists
register: ceph_prometheus_port_check
ignore_errors: yes
- name: Set ceph_prometheus_listening
set_fact:
ceph_prometheus_listening: true
when: not (item.failed | default(true))
with_items: "{{ ceph_prometheus_port_check.results }}"
- name: Check for etcd
stat:
path: /etc/etcd
@ -167,10 +184,10 @@
# Only enable ceph if something is listening on the ceph-rest-api port
# enable ceph on: cinder volume hosts when we have a list of ceph mons
# otherwise: all hosts which have /etc/ceph
ceph_enabled: |-
ceph_restapi_enabled: |-
{% set ceph_detect = false %}
{% if ceph_restapi_listening is defined %}
{% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume']) %}
{% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups[ceph_metricbeat_group]) %}
{% set ceph_detect = true %}
{% else %}
{% set ceph_detect = ceph.stat.exists | bool %}
@ -178,6 +195,18 @@
{% endif %}
{{ ceph_detect }}
ceph_prometheus_enabled: |-
{% set ceph_detect = false %}
{% if ceph_prometheus_listening is defined %}
{% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups[ceph_metricbeat_group]) %}
{% set ceph_detect = true %}
{% else %}
{% set ceph_detect = ceph.stat.exists | bool %}
{% endif %}
{% endif %}
{{ ceph_detect }}
docker_enabled: "{{ docker.stat.exists | bool }}"
etcd_enabled: "{{ etcd.stat.exists | bool }}"
haproxy_enabled: "{{ haproxy.stat.exists | bool }}"

View File

@ -152,7 +152,7 @@ metricbeat.modules:
{% endif %}
#
#-------------------------------- Ceph Module --------------------------------
{% if ceph_enabled | default(false) | bool %}
{% if ceph_restapi_enabled | default(false) | bool %}
- module: ceph
metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"]
enabled: true
@ -458,7 +458,7 @@ metricbeat.modules:
{% if (prometheus_enabled | default(false) | bool) and (prometheus_config is defined) %}
{% for prometheus in prometheus_config %}
- module: prometheus
metricsets: [{% for mset in prometheus.metricsets|default(["collector"]) %}"{{ mset }}"{% if not loop.last %},{% endif %}{% endfor %}]
metricsets: [{% for mset in prometheus.metricsets|default("collector") %}"{{ mset }}"{% if not loop.last %},{% endif %}{% endfor %}]
enabled: {{ prometheus.enabled | default('true') }}
period: {{ prometheus.period | default("10s") }}
hosts: [{% for phost in prometheus.hosts %}"{{ phost }}"{% if not loop.last %},{% endif %}{% endfor %}]
@ -467,6 +467,16 @@ metricbeat.modules:
#
{% endfor %}
{% endif %}
{% if (ceph_prometheus_enabled | default(false) | bool) %}
- module: prometheus
metricsets: ["collector"]
enabled: 'true'
period: "10s"
hosts: [{% for phost in ceph_stats_hosts %}"{{ phost | regex_replace(':\\d+$', '') }}"{% if not loop.last %},{% endif %}{% endfor %}]
metrics_path: "/metrics"
namespace: ceph
#
{% endif %}
##------------------------------ RabbitMQ Module ------------------------------
{% if (rabbitmq_enabled | default(false) | bool) and (rabbitmq_monitoring_password is defined) %}
- module: rabbitmq