Add auto discover for ceph prometheus metrics

Add to elk_metrics_6x the capability to auto discover the ceph mgr
prometheus plugin metric port.  Pull ceph metrics from prometheus when
the plugin is enabled.

Change-Id: I530a99f42e396ba7b2cd2c1b3d587f528ef84242

elk_metrics_6x/roles/elastic_metricbeat/defaults/main.yml

@@ -18,3 +18,7 @@ elastic_metricbeat_rabbitmq_monitoring_hosts: '"localhost:15672"'
 elastic_metricbeat_haproxy_monitoring_hosts: '"unix:///var/run/haproxy.stat"'
 
 metricbeat_service_state: restarted
+
+# Inventory group to configure metricbeat ceph monitoring
+# via either ceph-restapi or mgr prometheus module
+ceph_metricbeat_group: cinder_volume

elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml

@@ -86,7 +86,7 @@
       {% endif %}
       {{ ceph_stats }}
 
-- name: Check for Ceph metric port
+- name: Check for Ceph restapi metric port
   wait_for:
     port: "{{ item | regex_replace('^.+:', '') | int }}"
     delay: 2
@@ -103,6 +103,23 @@
   when: not (item.failed | default(true))
   with_items: "{{ ceph_restapi_port_check.results }}"
 
+- name: Check for Ceph prometheus metric port
+  wait_for:
+    port: 9283
+    delay: 2
+    timeout: 5
+    host: "{{ item | regex_replace(':\\d+$', '') }}"
+  with_items: "{{ ceph_stats_hosts }}"
+  when: ceph.stat.exists
+  register: ceph_prometheus_port_check
+  ignore_errors: yes
+
+- name: Set ceph_prometheus_listening
+  set_fact:
+    ceph_prometheus_listening: true
+  when: not (item.failed | default(true))
+  with_items: "{{ ceph_prometheus_port_check.results }}"
+
 - name: Check for etcd
   stat:
     path: /etc/etcd
@@ -167,10 +184,10 @@
     # Only enable ceph if something is listening on the ceph-rest-api port
     # enable ceph on:  cinder volume hosts when we have a list of ceph mons
     #      otherwise:  all hosts which have /etc/ceph
-    ceph_enabled: |-
+    ceph_restapi_enabled: |-
       {% set ceph_detect = false %}
       {% if ceph_restapi_listening is defined %}
-      {%   if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume'])  %}
+      {%   if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups[ceph_metricbeat_group])  %}
       {%     set ceph_detect = true %}
       {%   else %}
       {%     set ceph_detect = ceph.stat.exists | bool %}
@@ -178,6 +195,18 @@
       {% endif %}
       {{ ceph_detect }}
 
+    ceph_prometheus_enabled: |-
+      {% set ceph_detect = false %}
+      {% if ceph_prometheus_listening is defined %}
+      {%   if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups[ceph_metricbeat_group])  %}
+      {%     set ceph_detect = true %}
+      {%   else %}
+      {%     set ceph_detect = ceph.stat.exists | bool %}
+      {%   endif %}
+      {% endif %}
+      {{ ceph_detect }}
+
+
     docker_enabled: "{{ docker.stat.exists | bool }}"
     etcd_enabled: "{{ etcd.stat.exists | bool }}"
     haproxy_enabled: "{{ haproxy.stat.exists | bool }}"

elk_metrics_6x/roles/elastic_metricbeat/templates/metricbeat.yml.j2

@@ -152,7 +152,7 @@ metricbeat.modules:
 {% endif %}
 #
 #-------------------------------- Ceph Module --------------------------------
-{% if ceph_enabled | default(false) | bool %}
+{% if ceph_restapi_enabled | default(false) | bool %}
 - module: ceph
   metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"]
   enabled: true
@@ -458,7 +458,7 @@ metricbeat.modules:
 {% if (prometheus_enabled | default(false) | bool) and (prometheus_config is defined) %}
 {% for prometheus in prometheus_config %}
 - module: prometheus
-  metricsets: [{% for mset in prometheus.metricsets|default(["collector"]) %}"{{ mset }}"{% if not loop.last %},{% endif %}{% endfor %}]
+  metricsets: [{% for mset in prometheus.metricsets|default("collector") %}"{{ mset }}"{% if not loop.last %},{% endif %}{% endfor %}]
   enabled: {{ prometheus.enabled | default('true') }}
   period: {{ prometheus.period | default("10s") }}
   hosts: [{% for phost in prometheus.hosts %}"{{ phost }}"{% if not loop.last %},{% endif %}{% endfor %}]
@@ -467,6 +467,16 @@ metricbeat.modules:
 #
 {% endfor %}
 {% endif %}
+{% if (ceph_prometheus_enabled | default(false) | bool) %}
+- module: prometheus
+  metricsets: ["collector"]
+  enabled: 'true'
+  period: "10s"
+  hosts: [{% for phost in ceph_stats_hosts %}"{{ phost | regex_replace(':\\d+$', '') }}"{% if not loop.last %},{% endif %}{% endfor %}]
+  metrics_path: "/metrics"
+  namespace: ceph
+#
+{% endif %}
 ##------------------------------ RabbitMQ Module ------------------------------
 {% if (rabbitmq_enabled | default(false) | bool) and (rabbitmq_monitoring_password is defined) %}
 - module: rabbitmq