diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 97e03a93de..3bc8a97692 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -387,6 +387,9 @@ prometheus_alertmanager_cluster_port: "9094" prometheus_openstack_exporter_port: "9198" prometheus_elasticsearch_exporter_port: "9108" +# Prometheus blackbox-exporter ports +prometheus_blackbox_exporter_port: "9115" + qdrouterd_port: "31459" qinling_api_port: "7070" @@ -1059,7 +1062,7 @@ enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}" enable_prometheus_ceph_mgr_exporter: "{{ enable_ceph | bool and enable_prometheus | bool }}" enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}" enable_prometheus_elasticsearch_exporter: "{{ enable_prometheus | bool and enable_elasticsearch | bool }}" - +enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}" prometheus_alertmanager_user: "admin" prometheus_openstack_exporter_interval: "60s" prometheus_elasticsearch_exporter_interval: "60s" diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 78aa966021..5f00f0024e 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -757,6 +757,9 @@ monitoring [prometheus-elasticsearch-exporter:children] elasticsearch +[prometheus-blackbox-exporter:children] +monitoring + [masakari-api:children] control diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 28ec388f4e..d6ef71bd28 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -776,6 +776,9 @@ monitoring [prometheus-elasticsearch-exporter:children] elasticsearch +[prometheus-blackbox-exporter:children] +monitoring + [masakari-api:children] control diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml index fd4bb51e35..ce11ce0cbe 100644 --- a/ansible/roles/prometheus/defaults/main.yml +++ b/ansible/roles/prometheus/defaults/main.yml @@ -87,12 +87,32 @@ prometheus_services: image: "{{ prometheus_elasticsearch_exporter_image_full }}" volumes: "{{ prometheus_elasticsearch_exporter_default_volumes + prometheus_elasticsearch_exporter_extra_volumes }}" dimensions: "{{ prometheus_elasticsearch_exporter_dimensions }}" + prometheus-blackbox-exporter: + container_name: "prometheus_blackbox_exporter" + group: "prometheus-blackbox-exporter" + enabled: "{{ enable_prometheus_blackbox_exporter | bool }}" + image: "{{ prometheus_blackbox_exporter_image_full }}" + volumes: "{{ prometheus_blackbox_exporter_default_volumes + prometheus_blackbox_exporter_extra_volumes }}" + dimensions: "{{ prometheus_blackbox_exporter_dimensions }}" #################### # Database #################### prometheus_mysql_exporter_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}prometheus{% endif %}" +#################### +# Blackbox +#################### + +# A list of endpoints to monitor. Each target is in the format: +# 'service_name:blackbox_exporter_module:endpoint' for example: +# +# prometheus_blackbox_exporter_targets: +# - 'glance:os_endpoint:http://{{ kolla_external_vip_address }}:{{ glance_api_port}}' +# +# For a list of modules see the alertmanager config. +prometheus_blackbox_exporter_endpoints: [] + #################### # Docker #################### @@ -136,6 +156,10 @@ prometheus_elasticsearch_exporter_image: "{{ docker_registry ~ '/' if docker_reg prometheus_elasticsearch_exporter_tag: "{{ prometheus_tag }}" prometheus_elasticsearch_exporter_image_full: "{{ prometheus_elasticsearch_exporter_image }}:{{ prometheus_elasticsearch_exporter_tag }}" +prometheus_blackbox_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-blackbox-exporter" +prometheus_blackbox_exporter_tag: "{{ prometheus_tag }}" +prometheus_blackbox_exporter_image_full: "{{ prometheus_blackbox_exporter_image }}:{{ prometheus_blackbox_exporter_tag }}" + prometheus_server_dimensions: "{{ default_container_dimensions }}" prometheus_haproxy_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_mysqld_exporter_dimensions: "{{ default_container_dimensions }}" @@ -145,6 +169,7 @@ prometheus_cadvisor_dimensions: "{{ default_container_dimensions }}" prometheus_alertmanager_dimensions: "{{ default_container_dimensions }}" prometheus_openstack_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_elasticsearch_exporter_dimensions: "{{ default_container_dimensions }}" +prometheus_blackbox_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_server_default_volumes: - "{{ node_config_directory }}/prometheus-server/:{{ container_config_directory }}/:ro" @@ -193,6 +218,10 @@ prometheus_elasticsearch_exporter_default_volumes: - "{{ node_config_directory }}/prometheus-elasticsearch-exporter/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" +prometheus_blackbox_exporter_default_volumes: + - "{{ node_config_directory }}/prometheus-blackbox-exporter/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla/" prometheus_extra_volumes: "{{ default_extra_volumes }}" prometheus_server_extra_volumes: "{{ prometheus_extra_volumes }}" @@ -204,3 +233,4 @@ prometheus_cadvisor_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_alertmanager_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_openstack_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_elasticsearch_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" +prometheus_blackbox_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" diff --git a/ansible/roles/prometheus/handlers/main.yml b/ansible/roles/prometheus/handlers/main.yml index db155b4bb2..42820e6365 100644 --- a/ansible/roles/prometheus/handlers/main.yml +++ b/ansible/roles/prometheus/handlers/main.yml @@ -134,3 +134,18 @@ dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" + +- name: Restart prometheus-blackbox-exporter container + vars: + service_name: "prometheus-blackbox-exporter" + service: "{{ prometheus_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" diff --git a/ansible/roles/prometheus/tasks/config.yml b/ansible/roles/prometheus/tasks/config.yml index 3e297418a1..b2c305f294 100644 --- a/ansible/roles/prometheus/tasks/config.yml +++ b/ansible/roles/prometheus/tasks/config.yml @@ -123,6 +123,24 @@ notify: - Restart prometheus-openstack-exporter container +- name: Copying config file for blackbox exporter + become: true + vars: + service: "{{ prometheus_services['prometheus-blackbox-exporter'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/prometheus-blackbox-exporter/prometheus-blackbox-exporter.yml" + mode: "0660" + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + with_first_found: + - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus-blackbox-exporter.yml" + - "{{ node_custom_config }}/prometheus/prometheus-blackbox-exporter.yml" + - "{{ role_path }}/templates/prometheus-blackbox-exporter.yml.j2" + notify: + - Restart prometheus-blackbox-exporter container + - name: Check prometheus containers become: true kolla_docker: diff --git a/ansible/roles/prometheus/tasks/precheck.yml b/ansible/roles/prometheus/tasks/precheck.yml index 058278cf42..1406f013cd 100644 --- a/ansible/roles/prometheus/tasks/precheck.yml +++ b/ansible/roles/prometheus/tasks/precheck.yml @@ -12,6 +12,7 @@ - prometheus_alertmanager - prometheus_openstack_exporter - prometheus_elasticsearch_exporter + - prometheus_blackbox_exporter register: container_facts - name: Checking free port for Prometheus server @@ -127,3 +128,17 @@ - enable_prometheus_elasticsearch_exporter | bool with_items: - "{{ prometheus_elasticsearch_exporter_port }}" + +- name: Checking free ports for Prometheus blackbox-exporter + wait_for: + host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" + port: "{{ item }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['prometheus_blackbox_exporter'] is not defined + - inventory_hostname in groups['prometheus-blackbox-exporter'] + - enable_prometheus_blackbox_exporter | bool + with_items: + - "{{ prometheus_blackbox_exporter_port }}" diff --git a/ansible/roles/prometheus/templates/prometheus-blackbox-exporter.json.j2 b/ansible/roles/prometheus/templates/prometheus-blackbox-exporter.json.j2 new file mode 100644 index 0000000000..0ea688fac7 --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-blackbox-exporter.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "/opt/blackbox_exporter/blackbox_exporter --config.file=/etc/prometheus/blackbox.yml --web.listen-address={{ api_interface_address }}:{{ prometheus_blackbox_exporter_port }} --log.level=info", + "config_files": [ + { + "source": "{{ container_config_directory }}/prometheus-blackbox-exporter.yml", + "dest": "/etc/prometheus/blackbox.yml", + "owner": "prometheus", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/prometheus", + "owner": "prometheus:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/prometheus/templates/prometheus-blackbox-exporter.yml.j2 b/ansible/roles/prometheus/templates/prometheus-blackbox-exporter.yml.j2 new file mode 100644 index 0000000000..5b39336ab1 --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-blackbox-exporter.yml.j2 @@ -0,0 +1,24 @@ +modules: + http_2xx: + prober: http + os_endpoint: + prober: http + http: + valid_status_codes: [200,300] + headers: + Content-Type: application/json + fail_if_body_not_matches_regexp: + - "versions" + http_post_2xx: + prober: http + http: + method: POST + tcp_connect: + prober: tcp + ssh_banner: + prober: tcp + tcp: + query_response: + - expect: "^SSH-2.0-" + icmp: + prober: icmp diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index 73cd773ae9..e72ab31a13 100644 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -96,6 +96,36 @@ scrape_configs: {% endfor %} {% endif %} +{% if enable_prometheus_blackbox_exporter | bool and prometheus_blackbox_exporter_endpoints | length > 0 | bool %} + - job_name: blackbox_exporter + metrics_path: /probe + honor_labels: true + static_configs: + - targets: +{% for target in prometheus_blackbox_exporter_endpoints %} + - '{{ target }}' +{% endfor %} + relabel_configs: + - source_labels: [__address__] + regex: (\w+):(\w+):(.+) + target_label: service + replacement: ${1} + - source_labels: [__address__] + regex: (\w+):(\w+):(.+) + target_label: __param_module + replacement: ${2} + - source_labels: [__param_module] + target_label: module + - source_labels: [__address__] + regex: (\w+):(\w+):(.+) + target_label: __param_target + replacement: ${3} + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: '{{ api_interface_address }}:{{ prometheus_blackbox_exporter_port }}' +{% endif %} + {% if enable_prometheus_alertmanager | bool %} alerting: alertmanagers: diff --git a/ansible/site.yml b/ansible/site.yml index 362890eabe..5f414c6b60 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -490,6 +490,7 @@ - prometheus-alertmanager - prometheus-openstack-exporter - prometheus-elasticsearch-exporter + - prometheus-blackbox-exporter - '&enable_prometheus_True' serial: '{{ kolla_serial|default("0") }}' roles: diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index c9c0a513cc..116b1e75cb 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -582,6 +582,7 @@ #enable_prometheus_ceph_mgr_exporter: "{{ enable_prometheus | bool and enable_ceph | bool }}" #enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}" #enable_prometheus_elasticsearch_exporter: "{{ enable_prometheus | bool and enable_elasticsearch | bool }}" +#enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}" ######### # Freezer diff --git a/releasenotes/notes/prometheus-blackbox-exporter-ee096376656c3751.yaml b/releasenotes/notes/prometheus-blackbox-exporter-ee096376656c3751.yaml new file mode 100644 index 0000000000..7a8bed2b4b --- /dev/null +++ b/releasenotes/notes/prometheus-blackbox-exporter-ee096376656c3751.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Prometheus blackbox exporter + + An example blackbox-exporter module has been added (disabled by default) + called os_endpoint. This allows for the probing of endpoints over HTTP + and HTTPS. This can be used to monitor that OpenStack endpoints return a status + code of either 200 or 300, and the word 'versions' in the payload. diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index 8b5633b28a..f6a4e63f33 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -696,3 +696,6 @@ monitoring [prometheus-elasticsearch-exporter:children] elasticsearch + +[prometheus-blackbox-exporter:children] +monitoring