From 4d1f37359d51687e8a96e706664ef3c309d633b2 Mon Sep 17 00:00:00 2001 From: Mathias Ewald Date: Mon, 17 Jul 2017 15:59:05 +0000 Subject: [PATCH] Add role to deploy prometheus This patch adds the ansible role to deploy the prometheus service which can be used to collect performance metrics accross the environment Partially-Implements: blueprint prometheus Change-Id: I908b9c9dad63ab5c9b80be1e3a80a4fc8191cb9e --- README.rst | 3 +- ansible/group_vars/all.yml | 13 +++ ansible/inventory/all-in-one | 17 ++++ ansible/inventory/multinode | 17 ++++ .../roles/haproxy/templates/haproxy.cfg.j2 | 14 +++- ansible/roles/prometheus/defaults/main.yml | 72 +++++++++++++++++ ansible/roles/prometheus/handlers/main.yml | 79 +++++++++++++++++++ ansible/roles/prometheus/meta/main.yml | 3 + ansible/roles/prometheus/tasks/bootstrap.yml | 19 +++++ ansible/roles/prometheus/tasks/config.yml | 74 +++++++++++++++++ ansible/roles/prometheus/tasks/deploy.yml | 7 ++ ansible/roles/prometheus/tasks/main.yml | 2 + ansible/roles/prometheus/tasks/precheck.yml | 56 +++++++++++++ ansible/roles/prometheus/tasks/pull.yml | 10 +++ .../roles/prometheus/tasks/reconfigure.yml | 2 + ansible/roles/prometheus/tasks/upgrade.yml | 5 ++ ansible/roles/prometheus/templates/my.cnf.j2 | 5 ++ .../prometheus-haproxy-exporter.json.j2 | 11 +++ .../prometheus-mysqld-exporter.json.j2 | 18 +++++ .../prometheus-node-exporter.json.j2 | 11 +++ .../templates/prometheus-server.json.j2 | 23 ++++++ .../prometheus/templates/prometheus.yml.j2 | 41 ++++++++++ ansible/site.yml | 13 +++ etc/kolla/globals.yml | 8 ++ etc/kolla/passwords.yml | 5 ++ .../notes/prometheus-dbb1aee8c88943c4.yaml | 6 ++ tests/templates/inventory.j2 | 17 ++++ 27 files changed, 549 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/prometheus/defaults/main.yml create mode 100644 ansible/roles/prometheus/handlers/main.yml create mode 100644 ansible/roles/prometheus/meta/main.yml create mode 100644 ansible/roles/prometheus/tasks/bootstrap.yml create mode 100644 ansible/roles/prometheus/tasks/config.yml create mode 100644 ansible/roles/prometheus/tasks/deploy.yml create mode 100644 ansible/roles/prometheus/tasks/main.yml create mode 100644 ansible/roles/prometheus/tasks/precheck.yml create mode 100644 ansible/roles/prometheus/tasks/pull.yml create mode 100644 ansible/roles/prometheus/tasks/reconfigure.yml create mode 100644 ansible/roles/prometheus/tasks/upgrade.yml create mode 100644 ansible/roles/prometheus/templates/my.cnf.j2 create mode 100644 ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2 create mode 100644 ansible/roles/prometheus/templates/prometheus-mysqld-exporter.json.j2 create mode 100644 ansible/roles/prometheus/templates/prometheus-node-exporter.json.j2 create mode 100644 ansible/roles/prometheus/templates/prometheus-server.json.j2 create mode 100644 ansible/roles/prometheus/templates/prometheus.yml.j2 create mode 100644 releasenotes/notes/prometheus-dbb1aee8c88943c4.yaml diff --git a/README.rst b/README.rst index 5e2121734a..94216c3ec5 100644 --- a/README.rst +++ b/README.rst @@ -89,7 +89,8 @@ Kolla-Ansible deploys containers for the following infrastructure components: - `Ceph `__ implementation for Cinder, Glance and Nova. - `Collectd `__, `Telegraf `__, - `InfluxDB `__, and + `InfluxDB `__, + `Prometheus `__, and `Grafana `__ for performance monitoring. - `Elasticsearch `__ and `Kibana `__ to search, analyze, diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 5ad218ab8d..46981bab52 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -262,6 +262,11 @@ panko_api_port: "8977" placement_api_port: "8780" +prometheus_port: "9091" +prometheus_node_exporter_port: "9100" +prometheus_mysqld_exporter_port: "9104" +prometheus_haproxy_exporter_port: "9101" + qdrouterd_port: "31459" rabbitmq_port: "5672" @@ -507,6 +512,7 @@ num_nova_fake_per_node: 5 # Monitoring options are specified here enable_collectd: "no" +enable_prometheus: "no" # Clean images options are specified here enable_destroy_images: "no" @@ -770,3 +776,10 @@ use_preconfigured_databases: "no" # whether to use a common, preconfigured user # for all component databases use_common_mariadb_user: "no" + +############ +# Prometheus +############ +enable_prometheus_haproxy_exporter: "{{ enable_haproxy | bool }}" +enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}" +enable_prometheus_node_exporter: "yes" diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 89ef1c8a21..1b28267032 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -159,6 +159,9 @@ control [influxdb:children] monitoring +[prometheus:children] +monitoring + [magnum:children] control @@ -641,3 +644,17 @@ blazar [blazar-manager:children] blazar + +# Prometheus +[prometheus-node-exporter:children] +monitoring +control +compute +network +storage + +[prometheus-mysqld-exporter:children] +mariadb + +[prometheus-haproxy-exporter:children] +haproxy diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index b19f84a02b..e66d3e137d 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -78,6 +78,9 @@ compute [influxdb:children] monitoring +[prometheus:children] +monitoring + [kafka:children] control @@ -660,3 +663,17 @@ blazar [blazar-manager:children] blazar + +# Prometheus +[prometheus-node-exporter:children] +monitoring +control +compute +network +storage + +[prometheus-mysqld-exporter:children] +mariadb + +[prometheus-haproxy-exporter:children] +haproxy diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2 index 1eefd9d0fa..67f0ea06a1 100644 --- a/ansible/roles/haproxy/templates/haproxy.cfg.j2 +++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2 @@ -7,7 +7,7 @@ global daemon log {{ syslog_server }}:{{ syslog_udp_port }} {{ syslog_haproxy_facility }} maxconn 4000 - stats socket /var/lib/kolla/haproxy/haproxy.sock + stats socket /var/lib/kolla/haproxy/haproxy.sock group kolla mode 660 {% if kolla_enable_tls_external | bool %} ssl-default-bind-ciphers DEFAULT:!MEDIUM:!3DES ssl-default-bind-options no-sslv3 no-tlsv10 @@ -1196,6 +1196,18 @@ listen blazar_api_external {% endif %} {% endif %} +{% if enable_prometheus | bool %} +listen prometheus_server + bind {{ kolla_internal_vip_address }}:{{ prometheus_port }} + http-request del-header X-Forwarded-Proto +{% for http_option in haproxy_listen_http_extra %} + {{ http_option }} +{% endfor %} +{% for host in groups['prometheus'] %} + server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ prometheus_port }} check inter 2000 rise 2 fall 5 +{% endfor %} +{% endif %} + # (NOTE): This defaults section deletes forwardfor as recommended by: # https://marc.info/?l=haproxy&m=141684110710132&w=1 diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml new file mode 100644 index 0000000000..198c54d729 --- /dev/null +++ b/ansible/roles/prometheus/defaults/main.yml @@ -0,0 +1,72 @@ +--- +project_name: "prometheus" + +prometheus_services: + prometheus-server: + container_name: prometheus_server + group: prometheus + enabled: true + image: "{{ prometheus_server_image_full }}" + volumes: + - "{{ node_config_directory }}/prometheus-server/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "prometheus:/var/lib/prometheus" + - "kolla_logs:/var/log/kolla/" + prometheus-node-exporter: + container_name: prometheus_node_exporter + group: prometheus-node-exporter + enabled: "{{ enable_prometheus_node_exporter | bool }}" + image: "{{ prometheus_node_exporter_image_full }}" + pid_mode: "host" + volumes: + - "{{ node_config_directory }}/prometheus-node-exporter/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla/" + - "/proc:/host/proc:ro" + - "/sys:/host/sys:ro" + prometheus-mysqld-exporter: + container_name: prometheus_mysqld_exporter + group: prometheus-mysqld-exporter + enabled: "{{ enable_prometheus_mysqld_exporter | bool }}" + image: "{{ prometheus_mysqld_exporter_image_full }}" + volumes: + - "{{ node_config_directory }}/prometheus-mysqld-exporter/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla/" + prometheus-haproxy-exporter: + container_name: prometheus_haproxy_exporter + group: prometheus-haproxy-exporter + enabled: "{{ enable_prometheus_haproxy_exporter | bool }}" + image: "{{ prometheus_haproxy_exporter_image_full }}" + volumes: + - "{{ node_config_directory }}/prometheus-haproxy-exporter/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla/" + - "haproxy_socket:/var/lib/kolla/haproxy" + +#################### +# Database +#################### +prometheus_mysql_exporter_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}prometheus{% endif %}" + +#################### +# Docker +#################### +prometheus_install_type: "{{ kolla_install_type }}" +prometheus_tag: "{{ openstack_release }}" + +prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-server" +prometheus_server_tag: "{{ prometheus_tag }}" +prometheus_server_image_full: "{{ prometheus_server_image }}:{{ prometheus_server_tag }}" + +prometheus_haproxy_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-haproxy-exporter" +prometheus_haproxy_exporter_tag: "{{ prometheus_tag }}" +prometheus_haproxy_exporter_image_full: "{{ prometheus_haproxy_exporter_image }}:{{ prometheus_haproxy_exporter_tag }}" + +prometheus_mysqld_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-mysqld-exporter" +prometheus_mysqld_exporter_tag: "{{ prometheus_tag }}" +prometheus_mysqld_exporter_image_full: "{{ prometheus_mysqld_exporter_image }}:{{ prometheus_mysqld_exporter_tag }}" + +prometheus_node_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-node-exporter" +prometheus_node_exporter_tag: "{{ prometheus_tag }}" +prometheus_node_exporter_image_full: "{{ prometheus_node_exporter_image }}:{{ prometheus_node_exporter_tag }}" diff --git a/ansible/roles/prometheus/handlers/main.yml b/ansible/roles/prometheus/handlers/main.yml new file mode 100644 index 0000000000..be79e58d2f --- /dev/null +++ b/ansible/roles/prometheus/handlers/main.yml @@ -0,0 +1,79 @@ +--- +- name: Restart prometheus-server container + vars: + service_name: "prometheus-server" + service: "{{ prometheus_services[service_name] }}" + config_json: "{{ prometheus_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + prometheus_container: "{{ check_prometheus_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or prometheus_confs.changed | bool + or prometheus_container.changed | bool + +- name: Restart prometheus-node-exporter container + vars: + service_name: "prometheus-node-exporter" + service: "{{ prometheus_services[service_name] }}" + config_json: "{{ prometheus_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + prometheus_container: "{{ check_prometheus_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + pid_mode: "{{ service.pid_mode | default(emit) }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or prometheus_container.changed | bool + +- name: Restart prometheus-mysqld-exporter container + vars: + service_name: "prometheus-mysqld-exporter" + service: "{{ prometheus_services[service_name] }}" + config_json: "{{ prometheus_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + prometheus_container: "{{ check_prometheus_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or prometheus_conf_mycnf.changed | bool + or prometheus_container.changed | bool + +- name: Restart prometheus-haproxy-exporter container + vars: + service_name: "prometheus-haproxy-exporter" + service: "{{ prometheus_services[service_name] }}" + config_json: "{{ prometheus_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + prometheus_container: "{{ check_prometheus_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or prometheus_container.changed | bool diff --git a/ansible/roles/prometheus/meta/main.yml b/ansible/roles/prometheus/meta/main.yml new file mode 100644 index 0000000000..6b4fff8fef --- /dev/null +++ b/ansible/roles/prometheus/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: common } diff --git a/ansible/roles/prometheus/tasks/bootstrap.yml b/ansible/roles/prometheus/tasks/bootstrap.yml new file mode 100644 index 0000000000..67480e1e78 --- /dev/null +++ b/ansible/roles/prometheus/tasks/bootstrap.yml @@ -0,0 +1,19 @@ +--- +- name: Creating prometheus database user and setting permissions + kolla_toolbox: + module_name: mysql_user + module_args: + login_host: "{{ database_address }}" + login_port: "{{ database_port }}" + login_user: "{{ database_user }}" + login_password: "{{ database_password }}" + name: "{{ prometheus_mysql_exporter_database_user }}" + password: "{{ prometheus_mysql_exporter_database_password }}" + host: "%" + priv: "*.*:PROCESS,REPLICATION CLIENT,SELECT" + append_privs: "yes" + run_once: True + delegate_to: "{{ groups['prometheus-mysqld-exporter'][0] }}" + when: + - enable_prometheus_mysqld_exporter | bool + - not use_preconfigured_databases | bool diff --git a/ansible/roles/prometheus/tasks/config.yml b/ansible/roles/prometheus/tasks/config.yml new file mode 100644 index 0000000000..3d8ce5e13f --- /dev/null +++ b/ansible/roles/prometheus/tasks/config.yml @@ -0,0 +1,74 @@ +--- +- name: Ensuring config directories exist + file: + path: "{{ node_config_directory }}/{{ item.key }}" + state: "directory" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + mode: "0770" + become: true + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ prometheus_services }}" + +- name: Copying over config.json files + template: + src: "{{ item.key }}.json.j2" + dest: "{{ node_config_directory }}/{{ item.key }}/config.json" + register: prometheus_config_jsons + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ prometheus_services }}" + notify: + - Restart {{ item.key }} container + +- name: Copying over prometheus config file + vars: + service: "{{ prometheus_services['prometheus-server']}}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/prometheus-server/prometheus.yml" + register: prometheus_confs + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + with_first_found: + - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus.yml" + - "{{ node_custom_config }}/prometheus/prometheus.yml" + - "{{ role_path }}/templates/prometheus.yml.j2" + notify: + - Restart prometheus-server container + +- name: Copying over my.cnf for mysqld_exporter + vars: + service: "{{ prometheus_services['prometheus-mysqld-exporter']}}" + merge_configs: + sources: + - "{{ node_custom_config }}/prometheus-mysqld-exporter/{{ inventory_hostname }}/my.cnf" + - "{{ node_custom_config }}/prometheus-mysqld-exporter/my.cnf" + - "{{ role_path }}/templates/my.cnf.j2" + dest: "{{ node_config_directory }}/prometheus-mysqld-exporter/my.cnf" + register: prometheus_conf_mycnf + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + notify: + - Restart prometheus-mysqld-exporter container + +- name: Check prometheus containers + kolla_docker: + action: "compare_container" + common_options: "{{ docker_common_options }}" + name: "{{ item.value.container_name }}" + image: "{{ item.value.image }}" + volumes: "{{ item.value.volumes }}" + register: check_prometheus_containers + when: + - action != "config" + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ prometheus_services }}" + notify: + - Restart {{ item.key }} container diff --git a/ansible/roles/prometheus/tasks/deploy.yml b/ansible/roles/prometheus/tasks/deploy.yml new file mode 100644 index 0000000000..9eca42dec5 --- /dev/null +++ b/ansible/roles/prometheus/tasks/deploy.yml @@ -0,0 +1,7 @@ +--- +- include: config.yml + +- include: bootstrap.yml + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/prometheus/tasks/main.yml b/ansible/roles/prometheus/tasks/main.yml new file mode 100644 index 0000000000..b017e8b4ad --- /dev/null +++ b/ansible/roles/prometheus/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ action }}.yml" diff --git a/ansible/roles/prometheus/tasks/precheck.yml b/ansible/roles/prometheus/tasks/precheck.yml new file mode 100644 index 0000000000..a43eaa5273 --- /dev/null +++ b/ansible/roles/prometheus/tasks/precheck.yml @@ -0,0 +1,56 @@ +--- +- name: Get container facts + kolla_container_facts: + name: + - prometheus + - prometheus_node_exporter + - prometheus_haproxy_exporter + - prometheus_mysqld_exporter + register: container_facts + +- name: Checking free port for Prometheus + wait_for: + host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" + port: "{{ prometheus_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['prometheus'] is not defined + - inventory_hostname in groups['prometheus'] + +- name: Checking free port for Prometheus node_exporter + wait_for: + host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" + port: "{{ prometheus_node_exporter_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['prometheus_node_exporter'] is not defined + - inventory_hostname in groups['prometheus-node-exporter'] + - enable_prometheus_node_exporter | bool + +- name: Checking free port for Prometheus mysqld_exporter + wait_for: + host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" + port: "{{ prometheus_mysqld_exporter_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['prometheus_mysqld_exporter'] is not defined + - inventory_hostname in groups['prometheus-mysqld-exporter'] + - enable_prometheus_mysqld_exporter | bool + +- name: Checking free port for Prometheus haproxy_exporter + wait_for: + host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" + port: "{{ prometheus_haproxy_exporter_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['prometheus_mysqld_exporter'] is not defined + - inventory_hostname in groups['prometheus-haproxy-exporter'] + - enable_prometheus_haproxy_exporter | bool diff --git a/ansible/roles/prometheus/tasks/pull.yml b/ansible/roles/prometheus/tasks/pull.yml new file mode 100644 index 0000000000..0a9553fdbc --- /dev/null +++ b/ansible/roles/prometheus/tasks/pull.yml @@ -0,0 +1,10 @@ +--- +- name: Pulling prometheus images + kolla_docker: + action: "pull_image" + common_options: "{{ docker_common_options }}" + image: "{{ item.value.image }}" + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ prometheus_services }}" diff --git a/ansible/roles/prometheus/tasks/reconfigure.yml b/ansible/roles/prometheus/tasks/reconfigure.yml new file mode 100644 index 0000000000..e078ef1318 --- /dev/null +++ b/ansible/roles/prometheus/tasks/reconfigure.yml @@ -0,0 +1,2 @@ +--- +- include: deploy.yml diff --git a/ansible/roles/prometheus/tasks/upgrade.yml b/ansible/roles/prometheus/tasks/upgrade.yml new file mode 100644 index 0000000000..dd26ecc34d --- /dev/null +++ b/ansible/roles/prometheus/tasks/upgrade.yml @@ -0,0 +1,5 @@ +--- +- include: config.yml + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/prometheus/templates/my.cnf.j2 b/ansible/roles/prometheus/templates/my.cnf.j2 new file mode 100644 index 0000000000..6061acccc3 --- /dev/null +++ b/ansible/roles/prometheus/templates/my.cnf.j2 @@ -0,0 +1,5 @@ +[client] +host={{ database_address }} +port={{ database_port }} +user={{ prometheus_mysql_exporter_database_user }} +password={{ prometheus_mysql_exporter_database_password }} diff --git a/ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2 b/ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2 new file mode 100644 index 0000000000..0964be4abd --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2 @@ -0,0 +1,11 @@ +{ + "command": "/opt/haproxy_exporter/haproxy_exporter -haproxy.scrape-uri unix:/var/lib/kolla/haproxy/haproxy.sock -web.listen-address {{ api_interface_address }}:{{ prometheus_haproxy_exporter_port }}", + "config_files": [], + "permissions": [ + { + "path": "/var/log/kolla/prometheus", + "owner": "prometheus:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/prometheus/templates/prometheus-mysqld-exporter.json.j2 b/ansible/roles/prometheus/templates/prometheus-mysqld-exporter.json.j2 new file mode 100644 index 0000000000..25d88bb253 --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-mysqld-exporter.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "/opt/mysqld_exporter/mysqld_exporter -config.my-cnf /etc/prometheus/my.cnf -web.listen-address {{ api_interface_address }}:{{ prometheus_mysqld_exporter_port }}", + "config_files": [ + { + "source": "{{ container_config_directory }}/my.cnf", + "dest": "/etc/prometheus/my.cnf", + "owner": "prometheus", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/prometheus", + "owner": "prometheus:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/prometheus/templates/prometheus-node-exporter.json.j2 b/ansible/roles/prometheus/templates/prometheus-node-exporter.json.j2 new file mode 100644 index 0000000000..8b44c49689 --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-node-exporter.json.j2 @@ -0,0 +1,11 @@ +{ + "command": "/opt/node_exporter/node_exporter -collector.procfs /host/proc -collector.sysfs /host/sys -web.listen-address {{ api_interface_address }}:{{ prometheus_node_exporter_port }}", + "config_files": [], + "permissions": [ + { + "path": "/var/log/kolla/prometheus", + "owner": "prometheus:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/prometheus/templates/prometheus-server.json.j2 b/ansible/roles/prometheus/templates/prometheus-server.json.j2 new file mode 100644 index 0000000000..46b70425cd --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-server.json.j2 @@ -0,0 +1,23 @@ +{ + "command": "/opt/prometheus/prometheus -config.file /etc/prometheus/prometheus.yml -web.listen-address {{ api_interface_address }}:{{ prometheus_port }} -log.format logger:stdout -storage.local.path /var/lib/prometheus", + "config_files": [ + { + "source": "{{ container_config_directory }}/prometheus.yml", + "dest": "/etc/prometheus/prometheus.yml", + "owner": "prometheus", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/data", + "owner": "prometheus:kolla", + "recurse": true + }, + { + "path": "/var/log/kolla/prometheus", + "owner": "prometheus:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 new file mode 100644 index 0000000000..a033dfc4e1 --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -0,0 +1,41 @@ +global: + scrape_interval: 60s + scrape_timeout: 10s + evaluation_interval: 15s + external_labels: + monitor: 'kolla' + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: +{% for host in groups['prometheus'] %} + - '{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ prometheus_port }}' +{% endfor %} + +{% if enable_prometheus_node_exporter | bool %} + - job_name: node + static_configs: + - targets: +{% for host in groups['prometheus-node-exporter'] %} + - '{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}' +{% endfor %} +{% endif %} + +{% if enable_prometheus_mysqld_exporter | bool %} + - job_name: mysqld + static_configs: + - targets: +{% for host in groups['prometheus-mysqld-exporter'] %} + - '{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}' +{% endfor %} +{% endif %} + +{% if enable_prometheus_haproxy_exporter | bool %} + - job_name: haproxy + static_configs: + - targets: +{% for host in groups['prometheus-haproxy-exporter'] %} + - '{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}' +{% endfor %} +{% endif %} diff --git a/ansible/site.yml b/ansible/site.yml index e70995b06a..451f91b356 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -157,6 +157,19 @@ tags: mariadb, when: enable_mariadb | bool } +- name: Apply role prometheus + gather_facts: false + hosts: + - prometheus + - prometheus-node-exporter + - prometheus-mysqld-exporter + - prometheus-haproxy-exporter + serial: '{{ serial|default("0") }}' + roles: + - { role: prometheus, + tags: prometheus, + when: enable_prometheus | bool } + - name: Apply role iscsi gather_facts: false hosts: diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index d0a78c7a5b..ef93b39be1 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -220,6 +220,7 @@ kolla_internal_vip_address: "10.10.10.254" #enable_ovs_dpdk: "no" #enable_osprofiler: "no" #enable_panko: "no" +#enable_prometheus: "no" #enable_qdrouterd: "no" #enable_rally: "no" #enable_redis: "no" @@ -430,3 +431,10 @@ tempest_floating_network_name: #xenserver_himn_ip: #xenserver_username: #xenserver_connect_protocol: + +############ +# Prometheus +############ +#enable_prometheus_haproxy_exporter: "{{ enable_haproxy | bool }}" +#enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}" +#enable_prometheus_node_exporter: "yes" diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index 0094a156c9..4f63a244c9 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -244,3 +244,8 @@ redis_master_password: # XenAPI options ################ xenserver_password: + +#################### +# Prometheus options +#################### +prometheus_mysql_exporter_database_password: diff --git a/releasenotes/notes/prometheus-dbb1aee8c88943c4.yaml b/releasenotes/notes/prometheus-dbb1aee8c88943c4.yaml new file mode 100644 index 0000000000..7df198b386 --- /dev/null +++ b/releasenotes/notes/prometheus-dbb1aee8c88943c4.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Deploy prometheus (prometheus.io) as the timeseries database. + Containers for node_exporter, haproxy_exporter and mysqld_exporter are + provided and added to prometheus as scrape targets. diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index ea9150a203..c2d1d4b2eb 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -154,6 +154,9 @@ control [influxdb:children] monitoring +[prometheus:children] +monitoring + [magnum:children] control @@ -620,3 +623,17 @@ vitrage [vitrage-ml:children] vitrage + +# Prometheus +[prometheus-node-exporter:children] +monitoring +control +compute +network +storage + +[prometheus-mysqld-exporter:children] +mariadb + +[prometheus-haproxy-exporter:children] +haproxy