diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index f472a9d922..db48db29b3 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -462,6 +462,9 @@ prometheus_etcd_integration_port: "{{ etcd_client_port }}" prometheus_alertmanager_port: "9093" prometheus_alertmanager_cluster_port: "9094" +# Prometheus MSTeams port +prometheus_msteams_port: "9095" + # Prometheus openstack-exporter ports prometheus_openstack_exporter_port: "9198" prometheus_elasticsearch_exporter_port: "9108" @@ -1128,6 +1131,7 @@ enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}" enable_prometheus_rabbitmq_exporter: "{{ enable_prometheus | bool and enable_rabbitmq | bool }}" enable_prometheus_libvirt_exporter: "{{ enable_prometheus | bool and enable_nova | bool and nova_compute_virt_type in ['kvm', 'qemu'] }}" enable_prometheus_etcd_integration: "{{ enable_prometheus | bool and enable_etcd | bool }}" +enable_prometheus_msteams: "no" prometheus_alertmanager_user: "admin" prometheus_scrape_interval: "60s" @@ -1139,6 +1143,7 @@ prometheus_ceph_mgr_exporter_endpoints: [] prometheus_openstack_exporter_endpoint_type: "internal" prometheus_openstack_exporter_compute_api_version: "latest" prometheus_libvirt_exporter_interval: "60s" +prometheus_msteams_webhook_url: ############ # Vitrage diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 2f42f77a35..73a4ec82b8 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -700,6 +700,9 @@ monitoring [prometheus-libvirt-exporter:children] compute +[prometheus-msteams:children] +prometheus-alertmanager + [masakari-api:children] control diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 8018366940..e306caade3 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -718,6 +718,9 @@ monitoring [prometheus-libvirt-exporter:children] compute +[prometheus-msteams:children] +prometheus-alertmanager + [masakari-api:children] control diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml index e047ea590b..65a252101e 100644 --- a/ansible/roles/prometheus/defaults/main.yml +++ b/ansible/roles/prometheus/defaults/main.yml @@ -108,6 +108,14 @@ prometheus_services: image: "{{ prometheus_libvirt_exporter_image_full }}" volumes: "{{ prometheus_libvirt_exporter_default_volumes + prometheus_libvirt_exporter_extra_volumes }}" dimensions: "{{ prometheus_libvirt_exporter_dimensions }}" + prometheus-msteams: + container_name: "prometheus_msteams" + group: "prometheus-msteams" + enabled: "{{ enable_prometheus_msteams | bool }}" + environment: "{{ prometheus_msteams_container_proxy }}" + image: "{{ prometheus_msteams_image_full }}" + volumes: "{{ prometheus_msteams_default_volumes + prometheus_msteams_extra_volumes }}" + dimensions: "{{ prometheus_msteams_dimensions }}" #################### # Prometheus Server @@ -189,6 +197,10 @@ prometheus_libvirt_exporter_image: "{{ docker_registry ~ '/' if docker_registry prometheus_libvirt_exporter_tag: "{{ prometheus_tag }}" prometheus_libvirt_exporter_image_full: "{{ prometheus_libvirt_exporter_image }}:{{ prometheus_libvirt_exporter_tag }}" +prometheus_msteams_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/prometheus-msteams" +prometheus_msteams_tag: "{{ prometheus_tag }}" +prometheus_msteams_image_full: "{{ prometheus_msteams_image }}:{{ prometheus_msteams_tag }}" + prometheus_server_dimensions: "{{ default_container_dimensions }}" prometheus_haproxy_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_mysqld_exporter_dimensions: "{{ default_container_dimensions }}" @@ -200,6 +212,7 @@ prometheus_openstack_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_elasticsearch_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_blackbox_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_libvirt_exporter_dimensions: "{{ default_container_dimensions }}" +prometheus_msteams_dimensions: "{{ default_container_dimensions }}" prometheus_server_default_volumes: - "{{ node_config_directory }}/prometheus-server/:{{ container_config_directory }}/:ro" @@ -265,6 +278,11 @@ prometheus_libvirt_exporter_default_volumes: - "/etc/localtime:/etc/localtime:ro" - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" - "/run/libvirt:/run/libvirt:ro" +prometheus_msteams_default_volumes: + - "{{ node_config_directory }}/prometheus-msteams/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" + - "kolla_logs:/var/log/kolla/" prometheus_extra_volumes: "{{ default_extra_volumes }}" prometheus_server_extra_volumes: "{{ prometheus_extra_volumes }}" @@ -278,6 +296,9 @@ prometheus_openstack_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_elasticsearch_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_blackbox_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_libvirt_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" +prometheus_msteams_extra_volumes: "{{ prometheus_extra_volumes }}" + +prometheus_msteams_container_proxy: "{{ container_proxy }}" prometheus_openstack_exporter_disabled_volume: "{{ '--disable-service.volume' if not enable_cinder | bool else '' }}" prometheus_openstack_exporter_disabled_dns: "{{ '--disable-service.dns' if not enable_designate | bool else '' }}" diff --git a/ansible/roles/prometheus/handlers/main.yml b/ansible/roles/prometheus/handlers/main.yml index c9ac641b5f..ca6b319f86 100644 --- a/ansible/roles/prometheus/handlers/main.yml +++ b/ansible/roles/prometheus/handlers/main.yml @@ -165,3 +165,19 @@ dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" + +- name: Restart prometheus-msteams container + vars: + service_name: "prometheus-msteams" + service: "{{ prometheus_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + environment: "{{ service.environment }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" diff --git a/ansible/roles/prometheus/tasks/config.yml b/ansible/roles/prometheus/tasks/config.yml index ca2511b1f0..f55f6b5baf 100644 --- a/ansible/roles/prometheus/tasks/config.yml +++ b/ansible/roles/prometheus/tasks/config.yml @@ -237,3 +237,37 @@ when: - inventory_hostname in groups[service.group] - service.enabled | bool + +- name: Copying over prometheus msteams config file + vars: + service: "{{ prometheus_services['prometheus-msteams'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/prometheus-msteams/msteams.yml" + become: true + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + with_first_found: + - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus-msteams.yml" + - "{{ node_custom_config }}/prometheus/prometheus-msteams.yml" + - "{{ role_path }}/templates/prometheus-msteams.yml.j2" + notify: + - Restart prometheus-msteams container + +- name: Copying over prometheus msteams template file + vars: + service: "{{ prometheus_services['prometheus-msteams'] }}" + copy: + src: "{{ item }}" + dest: "{{ node_config_directory }}/prometheus-msteams/msteams.tmpl" + become: true + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + with_first_found: + - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus-msteams.tmpl" + - "{{ node_custom_config }}/prometheus/prometheus-msteams.tmpl" + - "{{ role_path }}/templates/prometheus-msteams.tmpl" + notify: + - Restart prometheus-msteams container diff --git a/ansible/roles/prometheus/tasks/precheck.yml b/ansible/roles/prometheus/tasks/precheck.yml index cc734eb0ba..dd98f3e482 100644 --- a/ansible/roles/prometheus/tasks/precheck.yml +++ b/ansible/roles/prometheus/tasks/precheck.yml @@ -164,3 +164,17 @@ - enable_prometheus_libvirt_exporter | bool with_items: - "{{ prometheus_libvirt_exporter_port }}" + +- name: Checking free ports for Prometheus msteams + wait_for: + host: "{{ 'api' | kolla_address }}" + port: "{{ item }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['prometheus_msteams'] is not defined + - inventory_hostname in groups['prometheus-msteams'] + - enable_prometheus_msteams | bool + with_items: + - "{{ prometheus_msteams_port }}" diff --git a/ansible/roles/prometheus/templates/prometheus-alertmanager.yml.j2 b/ansible/roles/prometheus/templates/prometheus-alertmanager.yml.j2 index f5fbe1d265..9e2dbd016e 100644 --- a/ansible/roles/prometheus/templates/prometheus-alertmanager.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus-alertmanager.yml.j2 @@ -6,6 +6,10 @@ route: group_wait: 10s group_interval: 5m repeat_interval: 3h +{% if enable_prometheus_msteams | bool %} + routes: + - receiver: 'prometheus-msteams' +{% endif %} receivers: - name: default-receiver {% if enable_vitrage | bool and enable_vitrage_prometheus_datasource | bool %} @@ -17,5 +21,11 @@ receivers: username: '{{ keystone_admin_user }}' password: '{{ keystone_admin_password }}' {% endif %} +{% if enable_prometheus_msteams | bool %} + - name: 'prometheus-msteams' + webhook_configs: + - send_resolved: true + url: 'http://localhost:{{ prometheus_msteams_port }}/alertmanager' +{% endif %} templates: - '/etc/prometheus/*.tmpl' diff --git a/ansible/roles/prometheus/templates/prometheus-msteams.json.j2 b/ansible/roles/prometheus/templates/prometheus-msteams.json.j2 new file mode 100644 index 0000000000..758be92a72 --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-msteams.json.j2 @@ -0,0 +1,24 @@ +{ + "command": "/opt/prometheus-msteams -http-addr localhost:{{ prometheus_msteams_port }} -config-file /etc/msteams/msteams.yml -template-file /etc/msteams/msteams.tmpl", + "config_files": [ + { + "source": "{{ container_config_directory }}/msteams.yml", + "dest": "/etc/msteams/msteams.yml", + "owner": "prometheus", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/msteams.tmpl", + "dest": "/etc/msteams/msteams.tmpl", + "owner": "prometheus", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/prometheus", + "owner": "prometheus:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/prometheus/templates/prometheus-msteams.tmpl b/ansible/roles/prometheus/templates/prometheus-msteams.tmpl new file mode 100644 index 0000000000..36f64f9c28 --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-msteams.tmpl @@ -0,0 +1,50 @@ +{{ define "teams.card" }} +{ + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + "themeColor": "{{- if eq .Status "resolved" -}}2DC72D + {{- else if eq .Status "firing" -}} + {{- if eq .CommonLabels.severity "critical" -}}8C1A1A + {{- else if eq .CommonLabels.severity "warning" -}}FFA500 + {{- else -}}808080{{- end -}} + {{- else -}}808080{{- end -}}", + "summary": "{{- if eq .CommonAnnotations.summary "" -}} + {{- if eq .CommonAnnotations.message "" -}} + {{- if eq .CommonLabels.alertname "" -}} + Prometheus Alert + {{- else -}} + {{- .CommonLabels.alertname -}} + {{- end -}} + {{- else -}} + {{- .CommonAnnotations.message -}} + {{- end -}} + {{- else -}} + {{- .CommonAnnotations.summary -}} + {{- end -}}", + "title": "Prometheus Alert ({{ .Status | title }})", + "sections": [ {{$externalUrl := .ExternalURL}} + {{- range $index, $alert := .Alerts }}{{- if $index }},{{- end }} + { + "activityTitle": "[{{ $alert.Annotations.description }}]({{ $externalUrl }})", + "facts": [ + {{- range $key, $value := $alert.Annotations }} + { + {{- if ne $key "description" -}} + "name": "{{ $key }}", + "value": "{{ $value }}" + {{- end -}} + }, + {{- end -}} + {{$c := counter}}{{ range $key, $value := $alert.Labels }}{{if call $c}},{{ end }} + { + "name": "{{ $key }}", + "value": "{{ $value }}" + } + {{- end }} + ], + "markdown": true + } + {{- end }} + ] +} +{{ end }} diff --git a/ansible/roles/prometheus/templates/prometheus-msteams.yml.j2 b/ansible/roles/prometheus/templates/prometheus-msteams.yml.j2 new file mode 100644 index 0000000000..49ad1d12dc --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-msteams.yml.j2 @@ -0,0 +1,2 @@ +connectors: + - alertmanager: "{{ prometheus_msteams_webhook_url }}" diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index e5cc45e682..635dab8856 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -718,6 +718,7 @@ workaround_ansible_issue_8743: yes #enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}" #enable_prometheus_libvirt_exporter: "{{ enable_prometheus | bool and enable_nova | bool and nova_compute_virt_type in ['kvm', 'qemu'] }}" #enable_prometheus_etcd_integration: "{{ enable_prometheus | bool and enable_etcd | bool }}" +#enable_prometheus_msteams: "no" # The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager). # prometheus_external_labels: diff --git a/releasenotes/notes/prometheus-msteams-361a2b76300e7921.yaml b/releasenotes/notes/prometheus-msteams-361a2b76300e7921.yaml new file mode 100644 index 0000000000..fbf07cb334 --- /dev/null +++ b/releasenotes/notes/prometheus-msteams-361a2b76300e7921.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds support for deploying ``prometheus-msteams``, which can be used to + forward Prometheus Alertmanager notifications to Microsoft Teams. It is + enabled by setting ``enable_prometheus_msteams`` to ``true``. diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index f1222e4134..50285850e8 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -771,6 +771,9 @@ monitoring [prometheus-libvirt-exporter:children] compute +[prometheus-msteams:children] +prometheus-alertmanager + # NOTE(yoctozepto): In CI we want to test Masakari HA but not of other services, # to conserve the resources. Hence, we set Masakari groups to use both # primary and secondary while the parent group (control) uses only primary.