From 56c8cb8a976bc1c83af1613af2d49a243404b3a2 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 5 Jan 2023 15:41:18 +0530 Subject: [PATCH] Add Virt Collectd plugin This patch adds the virt collectd plugin to Browbeat. The plugin monitors CPU, disk, network and memory resources on VMs on compute nodes through information provided by the hypervisor, so performance of the VMs is not majorly affected by the plugin. Change-Id: I617cb3c29ae417c87d6b2c2652610ff0e8bcd4b1 --- ansible/install/group_vars/all.yml | 5 ++ ansible/install/roles/collectd/tasks/main.yml | 3 + .../templates/compute.collectd.conf.j2 | 13 ++++ ...enstack_general_system_performance.yaml.j2 | 2 + .../templates/partials/virt_monitoring.yaml | 70 +++++++++++++++++++ .../collectd-openstack/Dockerfile | 4 +- 6 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 ansible/install/roles/grafana-dashboards/templates/partials/virt_monitoring.yaml diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index d338933fb..c71079537 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -409,6 +409,11 @@ osp_individual_resources_response_time: false # all metrics on all compute nodes ovn_compute_collectd_plugin: false +####################################################################### +# Virt plugin to monitor resources in VMs on computes +####################################################################### +virt_collectd_plugin: false + ############################################ # Prometheus/Collectd-Exporter Configuration ############################################ diff --git a/ansible/install/roles/collectd/tasks/main.yml b/ansible/install/roles/collectd/tasks/main.yml index 176730d2e..ddc387637 100644 --- a/ansible/install/roles/collectd/tasks/main.yml +++ b/ansible/install/roles/collectd/tasks/main.yml @@ -270,6 +270,9 @@ {% if config_type == 'controller' and ceph_controller_collectd_plugin and inventory_hostname == groups['Controller'][0] %} -v /etc/ceph/:/etc/ceph/ -v /var/run/ceph/:/var/run/ceph \ {% endif %} + {% if config_type == 'compute' and virt_collectd_plugin %} + -v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \ + {% endif %} {% if config_type == 'cephstorage' and ceph_storage_collectd_plugin and inventory_hostname == groups['CephStorage'][0] %} -v /var/run/ceph/:/var/run/ceph \ {% endif %} diff --git a/ansible/install/roles/collectd/templates/compute.collectd.conf.j2 b/ansible/install/roles/collectd/templates/compute.collectd.conf.j2 index 3b25c946c..1e4477a22 100644 --- a/ansible/install/roles/collectd/templates/compute.collectd.conf.j2 +++ b/ansible/install/roles/collectd/templates/compute.collectd.conf.j2 @@ -48,6 +48,9 @@ LoadPlugin processes LoadPlugin swap LoadPlugin tail LoadPlugin uptime +{% if virt_collectd_plugin %} +LoadPlugin virt +{% endif %} PreCacheChain "PreCache" @@ -127,6 +130,15 @@ PreCacheChain "PreCache" ReportInodes true +{% if virt_collectd_plugin %} + + Connection "qemu:///system" + RefreshInterval 60 + HostnameFormat name hostname uuid + ExtraStats "cpu_util disk job_stats_background" + +{% endif %} + {%if disk_compute_collectd_plugin %} Disk "/^[hsv]d[a-z]+[0-9]?$/" @@ -159,6 +171,7 @@ PreCacheChain "PreCache" {% endif %} + # (akrzos) Including the version of OpenStack that the process was verified as running after # OpenStack Installation with a comment at the end of each Process/ProcessMatch statement. # A Minus before the version means the process was not found in that version. (Ex -10) diff --git a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 index c117a6767..9c0c23226 100644 --- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 @@ -177,6 +177,8 @@ dashboard: {% include 'partials/tail.yaml' %} +{% include 'partials/virt_monitoring.yaml' %} + {% if item.template_node_type in sqlalchemy_groups %} {% include 'partials/sqlalchemy.yaml' %} {% endif %} diff --git a/ansible/install/roles/grafana-dashboards/templates/partials/virt_monitoring.yaml b/ansible/install/roles/grafana-dashboards/templates/partials/virt_monitoring.yaml new file mode 100644 index 000000000..e492d61f5 --- /dev/null +++ b/ansible/install/roles/grafana-dashboards/templates/partials/virt_monitoring.yaml @@ -0,0 +1,70 @@ + - title: Virt Metrics + collapse: true + height: 200px + showTitle: true + panels: + - title: $Cloud - $Node - Virt Disk Metrics + type: graph + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.virt.disk*.*, 3, 4) + - target: aliasByMetric($Cloud.$Node.virt.total*) + + - title: $Cloud - $Node - Virt Network Metrics + type: graph + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.virt.if*.*, 3, 4) + + - title: $Cloud - $Node - Virt Memory Metrics + type: graph + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: alias($Cloud.$Node.virt.memory-total, 'memory-total') + + - title: $Cloud - $Node - Virt vCPU Metrics + type: graph + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: alias($Cloud.$Node.virt.percent-virt_cpu_total, 'percent-virt_cpu_total') + - target: alias($Cloud.$Node.virt.virt_cpu_total, 'virt_cpu_total') diff --git a/browbeat-containers/collectd-openstack/Dockerfile b/browbeat-containers/collectd-openstack/Dockerfile index c082a95a3..30f7d3ce0 100644 --- a/browbeat-containers/collectd-openstack/Dockerfile +++ b/browbeat-containers/collectd-openstack/Dockerfile @@ -5,7 +5,7 @@ RUN dnf clean all && \ dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ dnf install -y centos-release-opstools && \ dnf install -y collectd collectd-turbostat collectd-disk collectd-apache collectd-ceph \ - collectd-mysql collectd-python collectd-ping python3-sqlalchemy-collectd && \ + collectd-mysql collectd-python collectd-ping collectd-virt python3-sqlalchemy-collectd && \ dnf install -y sysstat && \ dnf install -y python3-pip python3-devel && \ pip3 install --upgrade pip && \ @@ -22,7 +22,7 @@ RUN useradd stack RUN echo stack | passwd stack --stdin RUN echo "stack ALL=(root) NOPASSWD:ALL" | tee -a /etc/sudoers.d/stack RUN chmod 0440 /etc/sudoers.d/stack - +RUN rm /etc/collectd.d/virt.conf ADD files/collectd_ceph_storage.py /usr/local/bin/collectd_ceph_storage.py ADD files/collectd_gnocchi_status.py /usr/local/bin/collectd_gnocchi_status.py