Merge "Add node_exporter role for stats reporting"

inventory/service/group_vars/prometheus-node-exporter.yaml

@@ -0,0 +1,2 @@
+iptables_extra_allowed_groups:
+  - {'protocol': 'tcp', 'port': '9100', 'group': 'prometheus'}

inventory/service/groups.yaml

@@ -117,6 +117,9 @@ groups:
     - paste[0-9]*.opendev.org
   prometheus:
     - prometheus[0-9]*.opendev.org
+  prometheus-node-exporter:
+    - node-exporter[0-9]*.opendev.org
+    - prometheus[0-9]*.opendev.org
   puppet:
     - cacti[0-9]*.open*.org
     - storyboard-dev[0-9]*.opendev.org

playbooks/roles/prometheus-node-exporter/README.rst

@@ -0,0 +1 @@
+Run Prometheus Node Exporter

playbooks/roles/prometheus-node-exporter/tasks/main.yaml

@@ -0,0 +1,31 @@
+- name: Ensure docker-compose directory exists
+  file:
+    state: directory
+    path: /etc/node-exporter-docker
+
+- name: Ensure data directory exists
+  file:
+    state: directory
+    path: /var/lib/node-exporter
+    owner: '65534'
+    group: '65534'
+    mode: '0755'
+
+- name: Write docker-compose settings file
+  template:
+    src: docker-compose.yaml.j2
+    dest: /etc/node-exporter-docker/docker-compose.yaml
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/node-exporter-docker/
+
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d
+    chdir: /etc/node-exporter-docker/
+
+- name: Run docker prune to cleanup not needed images
+  shell:
+    cmd: docker image prune -f

playbooks/roles/prometheus-node-exporter/templates/docker-compose.yaml.j2

@@ -0,0 +1,15 @@
+services:
+  node-exporter:
+    image: quay.io/prometheus/node-exporter:v1.11.1
+    network_mode: host
+    restart: always
+    command:
+      - '--path.rootfs=/host'
+    # rslave is required for the for the container to dynamically see new filesystems
+    # mounted on the host after starting the container
+    volumes:
+      - /:/host:ro,rslave
+    logging:
+      driver: journald
+      options:
+        tag: "docker-node-exporter"

playbooks/roles/prometheus/templates/prometheus.yml.j2

@@ -6,3 +6,10 @@ scrape_configs:
   - job_name: 'prometheus'
     static_configs:
       - targets: ['localhost:9090']
+  - job_name: 'node'
+    static_configs:
+{% for host in groups['prometheus-node-exporter'] | sort %}
+      - targets: ['{{ hostvars[host]['ansible_host'] }}:9100']
+        labels:
+          instance: '{{ host }}'
+{% endfor %}

playbooks/service-node-exporter.yaml

@@ -0,0 +1,6 @@
+- hosts: "prometheus-node-exporter:!disabled"
+  name: "Base: configure node exporter"
+  roles:
+    - iptables
+    - install-docker
+    - prometheus-node-exporter

testinfra/test_node_exporter.py

@@ -0,0 +1,20 @@
+# Copyright 2026 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+testinfra_hosts = ['prometheus-node-exporter']
+
+
+def test_node_exporter(host):
+    socket = host.socket("tcp://0.0.0.0:9100")
+    assert socket.is_listening

testinfra/test_prometheus.py

@@ -12,6 +12,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import re
+
 import requests
 
 testinfra_hosts = ['prometheus99.opendev.org']
@@ -56,3 +58,21 @@ def test_prometheus_admin_api_blocked(host):
                    '-XPOST '
                    'https://prometheus.opendev.org/api/v1/admin/tsdb/snapshot')
     assert '403 Forbidden' in cmd.stdout
+
+
+def test_prometheus_node_scrape_config(host):
+    config = host.file("/etc/prometheus/prometheus.yml")
+    assert config.exists
+    assert "job_name: 'node'" in config.content_string
+
+
+def test_prometheus_can_reach_node_exporters(host):
+    config = host.file("/etc/prometheus/prometheus.yml")
+    assert config.exists
+    targets = re.findall(r"targets: \['(.+):9100'\]",
+                         config.content_string)
+    assert len(targets) > 0, "No node exporter targets found in prometheus config"
+    for target in targets:
+        cmd = host.run('curl -sf http://{}:9100/metrics'.format(target))
+        assert cmd.rc == 0, \
+            'prometheus cannot reach node exporter at {}:9100'.format(target)

zuul.d/system-config-run.yaml

@@ -745,22 +745,30 @@
         - <<: *bridge_node
         - name: prometheus99.opendev.org
           label: ubuntu-noble
+        - name: node-exporter99.opendev.org
+          label: ubuntu-noble
       groups:
         - <<: *bastion_group
     vars:
       run_playbooks:
         - playbooks/letsencrypt.yaml
         - playbooks/service-prometheus.yaml
+        - playbooks/service-node-exporter.yaml
     files:
       - playbooks/bootstrap-bridge.yaml
       - playbooks/letsencrypt.yaml
       - playbooks/service-prometheus.yaml
+      - playbooks/service-node-exporter.yaml
       - playbooks/roles/prometheus/
+      - playbooks/roles/prometheus-node-exporter/
       - playbooks/roles/install-docker/
       - playbooks/roles/pip3/
+      - playbooks/roles/iptables/
       - playbooks/roles/run-selenium/
+      - inventory/service/group_vars/prometheus-node-exporter.yaml
       - testinfra/util.py
       - testinfra/test_prometheus.py
+      - testinfra/test_node_exporter.py
 
 - job:
     name: system-config-run-keycloak