Merge "Report cherrypy stats/metrics at least every 60 seconds"

tests/base.py

@@ -34,6 +34,7 @@ from typing import Generator, List
 from unittest.case import skipIf
 import zlib
 
+from apscheduler.triggers.interval import IntervalTrigger
 import prometheus_client
 import requests
 import responses
@@ -1662,7 +1663,7 @@ class ZuulWebFixture(fixtures.Fixture):
     def __init__(self, config, test_config,
                  additional_event_queues, upstream_root,
                  poller_events, git_url_with_auth, add_cleanup,
-                 test_root, info=None):
+                 test_root, info=None, stats_interval=None):
         super(ZuulWebFixture, self).__init__()
         self.config = config
         self.connections = TestConnectionRegistry(
@@ -1680,6 +1681,7 @@ class ZuulWebFixture(fixtures.Fixture):
         else:
             self.info = info
         self.test_root = test_root
+        self.stats_interval = stats_interval
 
     def _setUp(self):
         # Start the web server
@@ -1689,6 +1691,9 @@ class ZuulWebFixture(fixtures.Fixture):
             connections=self.connections,
             authenticators=self.authenticators)
         self.connections.load(self.web.zk_client, self.web.component_registry)
+        if self.stats_interval:
+            self.web._stats_interval = IntervalTrigger(
+                seconds=self.stats_interval)
         self.web.start()
         self.addCleanup(self.stop)
 

tests/unit/test_web.py

@@ -57,6 +57,7 @@ class FakeConfig(object):
 
 class WebMixin:
     config_ini_data = {}
+    stats_interval = None
 
     def startWebServer(self):
         self.zuul_ini_config = FakeConfig(self.config_ini_data)
@@ -68,7 +69,8 @@ class WebMixin:
                            self.git_url_with_auth, self.addCleanup,
                            self.test_root,
                            info=zuul.model.WebInfo.fromConfig(
-                               self.zuul_ini_config)))
+                               self.zuul_ini_config),
+                           stats_interval=self.stats_interval))
 
         self.executor_server.hold_jobs_in_build = True
 
@@ -1496,6 +1498,23 @@ class TestWeb(BaseTestWeb):
         self.assertEqual(job_params, resp.json())
 
 
+class TestWebStatsReporting(BaseTestWeb):
+    stats_interval = 1
+
+    def test_default_statsd_reporting(self):
+        # Let the system settle out then wait 2x the reporting interval
+        # before we check that things have reported.
+        self.waitUntilSettled()
+        time.sleep(2)
+        hostname = normalize_statsd_name(socket.getfqdn())
+        self.assertReportedStat(
+            f'zuul.web.server.{hostname}.threadpool.idle',
+            value='10', kind='g')
+        self.assertReportedStat(
+            f'zuul.web.server.{hostname}.threadpool.queue',
+            value='0', kind='g')
+
+
 class TestWebProviders(LauncherBaseTestCase, WebMixin):
     config_file = 'zuul-connections-nodepool.conf'
     tenant_config_file = 'config/multi-tenant-provider/main.yaml'

zuul/web/__init__.py

@@ -17,6 +17,8 @@ import cherrypy
 import socket
 from collections import defaultdict
 
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.interval import IntervalTrigger
 from opentelemetry import trace
 from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
 from ws4py.websocket import WebSocket
@@ -3085,6 +3087,7 @@ class FakeCacheKey:
 class ZuulWeb(object):
     log = logging.getLogger("zuul.web")
     tracer = trace.get_tracer("zuul")
+    _stats_interval = IntervalTrigger(seconds=60)
 
     @staticmethod
     def generateRouteMap(api, oidc, include_auth):
@@ -3423,6 +3426,7 @@ class ZuulWeb(object):
                     controller,
                     '/api/connection/%s' % connection.connection_name)
 
+        self.apsched = BackgroundScheduler()
         cherrypy.tools.stats = StatsTool(self.statsd, self.metrics)
 
         conf = {
@@ -3489,6 +3493,11 @@ class ZuulWeb(object):
             else:
                 break
 
+        self.apsched.start()
+        self.apsched.add_job(cherrypy.tools.stats.emitStats,
+                             name="Regular cherrypy stats reporting",
+                             trigger=self._stats_interval)
+
         self.log.info("Starting HTTP listeners")
         self.stream_manager.start()
         self.wsplugin = WebSocketPlugin(cherrypy.engine)
@@ -3509,6 +3518,9 @@ class ZuulWeb(object):
         self.keystore.stop()
         self._running = False
         self.component_info.state = self.component_info.STOPPED
+
+        self.apsched.shutdown()
+
         cherrypy.engine.exit()
         # Not strictly necessary, but without this, if the server is
         # started again (e.g., in the unit tests) it will reuse the