From 753bebbb2c4d78a2be7c204865e841729c43c032 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 6 Feb 2023 09:41:34 -0800 Subject: [PATCH] Cleanup some Python ResourceWarnings in the test suite First thing we add support for PYTHONTRACEMALLOC being passed through nox to easily enable tracebacks on emitted ResourceWarnings. We set this value to 0 by deafult as enabling this slows things down and requires more memory. But it is super useful locally when debugging specific ResourceWarnings to set `PYTHONTRACEMALLOC=5` in order o correct these issues. With that in place we identify and correct two classes of ResourceWarnings. First up is the executor server not closing its statsd socket when stopping the executor server. Address that by closing the socket if statsd is enabled and set the statsd attribute to None to prevent anything else from using it later. Second is a test only issue where we don't close the fake Gerrit, Gitlab, or Web Proxy Fixture server's HTTP socket we only shutdown the server. Add a close call to the server after it is shutdown to correct this. There are potentially other ResourceWarnings to be found and cleaned up, but sifting through the noise will be easier as we eliminate these more widespread warnings. Change-Id: Iddabe79be1c8557e300dde21a6b34e57b04c48e0 --- noxfile.py | 4 ++++ tests/base.py | 2 ++ tests/fakegitlab.py | 1 + zuul/executor/server.py | 2 ++ 4 files changed, 9 insertions(+) diff --git a/noxfile.py b/noxfile.py index cdba605f06..e920f053e3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -40,6 +40,10 @@ def set_standard_env_vars(session): 'always::DeprecationWarning:zuul.driver.sql.alembic.env', 'always::DeprecationWarning:zuul.driver.sql.alembic.script', ]) + # Set PYTHONTRACEMALLOC to a value greater than 0 in the calling env + # to get tracebacks of that depth for ResourceWarnings. Disabled by + # default as this consumes more resources and is slow. + set_env(session, 'PYTHONTRACEMALLOC', '0') @nox.session(python='3') diff --git a/tests/base.py b/tests/base.py index cdcf633903..0a0ec4ae06 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1227,6 +1227,7 @@ class GerritWebServer(object): def stop(self): self.httpd.shutdown() self.thread.join() + self.httpd.server_close() class FakeGerritPoller(gerritconnection.GerritPoller): @@ -4082,6 +4083,7 @@ class WebProxyFixture(fixtures.Fixture): def _cleanup(self): self.httpd.shutdown() self.thread.join() + self.httpd.server_close() class ZuulWebFixture(fixtures.Fixture): diff --git a/tests/fakegitlab.py b/tests/fakegitlab.py index c4706b3b11..294887af0b 100644 --- a/tests/fakegitlab.py +++ b/tests/fakegitlab.py @@ -262,3 +262,4 @@ class GitlabWebServer(object): def stop(self): self.httpd.shutdown() self.thread.join() + self.httpd.server_close() diff --git a/zuul/executor/server.py b/zuul/executor/server.py index c3737b5cc0..82df57b77c 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -3481,6 +3481,8 @@ class ExecutorServer(BaseMergeServer): self.statsd.gauge(base_key + '.load_average', 0) self.statsd.gauge(base_key + '.pct_used_ram', 0) self.statsd.gauge(base_key + '.running_builds', 0) + self.statsd.close() + self.statsd = None # Use the BaseMergeServer's stop method to disconnect from # ZooKeeper. We do this as one of the last steps to ensure