Browse Source

Properly close streams

When running test cases in the debugger it spits out resource warnings
about non-closed streams [1]. Explicitly closing the streams of the
subprocesses and log streamer fixes this warning. Maybe this even
solves the memory leak we're currently seeing in the executors.

[1] Trace:
/zuul/executor/server.py:1894: ResourceWarning: unclosed file <_io.BufferedReader name=60>
  self.proc = None
ResourceWarning: Enable tracemalloc to get the object allocation traceback

/zuul/executor/server.py:124: ResourceWarning: unclosed file <_io.BufferedReader name=11>
  stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
ResourceWarning: Enable tracemalloc to get the object allocation traceback

Change-Id: I65f191dc2e50f9c84f5bf6a3967d768d7ebe6b04
changes/98/634598/2
Tobias Henkel 2 years ago
parent
commit
3d9f47be1d
No known key found for this signature in database GPG Key ID: 3750DEC158E5FA2
4 changed files with 10 additions and 0 deletions
  1. +1
    -0
      tests/unit/test_streaming.py
  2. +2
    -0
      zuul/executor/server.py
  3. +5
    -0
      zuul/lib/log_streamer.py
  4. +2
    -0
      zuul/web/__init__.py

+ 1
- 0
tests/unit/test_streaming.py View File

@ -55,6 +55,7 @@ class WSClient(WebSocketBaseClient):
self.send(json.dumps(req))
self.event.set()
super(WSClient, self).run()
self.close()
class TestLogStreamer(tests.base.BaseTestCase):


+ 2
- 0
zuul/executor/server.py View File

@ -141,6 +141,7 @@ class DiskAccountant(object):
.format(size=size, job=dirname, limit=self.limit))
self.usage_func(dirname, size)
du.wait()
du.stdout.close()
after = time.time()
# Sleep half as long as that took, or 1s, whichever is longer
delay_time = max((after - before) / 2, 1.0)
@ -1891,6 +1892,7 @@ class AnsibleJob(object):
self.log.debug("Stopped disk job killer")
with self.proc_lock:
self.proc.stdout.close()
self.proc = None
if timeout and watchdog.timed_out:


+ 5
- 0
zuul/lib/log_streamer.py View File

@ -98,6 +98,11 @@ class RequestHandler(streamer_utils.BaseFingerRequestHandler):
if self.follow_log(log):
break
else:
if log is not None:
try:
log.file.close()
except Exception:
pass
return
def chunk_log(self, log_file):


+ 2
- 0
zuul/web/__init__.py View File

@ -185,9 +185,11 @@ class LogStreamer(object):
if data:
self.websocket.send(data, False)
self.zuulweb.stream_manager.unregisterStreamer(self)
self.finger_socket.close()
return self.websocket.logClose(1000, "No more data")
else:
self.zuulweb.stream_manager.unregisterStreamer(self)
self.finger_socket.close()
return self.websocket.logClose(1000, "Remote error")


Loading…
Cancel
Save