
We're seeing noise like the following in doc builds: /foo/.tox/docs/lib/python3.9/site-packages/docutils/statemachine.py:707: ResourceWarning: unclosed file <_io.BufferedReader name='/foo/.git/objects/pack/pack-dd69481843ca1b7377f2f109b0022221437aca20.pack'> if not hasattr(pattern, 'match'): ResourceWarning: Enable tracemalloc to get the object allocation traceback Enable tracemalloc for Sphinx gives us the 'Loader' as the root cause: $ python -W all::ResourceWarning -X tracemalloc=100 -m sphinx.cmd.build ... ... File "/foo/.tox/docs/lib/python3.9/site-packages/reno/sphinxext.py", lineno 114 ldr = loader.Loader(conf) File "/foo/.tox/docs/lib/python3.9/site-packages/reno/loader.py", lineno 63 self._load_data() ... (you could also use PYTHONTRACEMALLOC and PYTHONWARNINGS env vars) Following this thread, it appears 'reno.scanner.Scanner' creates an instance of 'dulwich.repo.Repo', however, it fails to close it [1]. The 'reno.loader.Loader' uses 'Scanner', meaning this also leaves around open files. The solution is simple: add a 'close()' method to both the 'Scanner' and 'Loader', and provide the necessary '__enter__' and '__exit__' magic methods to use it as a context manager, like the 'Repo' object itself supports. [1] https://www.dulwich.io/docs/api/dulwich.repo.html#dulwich.repo.Repo.close Change-Id: I0b9776f431cf902a9ace5d52961eb77caaae8eaa Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
119 lines
3.8 KiB
Python
119 lines
3.8 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import collections
|
|
import os
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
from reno import loader
|
|
from reno import scanner
|
|
|
|
|
|
def build_cache_db(conf, versions_to_include):
|
|
with scanner.Scanner(conf) as s:
|
|
branches = [conf.branch]
|
|
if not conf.branch: # if no branch requested, scan all
|
|
branches += s.get_series_branches()
|
|
|
|
notes = collections.OrderedDict()
|
|
for branch in branches:
|
|
notes.update(s.get_notes_by_version(branch))
|
|
|
|
# Default to including all versions returned by the scanner.
|
|
if not versions_to_include:
|
|
versions_to_include = list(notes.keys())
|
|
|
|
# Build a cache data structure including the file contents as well
|
|
# as the basic data returned by the scanner.
|
|
file_contents = {}
|
|
for version in versions_to_include:
|
|
for filename, sha in notes[version]:
|
|
body = s.get_file_at_commit(filename, sha)
|
|
# We want to save the contents of the file, which is YAML,
|
|
# inside another YAML file. That looks terribly ugly with
|
|
# all of the escapes needed to format it properly as
|
|
# embedded YAML, so parse the input and convert it to a
|
|
# data structure that can be serialized cleanly.
|
|
y = yaml.safe_load(body)
|
|
file_contents[filename] = y
|
|
|
|
cache = {
|
|
'notes': [
|
|
{'version': k, 'files': v}
|
|
for k, v in notes.items()
|
|
],
|
|
'dates': [
|
|
{'version': k, 'date': v}
|
|
for k, v in s.get_version_dates().items()
|
|
],
|
|
'file-contents': file_contents,
|
|
}
|
|
return cache
|
|
|
|
|
|
def write_cache_db(conf, versions_to_include, outfilename=None):
|
|
"""Create a cache database file for the release notes data.
|
|
|
|
Build the cache database from scanning the project history and
|
|
write it to a file within the project.
|
|
|
|
By default, the data is written to the same file the scanner will
|
|
try to read when it cannot look at the git history. If outfilename
|
|
is given and is '-' the data is written to stdout
|
|
instead. Otherwise, if outfilename is given, the data overwrites
|
|
the named file.
|
|
|
|
Return the name of the file created, if any.
|
|
|
|
"""
|
|
encoding = conf.options['encoding']
|
|
if outfilename == '-':
|
|
stream = sys.stdout
|
|
close_stream = False
|
|
elif outfilename:
|
|
stream = open(outfilename, 'w', encoding=encoding)
|
|
close_stream = True
|
|
else:
|
|
outfilename = loader.get_cache_filename(conf)
|
|
if not os.path.exists(os.path.dirname(outfilename)):
|
|
os.makedirs(os.path.dirname(outfilename))
|
|
stream = open(outfilename, 'w', encoding=encoding)
|
|
close_stream = True
|
|
try:
|
|
cache = build_cache_db(
|
|
conf,
|
|
versions_to_include=versions_to_include,
|
|
)
|
|
yaml.safe_dump(
|
|
cache,
|
|
stream,
|
|
allow_unicode=True,
|
|
explicit_start=True,
|
|
encoding='utf-8',
|
|
)
|
|
finally:
|
|
if close_stream:
|
|
stream.close()
|
|
return outfilename
|
|
|
|
|
|
def cache_cmd(args, conf):
|
|
"Generates a release notes cache"
|
|
write_cache_db(
|
|
conf=conf,
|
|
versions_to_include=args.version,
|
|
outfilename=args.output,
|
|
)
|
|
return
|