From 205ca3336a7646ad3fbf92d8cf2509116a10bfcf Mon Sep 17 00:00:00 2001 From: Takashi Kajinami Date: Fri, 19 Apr 2024 21:26:24 +0900 Subject: [PATCH] Remove sendfile support pysendfile[1] was added as an optional dependency but the library hasn't been maintained and has got no release since 2014. What is worse, the sendfile implementation is not actually working since The SendFileIterator class was removed[2]. (Follow-up[3] removed the remaining reference to the class). The broken implementation has not been detected because the client is not currently used to upload contents. Remove the incomplete implementation to get rid of the dependency on the unmaintained library. [1] https://pypi.org/project/pysendfile/ [2] 76c3620c7effb15be577450c276db1e394c2de3e [3] 0c151d7d7ea00627fad182898958db55b1fef3fd Closes-Bug: #2062573 Change-Id: Ia4784f59d16660e8d40c0e409f092ac4e46870b4 --- doc/source/admin/requirements.rst | 5 ++-- glance/common/client.py | 40 ++----------------------------- glance/tests/stubs.py | 40 ------------------------------- test-requirements.txt | 1 - 4 files changed, 4 insertions(+), 82 deletions(-) diff --git a/doc/source/admin/requirements.rst b/doc/source/admin/requirements.rst index 4ac2d350d9..0f9d123a0e 100644 --- a/doc/source/admin/requirements.rst +++ b/doc/source/admin/requirements.rst @@ -23,9 +23,8 @@ External Requirements Affecting Glance Like other OpenStack projects, Glance uses some external libraries for a subset of its features. Some examples include the ``qemu-img`` utility used by the -tasks feature, ``sendfile`` to utilize the "zero-copy" way of copying data -faster, ``pydev`` to debug using popular IDEs, ``python-xattr`` for Image Cache -using "xattr" driver. +tasks feature, ``pydev`` to debug using popular IDEs, ``python-xattr`` for +Image Cache using "xattr" driver. On the other hand, if ``dnspython`` is installed in the environment, Glance provides a workaround to make it work with IPV6. diff --git a/glance/common/client.py b/glance/common/client.py index 5ebdeee33e..e62965f084 100644 --- a/glance/common/client.py +++ b/glance/common/client.py @@ -19,7 +19,6 @@ import collections.abc import copy -import errno import functools import http.client import os @@ -35,12 +34,6 @@ except ImportError: import osprofiler.web -try: - import sendfile # noqa - SENDFILE_SUPPORTED = True -except ImportError: - SENDFILE_SUPPORTED = False - from oslo_log import log as logging from oslo_utils import encodeutils from oslo_utils import netutils @@ -446,12 +439,6 @@ class BaseClient(object): def _filelike(body): return hasattr(body, 'read') - def _sendbody(connection, iter): - connection.endheaders() - for sent in iter: - # iterator has done the heavy lifting - pass - def _chunkbody(connection, iter): connection.putheader('Transfer-Encoding', 'chunked') connection.endheaders() @@ -469,22 +456,15 @@ class BaseClient(object): elif _filelike(body) or self._iterable(body): c.putrequest(method, path) - use_sendfile = self._sendable(body) - # According to HTTP/1.1, Content-Length and Transfer-Encoding # conflict. for header, value in headers.items(): - if use_sendfile or header.lower() != 'content-length': + if header.lower() != 'content-length': c.putheader(header, str(value)) iter = utils.chunkreadable(body) - if use_sendfile: - # send actual file without copying into userspace - _sendbody(c, iter) - else: - # otherwise iterate and chunk - _chunkbody(c, iter) + _chunkbody(c, iter) else: raise TypeError('Unsupported image type: %s' % body.__class__) @@ -528,22 +508,6 @@ class BaseClient(object): except (socket.error, IOError) as e: raise exception.ClientConnectionError(e) - def _seekable(self, body): - # pipes are not seekable, avoids sendfile() failure on e.g. - # cat /path/to/image | glance add ... - # or where add command is launched via popen - try: - os.lseek(body.fileno(), 0, os.SEEK_CUR) - return True - except OSError as e: - return (e.errno != errno.ESPIPE) - - def _sendable(self, body): - return (SENDFILE_SUPPORTED and - hasattr(body, 'fileno') and - self._seekable(body) and - not self.use_ssl) - def _iterable(self, body): return isinstance(body, collections.abc.Iterable) diff --git a/glance/tests/stubs.py b/glance/tests/stubs.py index e7159d6681..8c42b72116 100644 --- a/glance/tests/stubs.py +++ b/glance/tests/stubs.py @@ -15,23 +15,11 @@ """Stubouts, mocks and fixtures for the test suite""" -import os - -try: - import sendfile - SENDFILE_SUPPORTED = True -except ImportError: - SENDFILE_SUPPORTED = False - import routes import webob from glance.api.middleware import context from glance.api.v2 import router -import glance.common.client - - -DEBUG = False def stub_out_store_server(stubs, base_dir, **kwargs): @@ -49,23 +37,10 @@ def stub_out_store_server(stubs, base_dir, **kwargs): def fileno(self): return 42 - class FakeSendFile(object): - - def __init__(self, req): - self.req = req - - def sendfile(self, o, i, offset, nbytes): - os.lseek(i, offset, os.SEEK_SET) - prev_len = len(self.req.body) - self.req.body += os.read(i, nbytes) - return len(self.req.body) - prev_len - class FakeGlanceConnection(object): def __init__(self, *args, **kwargs): self.sock = FakeSocket() - self.stub_force_sendfile = kwargs.get('stub_force_sendfile', - SENDFILE_SUPPORTED) def connect(self): return True @@ -75,9 +50,6 @@ def stub_out_store_server(stubs, base_dir, **kwargs): def putrequest(self, method, url): self.req = webob.Request.blank(url) - if self.stub_force_sendfile: - fake_sendfile = FakeSendFile(self.req) - stubs.Set(sendfile, 'sendfile', fake_sendfile.sendfile) self.req.method = method def putheader(self, key, value): @@ -118,15 +90,3 @@ def stub_out_store_server(stubs, base_dir, **kwargs): def fake_image_iter(self): for i in self.source.app_iter: yield i - - def fake_sendable(self, body): - force = getattr(self, 'stub_force_sendfile', None) - if force is None: - return self._stub_orig_sendable(body) - else: - if force: - assert glance.common.client.SENDFILE_SUPPORTED - return force - - setattr(glance.common.client.BaseClient, '_stub_orig_sendable', - glance.common.client.BaseClient._sendable) diff --git a/test-requirements.txt b/test-requirements.txt index 3123bfd970..dc04b9d0a1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -20,7 +20,6 @@ boto3>=1.9.199 # Apache-2.0 # Optional packages that should be installed when testing PyMySQL>=0.7.6 # MIT License psycopg2>=2.8.4 # LGPL/ZPL -pysendfile>=2.0.0;sys_platform!='win32' # MIT xattr>=0.9.2;sys_platform!='win32' # MIT python-swiftclient>=3.2.0 # Apache-2.0 python-cinderclient>=4.1.0 # Apache-2.0