openstack/swift
Code Issues Proposed changes

proxy-logging: Add real-time transfer bytes counters

Browse Source 
Currently we can get one proxy-logging transfer stat emission over the
duration of the upload/download. We want another stat coming out of
proxy-logging: something that gets emitted periodically as bytes are
actually sent/received so we can get reasonably accurate point-in-time
breakdowns of bandwidth usage.

Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Co-Authored-By: Shreeya Deshpande <shreeyad@nvidia.com>

Change-Id: Ideecd0aa58ddf091c9f25f15022a9066088f532b
Signed-off-by: Yan Xiao <yanxiao@nvidia.com>
This commit is contained in:
Yan Xiao
2024-09-26 09:35:35 -04:00
parent 9d7e7e27a5
commit dcd5a265f6
6 changed files with 1528 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
test/unit/common/middleware/helpers.py
View File

@@ -193,8 +193,9 @@ class FakeSwift(object):
container_existence_skip_cache = 0.0
account_existence_skip_cache = 0.0
def __init__(self, capture_unexpected_calls=True):
def __init__(self, capture_unexpected_calls=True, test_read_size=-1):
self.capture_unexpected_calls = capture_unexpected_calls
self.read_size = test_read_size
self._calls = []
self._unclosed_req_keys = defaultdict(int)
self._unread_req_paths = defaultdict(int)
@@ -325,7 +326,14 @@ class FakeSwift(object):
if (cont and not obj and method == 'UPDATE') or (
obj and method == 'PUT'):
call.body = b''.join(iter(env['wsgi.input'].read, b''))
if self.read_size < 0:
call.body = b''.join(iter(env['wsgi.input'].read, b''))
else:
call.body = b''
buf = env['wsgi.input'].read(self.read_size)
while buf:
call.body += buf
buf = env['wsgi.input'].read(self.read_size)
# simulate object PUT
if method == 'PUT' and obj:
@@ -377,7 +385,7 @@ class FakeSwift(object):
resolve_ignore_range_header(req, headers)
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
if not isinstance(body, (bytes, str)):
resp = resp_class(
req=req, headers=headers, app_iter=body,
conditional_response=req.method in ('GET', 'HEAD'),

1376
test/unit/common/middleware/test_proxy_logging.py
View File

File diff suppressed because it is too large Load Diff

12
test/unit/common/test_wsgi.py
View File

@@ -1264,6 +1264,18 @@ class TestWSGI(unittest.TestCase):
newenv = wsgi.make_env(oldenv)
self.assertIs(newenv.get('swift.infocache'), oldenv['swift.infocache'])
def test_make_env_keeps_shard_listing_history(self):
oldenv = {'swift.shard_listing_history': []}
newenv = wsgi.make_env(oldenv)
self.assertIs(newenv.get('swift.shard_listing_history'),
oldenv['swift.shard_listing_history'])
def test_make_env_keeps_base_labels(self):
oldenv = {'swift.base_labels': []}
newenv = wsgi.make_env(oldenv)
self.assertIs(newenv.get('swift.base_labels'),
oldenv['swift.base_labels'])
class CommonTestMixin(object):