openstack
/
glance
Code Issues Proposed changes

VMware store: Use the Content-Length if available 

Change I579084460e7f61ab4042632d17ec0f045fa6f5af changed the VMware
store to use chunked encoding to upload data to the underlying backend.
We should do that only if the image size is not provided or zero and not
all the time.

This patch addressed the issue by checking the image_size before upload.

Change-Id: If348be7fd24fd05146b0c86bef45b79f600cc0f7
Closes-Bug: #1336970
This commit is contained in:
Arnaud Legendre 2014-07-02 15:33:54 -07:00
parent d67a94b602
commit 8c161b6a4b
2 changed files with 82 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
glance/store/vmware_datastore.py
View File

@ -114,13 +114,38 @@ def http_response_iterator(conn, response, size):
class _Reader(object):
def __init__(self, data, checksum, blocksize=8192):
self.data = data
self.checksum = checksum
def __init__(self, data):
self._size = 0
self.data = data
self.checksum = hashlib.md5()
def read(self, size=None):
result = self.data.read(size)
self._size += len(result)
self.checksum.update(result)
return result
def rewind(self):
try:
self.data.seek(0)
self._size = 0
self.checksum = hashlib.md5()
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to rewind image content'))
@property
def size(self):
return self._size
class _ChunkReader(_Reader):
def __init__(self, data, blocksize=8192):
self.blocksize = blocksize
self.current_chunk = ""
self.closed = False
super(_ChunkReader, self).__init__(data)
def read(self, size=None):
ret = ""
@ -137,13 +162,6 @@ class _Reader(object):
self.current_chunk = self.current_chunk[size:]
return ret
def rewind(self):
try:
self.data.seek(0)
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to rewind image content'))
def _get_chunk(self):
if not self.closed:
chunk = self.data.read(self.blocksize)
@ -156,10 +174,6 @@ class _Reader(object):
self.current_chunk = '0\r\n\r\n'
self.closed = True
@property
def size(self):
return self._size
class StoreLocation(glance.store.location.StoreLocation):
"""Class describing an VMware URI.
@ -320,8 +334,15 @@ class Store(glance.store.base.Store):
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
if image_size > 0:
headers = {'Content-Length': image_size}
image_file = _Reader(image_file)
else:
# NOTE (arnaud): use chunk encoding when the image is still being
# generated by the server (ex: stream optimized disks generated by
# Nova).
headers = {'Transfer-Encoding': 'chunked'}
image_file = _ChunkReader(image_file)
loc = StoreLocation({'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
@ -330,9 +351,7 @@ class Store(glance.store.base.Store):
'image_id': image_id})
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Connection': 'Keep-Alive',
'Cookie': cookie,
'Transfer-Encoding': 'chunked'}
headers = dict(headers.items() + {'Cookie': cookie}.items())
try:
conn = self._get_http_conn('PUT', loc, headers,
content=image_file)
@ -358,7 +377,8 @@ class Store(glance.store.base.Store):
raise exception.UnexpectedStatus(status=res.status,
body=res.read())
return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
return (loc.get_uri(), image_file.size,
image_file.checksum.hexdigest(), {})
def get(self, location):
"""Takes a `glance.store.location.Location` object that indicates

51
glance/tests/unit/test_vmware_store.py
View File

@ -254,7 +254,43 @@ class TestStore(base.StoreClearingUnitTest):
HttpConn.return_value = FakeHTTPConnection(status=404)
self.assertRaises(exception.NotFound, self.store.get_size, loc)
def test_reader_image_fits_in_blocksize(self):
def test_reader_full(self):
content = 'XXX'
image = six.StringIO(content)
expected_checksum = hashlib.md5(content).hexdigest()
reader = vm_store._Reader(image)
ret = reader.read()
self.assertEqual(content, ret)
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
self.assertEqual(len(content), reader.size)
def test_reader_partial(self):
content = 'XXX'
image = six.StringIO(content)
expected_checksum = hashlib.md5('X').hexdigest()
reader = vm_store._Reader(image)
ret = reader.read(1)
self.assertEqual('X', ret)
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
self.assertEqual(1, reader.size)
def test_rewind(self):
content = 'XXX'
image = six.StringIO(content)
expected_checksum = hashlib.md5(content).hexdigest()
reader = vm_store._Reader(image)
reader.read(1)
ret = reader.read()
self.assertEqual('XX', ret)
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
self.assertEqual(len(content), reader.size)
reader.rewind()
ret = reader.read()
self.assertEqual(content, ret)
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
self.assertEqual(len(content), reader.size)
def test_chunkreader_image_fits_in_blocksize(self):
"""
Test that the image file reader returns the expected chunk of data
when the block size is larger than the image.
@ -262,8 +298,7 @@ class TestStore(base.StoreClearingUnitTest):
content = 'XXX'
image = six.StringIO(content)
expected_checksum = hashlib.md5(content).hexdigest()
checksum = hashlib.md5()
reader = vm_store._Reader(image, checksum)
reader = vm_store._ChunkReader(image)
ret = reader.read()
expected_chunk = '%x\r\n%s\r\n' % (len(content), content)
last_chunk = '0\r\n\r\n'
@ -277,7 +312,7 @@ class TestStore(base.StoreClearingUnitTest):
self.assertTrue(reader.closed)
self.assertEqual('', ret)
def test_reader_image_larger_blocksize(self):
def test_chunkreader_image_larger_blocksize(self):
"""
Test that the image file reader returns the expected chunks when
the block size specified is smaller than the image.
@ -285,9 +320,8 @@ class TestStore(base.StoreClearingUnitTest):
content = 'XXX'
image = six.StringIO(content)
expected_checksum = hashlib.md5(content).hexdigest()
checksum = hashlib.md5()
last_chunk = '0\r\n\r\n'
reader = vm_store._Reader(image, checksum, blocksize=1)
reader = vm_store._ChunkReader(image, blocksize=1)
ret = reader.read()
expected_chunk = '1\r\nX\r\n'
self.assertEqual('%s%s%s%s' % (expected_chunk, expected_chunk,
@ -296,13 +330,12 @@ class TestStore(base.StoreClearingUnitTest):
self.assertEqual(image.len, reader.size)
self.assertTrue(reader.closed)
def test_reader_size(self):
def test_chunkreader_size(self):
"""Test that the image reader takes into account the specified size."""
content = 'XXX'
image = six.StringIO(content)
expected_checksum = hashlib.md5(content).hexdigest()
checksum = hashlib.md5()
reader = vm_store._Reader(image, checksum, blocksize=1)
reader = vm_store._ChunkReader(image, blocksize=1)
ret = reader.read(size=3)
self.assertEqual('1\r\n', ret)
ret = reader.read(size=1)