openstack/glance_store
Code Issues Proposed changes

Correct such logic in store.get() when chunk_size param provided

Browse Source 
The change corrected the logic of image content reading function in
get() for some store drivers. if the chunk_size param provided from
outside, it means client want to read partial image content from
storage, current implementation is incomplete and has some problem.

The change didn't try to make drivers fully support random access from
backend with chunk_size (and offset) param, that stuff will be
implementated by a dedicated patch, this one only to make default logic
be correct.

Change-Id: I76fd3746a2c449e257ea2a19c2dc6118be207eec
Signed-off-by: Zhi Yan Liu <zhiyanl@cn.ibm.com>
This commit is contained in:
Zhi Yan Liu 2014-11-21 00:18:39 +08:00 committed by Zhi Yan Liu
parent c17d64563c
commit 89e88841fa
9 changed files with 51 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
glance_store/_drivers/filesystem.py
View File

@ -116,10 +116,12 @@ class ChunkedFile(object):
something that can iterate over a large file
"""
def __init__(self, filepath, offset=0, chunk_size=None, partial=False):
self.partial = partial
def __init__(self, filepath, offset=0, chunk_size=4096,
partial_length=None):
self.filepath = filepath
self.chunk_size = chunk_size
self.partial_length = partial_length
self.partial = self.partial_length is not None
self.fp = open(self.filepath, 'rb')
if offset:
self.fp.seek(offset)
@ -129,11 +131,18 @@ class ChunkedFile(object):
try:
if self.fp:
while True:
chunk = self.fp.read(self.chunk_size)
if self.partial:
size = min(self.chunk_size, self.partial_length)
else:
size = self.chunk_size
chunk = self.fp.read(size)
if chunk:
yield chunk
if self.partial:
self.partial_length -= len(chunk)
if self.partial_length <= 0:
break
else:
break
@ -451,8 +460,8 @@ class Store(glance_store.driver.Store):
LOG.debug(msg)
return (ChunkedFile(filepath,
offset=offset,
chunk_size=chunk_size or self.READ_CHUNKSIZE,
partial=chunk_size is not None),
chunk_size=self.READ_CHUNKSIZE,
partial_length=chunk_size),
chunk_size or filesize)
def get_size(self, location, context=None):

3
glance_store/_drivers/http.py
View File

@ -133,8 +133,7 @@ class Store(glance_store.driver.Store):
LOG.error(reason)
raise exceptions.RemoteServiceUnavailable()
cs = chunk_size or self.READ_CHUNKSIZE
iterator = http_response_iterator(conn, resp, cs)
iterator = http_response_iterator(conn, resp, self.READ_CHUNKSIZE)
class ResponseIndexable(glance_store.Indexable):
def another(self):

5
glance_store/_drivers/rbd.py
View File

@ -220,9 +220,8 @@ class Store(driver.Store):
:raises `glance_store.exceptions.NotFound` if image does not exist
"""
loc = location.store_location
return (ImageIterator(loc.pool, loc.image, loc.snapshot,
self, chunk_size=chunk_size),
chunk_size or self.get_size(location))
return (ImageIterator(loc.pool, loc.image, loc.snapshot, self),
self.get_size(location))
def get_size(self, location, context=None):
"""

2
glance_store/_drivers/s3.py
View File

@ -378,7 +378,7 @@ class Store(glance_store.driver.Store):
:raises `glance_store.exceptions.NotFound` if image does not exist
"""
key = self._retrieve_key(location)
cs = chunk_size or self.READ_CHUNKSIZE
cs = self.READ_CHUNKSIZE
key.BufferSize = cs
class ChunkedIndexable(glance_store.Indexable):

2
glance_store/_drivers/sheepdog.py
View File

@ -226,7 +226,7 @@ class Store(glance_store.driver.Store):
loc = location.store_location
image = SheepdogImage(self.addr, self.port, loc.image,
chunk_size or self.READ_CHUNKSIZE)
self.READ_CHUNKSIZE)
if not image.exist():
raise exceptions.NotFound(_("Sheepdog image %s does not exist")
% image.name)

6
tests/unit/test_http_store.py
View File

@ -60,6 +60,12 @@ class TestHttpStore(base.StoreBaseTest,
chunks = [c for c in image_file]
self.assertEqual(expected_returns, chunks)
def test_http_partial_get(self):
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.StoreRandomGetNotSupported,
self.store.get, loc, chunk_size=1)
def test_http_get_redirect(self):
# Add two layers of redirects to the response stack, which will
# return the default 200 OK with the expected data after resolving

6
tests/unit/test_rbd_store.py
View File

@ -283,6 +283,12 @@ class TestStore(base.StoreBaseTest,
self.called_commands_expected = ['remove']
def test_get_partial_image(self):
loc = Location('test_rbd_store', rbd_store.StoreLocation, self.conf,
store_specs=self.store_specs)
self.assertRaises(exceptions.StoreRandomGetNotSupported,
self.store.get, loc, chunk_size=1)
def tearDown(self):
self.assertEqual(self.called_commands_actual,
self.called_commands_expected)

8
tests/unit/test_s3_store.py
View File

@ -301,6 +301,14 @@ class TestStore(base.StoreBaseTest,
data += chunk
self.assertEqual(expected_data, data)
def test_partial_get(self):
"""Test a "normal" retrieval of an image in chunks."""
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
self.assertRaises(exceptions.StoreRandomGetNotSupported,
self.store.get, loc, chunk_size=1)
def test_get_calling_format_path(self):
"""Test a "normal" retrieval of an image in chunks."""
self.config(s3_store_bucket_url_format='path')

8
tests/unit/test_sheepdog_store.py
View File

@ -19,6 +19,8 @@ import mock
from oslo_concurrency import processutils
from glance_store._drivers import sheepdog
from glance_store import exceptions
from glance_store import location
from glance_store.tests import base
from tests.unit import test_store_capabilities
@ -53,3 +55,9 @@ class TestSheepdogStore(base.StoreBaseTest,
data = StringIO.StringIO('xx')
self.store.add('fake_image_id', data, 2)
self.assertEqual(called_commands, ['list -r', 'create', 'write'])
def test_partial_get(self):
loc = location.Location('test_sheepdog_store', sheepdog.StoreLocation,
self.conf, store_specs={'image': 'fake_image'})
self.assertRaises(exceptions.StoreRandomGetNotSupported,
self.store.get, loc, chunk_size=1)