diff --git a/glance_store/_drivers/filesystem.py b/glance_store/_drivers/filesystem.py index 54601028..ae2b29e2 100644 --- a/glance_store/_drivers/filesystem.py +++ b/glance_store/_drivers/filesystem.py @@ -116,10 +116,12 @@ class ChunkedFile(object): something that can iterate over a large file """ - def __init__(self, filepath, offset=0, chunk_size=None, partial=False): - self.partial = partial + def __init__(self, filepath, offset=0, chunk_size=4096, + partial_length=None): self.filepath = filepath self.chunk_size = chunk_size + self.partial_length = partial_length + self.partial = self.partial_length is not None self.fp = open(self.filepath, 'rb') if offset: self.fp.seek(offset) @@ -129,12 +131,19 @@ class ChunkedFile(object): try: if self.fp: while True: - chunk = self.fp.read(self.chunk_size) + if self.partial: + size = min(self.chunk_size, self.partial_length) + else: + size = self.chunk_size + + chunk = self.fp.read(size) if chunk: yield chunk if self.partial: - break + self.partial_length -= len(chunk) + if self.partial_length <= 0: + break else: break finally: @@ -451,8 +460,8 @@ class Store(glance_store.driver.Store): LOG.debug(msg) return (ChunkedFile(filepath, offset=offset, - chunk_size=chunk_size or self.READ_CHUNKSIZE, - partial=chunk_size is not None), + chunk_size=self.READ_CHUNKSIZE, + partial_length=chunk_size), chunk_size or filesize) def get_size(self, location, context=None): diff --git a/glance_store/_drivers/http.py b/glance_store/_drivers/http.py index eef578d3..a2073de2 100644 --- a/glance_store/_drivers/http.py +++ b/glance_store/_drivers/http.py @@ -133,8 +133,7 @@ class Store(glance_store.driver.Store): LOG.error(reason) raise exceptions.RemoteServiceUnavailable() - cs = chunk_size or self.READ_CHUNKSIZE - iterator = http_response_iterator(conn, resp, cs) + iterator = http_response_iterator(conn, resp, self.READ_CHUNKSIZE) class ResponseIndexable(glance_store.Indexable): def another(self): diff --git a/glance_store/_drivers/rbd.py b/glance_store/_drivers/rbd.py index 8d64e34d..c37adf89 100644 --- a/glance_store/_drivers/rbd.py +++ b/glance_store/_drivers/rbd.py @@ -221,9 +221,8 @@ class Store(driver.Store): :raises `glance_store.exceptions.NotFound` if image does not exist """ loc = location.store_location - return (ImageIterator(loc.pool, loc.image, loc.snapshot, - self, chunk_size=chunk_size), - chunk_size or self.get_size(location)) + return (ImageIterator(loc.pool, loc.image, loc.snapshot, self), + self.get_size(location)) def get_size(self, location, context=None): """ diff --git a/glance_store/_drivers/s3.py b/glance_store/_drivers/s3.py index a0b41ad2..e1d5f3d5 100644 --- a/glance_store/_drivers/s3.py +++ b/glance_store/_drivers/s3.py @@ -379,7 +379,7 @@ class Store(glance_store.driver.Store): :raises `glance_store.exceptions.NotFound` if image does not exist """ key = self._retrieve_key(location) - cs = chunk_size or self.READ_CHUNKSIZE + cs = self.READ_CHUNKSIZE key.BufferSize = cs class ChunkedIndexable(glance_store.Indexable): diff --git a/glance_store/_drivers/sheepdog.py b/glance_store/_drivers/sheepdog.py index 597f857e..18a5b297 100644 --- a/glance_store/_drivers/sheepdog.py +++ b/glance_store/_drivers/sheepdog.py @@ -226,7 +226,7 @@ class Store(glance_store.driver.Store): loc = location.store_location image = SheepdogImage(self.addr, self.port, loc.image, - chunk_size or self.READ_CHUNKSIZE) + self.READ_CHUNKSIZE) if not image.exist(): raise exceptions.NotFound(_("Sheepdog image %s does not exist") % image.name) diff --git a/tests/unit/test_http_store.py b/tests/unit/test_http_store.py index b133e946..5204f617 100644 --- a/tests/unit/test_http_store.py +++ b/tests/unit/test_http_store.py @@ -60,6 +60,12 @@ class TestHttpStore(base.StoreBaseTest, chunks = [c for c in image_file] self.assertEqual(expected_returns, chunks) + def test_http_partial_get(self): + uri = "http://netloc/path/to/file.tar.gz" + loc = location.get_location_from_uri(uri, conf=self.conf) + self.assertRaises(exceptions.StoreRandomGetNotSupported, + self.store.get, loc, chunk_size=1) + def test_http_get_redirect(self): # Add two layers of redirects to the response stack, which will # return the default 200 OK with the expected data after resolving diff --git a/tests/unit/test_rbd_store.py b/tests/unit/test_rbd_store.py index ede804a5..e6148ccf 100644 --- a/tests/unit/test_rbd_store.py +++ b/tests/unit/test_rbd_store.py @@ -174,9 +174,9 @@ class TestStore(base.StoreBaseTest, with mock.patch.object(rbd_store.rbd.Image, 'write') as write: ret = self.store.add('fake_image_id', self.data_iter, 0) - self.assertTrue(resize.called) - self.assertTrue(write.called) - self.assertEqual(ret[1], self.data_len) + self.assertTrue(resize.called) + self.assertTrue(write.called) + self.assertEqual(ret[1], self.data_len) @mock.patch.object(MockRBD.Image, '__enter__') @mock.patch.object(rbd_store.Store, '_create_image') @@ -284,6 +284,12 @@ class TestStore(base.StoreBaseTest, self.called_commands_expected = ['remove'] + def test_get_partial_image(self): + loc = Location('test_rbd_store', rbd_store.StoreLocation, self.conf, + store_specs=self.store_specs) + self.assertRaises(exceptions.StoreRandomGetNotSupported, + self.store.get, loc, chunk_size=1) + def tearDown(self): self.assertEqual(self.called_commands_actual, self.called_commands_expected) diff --git a/tests/unit/test_s3_store.py b/tests/unit/test_s3_store.py index 9345ac18..f68e409e 100644 --- a/tests/unit/test_s3_store.py +++ b/tests/unit/test_s3_store.py @@ -301,6 +301,14 @@ class TestStore(base.StoreBaseTest, data += chunk self.assertEqual(expected_data, data) + def test_partial_get(self): + """Test a "normal" retrieval of an image in chunks.""" + loc = location.get_location_from_uri( + "s3://user:key@auth_address/glance/%s" % FAKE_UUID, + conf=self.conf) + self.assertRaises(exceptions.StoreRandomGetNotSupported, + self.store.get, loc, chunk_size=1) + def test_get_calling_format_path(self): """Test a "normal" retrieval of an image in chunks.""" self.config(s3_store_bucket_url_format='path') diff --git a/tests/unit/test_sheepdog_store.py b/tests/unit/test_sheepdog_store.py index 4e4d917d..41608fbb 100644 --- a/tests/unit/test_sheepdog_store.py +++ b/tests/unit/test_sheepdog_store.py @@ -19,6 +19,8 @@ import mock from oslo_concurrency import processutils from glance_store._drivers import sheepdog +from glance_store import exceptions +from glance_store import location from glance_store.tests import base from tests.unit import test_store_capabilities @@ -53,3 +55,9 @@ class TestSheepdogStore(base.StoreBaseTest, data = StringIO.StringIO('xx') self.store.add('fake_image_id', data, 2) self.assertEqual(called_commands, ['list -r', 'create', 'write']) + + def test_partial_get(self): + loc = location.Location('test_sheepdog_store', sheepdog.StoreLocation, + self.conf, store_specs={'image': 'fake_image'}) + self.assertRaises(exceptions.StoreRandomGetNotSupported, + self.store.get, loc, chunk_size=1)