openstack
/
glance_store
Code Issues Proposed changes

Split CHUNKSIZE into WRITE/READ_CHUNKSIZE 

See I4e0c563b8f3a5ced8f65fcca83d341a97729a5d4 for a detailed
explanation.

Change-Id: I7d366f324b2699be0be610c1dde1db7fc4045dfd
This commit is contained in:
Flavio Percoco 2014-07-29 16:40:06 +02:00 committed by Flavio Percoco
parent 732d5e0251
commit 7b858be0f9
8 changed files with 47 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
glance/store/_drivers/filesystem.py
View File

@ -88,10 +88,9 @@ class ChunkedFile(object):
something that can iterate over a large file
"""
CHUNKSIZE = 65536
def __init__(self, filepath):
def __init__(self, filepath, chunk_size=None):
self.filepath = filepath
self.chunk_size = chunk_size
self.fp = open(self.filepath, 'rb')
def __iter__(self):
@ -99,7 +98,7 @@ class ChunkedFile(object):
try:
if self.fp:
while True:
chunk = self.fp.read(ChunkedFile.CHUNKSIZE)
chunk = self.fp.read(self.chunk_size)
if chunk:
yield chunk
else:
@ -117,6 +116,8 @@ class ChunkedFile(object):
class Store(glance.store.driver.Store):
OPTIONS = _FILESYSTEM_CONFIGS
READ_CHUNKSIZE = 64 * units.Ki
WRITE_CHUNKSIZE = READ_CHUNKSIZE
def get_schemes(self):
return ('file', 'filesystem')
@ -313,7 +314,9 @@ class Store(glance.store.driver.Store):
filepath, filesize = self._resolve_location(location)
msg = _("Found image at %s. Returning in ChunkedFile.") % filepath
LOG.debug(msg)
return (ChunkedFile(filepath), filesize)
return (ChunkedFile(filepath,
chunk_size=chunk_size or self.READ_CHUNKSIZE),
filesize)
def get_size(self, location, context=None):
"""
@ -434,7 +437,7 @@ class Store(glance.store.driver.Store):
try:
with open(filepath, 'wb') as f:
for buf in utils.chunkreadable(image_file,
ChunkedFile.CHUNKSIZE):
self.WRITE_CHUNKSIZE):
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)

3
glance/store/_drivers/http.py
View File

@ -121,7 +121,8 @@ class Store(glance.store.driver.Store):
"""
conn, resp, content_length = self._query(location, 'GET')
iterator = http_response_iterator(conn, resp, self.CHUNKSIZE)
cs = chunk_size or self.READ_CHUNKSIZE
iterator = http_response_iterator(conn, resp, cs)
class ResponseIndexable(glance.store.Indexable):
def another(self):

13
glance/store/_drivers/rbd.py
View File

@ -141,12 +141,12 @@ class ImageIterator(object):
Reads data from an RBD image, one chunk at a time.
"""
def __init__(self, name, store):
def __init__(self, name, store, chunk_size=None):
self.name = name
self.pool = store.pool
self.user = store.user
self.conf_file = store.conf_file
self.chunk_size = store.chunk_size
self.chunk_size = chunk_size or store.chunk_size
def __iter__(self):
try:
@ -188,6 +188,8 @@ class Store(driver.Store):
try:
chunk = self.conf.glance_store.rbd_store_chunk_size
self.chunk_size = chunk * (1024 ^ 2)
self.READ_CHUNKSIZE = self.chunk_size
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
# these must not be unicode since they will be passed to a
# non-unicode-aware C library
@ -211,7 +213,8 @@ class Store(driver.Store):
:raises `glance.store.exceptions.NotFound` if image does not exist
"""
loc = location.store_location
return (ImageIterator(loc.image, self), self.get_size(location))
return (ImageIterator(loc.image, self),
self.get_size(location), chunk_size)
def get_size(self, location, context=None):
"""
@ -323,7 +326,7 @@ class Store(driver.Store):
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
order = int(math.log(self.WRITE_CHUNKSIZE, 2))
LOG.debug('creating image %s with order %d and size %d',
image_name, order, image_size)
if image_size == 0:
@ -343,7 +346,7 @@ class Store(driver.Store):
bytes_written = 0
offset = 0
chunks = utils.chunkreadable(image_file,
self.chunk_size)
self.WRITE_CHUNKSIZE)
for chunk in chunks:
# If the image size provided is zero we need to do
# a resize for the amount we are writing. This will

21
glance/store/_drivers/s3.py
View File

@ -30,6 +30,7 @@ import glance.store.driver
from glance.store import exceptions
from glance.store.i18n import _
import glance.store.location
from glance.store.openstack.common import units
LOG = logging.getLogger(__name__)
@ -173,17 +174,16 @@ class ChunkedFile(object):
something that can iterate over a ``boto.s3.key.Key``
"""
CHUNKSIZE = 65536
def __init__(self, fp):
def __init__(self, fp, chunk_size):
self.fp = fp
self.chunk_size = chunk_size
def __iter__(self):
"""Return an iterator over the image file"""
try:
if self.fp:
while True:
chunk = self.fp.read(ChunkedFile.CHUNKSIZE)
chunk = self.fp.read(self.chunk_size)
if chunk:
yield chunk
else:
@ -214,6 +214,9 @@ class Store(glance.store.driver.Store):
OPTIONS = _S3_OPTS
EXAMPLE_URL = "s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ>"
READ_CHUNKSIZE = 64 * units.Ki
WRITE_CHUNKSIZE = READ_CHUNKSIZE
def get_schemes(self):
return ('s3', 's3+http', 's3+https')
@ -268,15 +271,15 @@ class Store(glance.store.driver.Store):
:raises `glance.store.exceptions.NotFound` if image does not exist
"""
key = self._retrieve_key(location)
key.BufferSize = self.CHUNKSIZE
cs = chunk_size or self.READ_CHUNKSIZE
key.BufferSize = cs
class ChunkedIndexable(glance.store.Indexable):
def another(self):
return (self.wrapped.fp.read(ChunkedFile.CHUNKSIZE)
return (self.wrapped.fp.read(cs)
if self.wrapped.fp else None)
return (ChunkedIndexable(ChunkedFile(key), key.size), key.size)
return (ChunkedIndexable(ChunkedFile(key, cs), key.size), key.size)
def get_size(self, location, context=None):
"""
@ -404,7 +407,7 @@ class Store(glance.store.driver.Store):
tmpdir = self.s3_store_object_buffer_dir
temp_file = tempfile.NamedTemporaryFile(dir=tmpdir)
checksum = hashlib.md5()
for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE):
for chunk in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE):
checksum.update(chunk)
temp_file.write(chunk)
temp_file.flush()

19
glance/store/_drivers/sheepdog.py
View File

@ -192,6 +192,9 @@ class Store(glance.store.driver.Store):
try:
self.chunk_size = CONF.sheepdog_store_chunk_size * units.Mi
self.READ_CHUNKSIZE = self.chunk_size
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
self.addr = CONF.sheepdog_store_address
self.port = CONF.sheepdog_store_port
except cfg.ConfigFileValueError as e:
@ -208,7 +211,7 @@ class Store(glance.store.driver.Store):
raise exceptions.BadStoreConfiguration(store_name='sheepdog',
reason=reason)
def get(self, location):
def get(self, location, offset=0, chunk_size=None, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
@ -221,13 +224,13 @@ class Store(glance.store.driver.Store):
loc = location.store_location
image = SheepdogImage(self.addr, self.port, loc.image,
self.chunk_size)
chunk_size or self.READ_CHUNKSIZE)
if not image.exist():
raise exceptions.NotFound(_("Sheepdog image %s does not exist")
% image.name)
return (ImageIterator(image), image.get_size())
def get_size(self, location):
def get_size(self, location, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file and returns the image size
@ -240,13 +243,13 @@ class Store(glance.store.driver.Store):
loc = location.store_location
image = SheepdogImage(self.addr, self.port, loc.image,
self.chunk_size)
self.READ_CHUNKSIZE)
if not image.exist():
raise exceptions.NotFound(_("Sheepdog image %s does not exist")
% image.name)
return image.get_size()
def add(self, image_id, image_file, image_size):
def add(self, image_id, image_file, image_size, context=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
@ -262,7 +265,7 @@ class Store(glance.store.driver.Store):
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.chunk_size)
self.WRITE_CHUNKSIZE)
if image.exist():
raise exceptions.Duplicate(_("Sheepdog image %s already exists")
% image_id)
@ -288,7 +291,7 @@ class Store(glance.store.driver.Store):
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def delete(self, location):
def delete(self, location, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
@ -301,7 +304,7 @@ class Store(glance.store.driver.Store):
loc = location.store_location
image = SheepdogImage(self.addr, self.port, loc.image,
self.chunk_size)
self.WRITE_CHUNKSIZE)
if not image.exist():
raise exceptions.NotFound(_("Sheepdog image %s does not exist") %
loc.image)

3
glance/store/driver.py
View File

@ -43,7 +43,8 @@ def _exception_to_unicode(exc):
class Store(object):
OPTIONS = None
CHUNKSIZE = 16 * (1024 * 1024) # 16M
READ_CHUNKSIZE = 16 * (1024 * 1024) # 16M
WRITE_CHUNKSIZE = READ_CHUNKSIZE
def __init__(self, conf):
"""

4
tests/unit/test_filesystem_store.py
View File

@ -43,8 +43,8 @@ class TestStore(base.StoreBaseTest):
def setUp(self):
"""Establish a clean test environment."""
super(TestStore, self).setUp()
self.orig_chunksize = ChunkedFile.CHUNKSIZE
ChunkedFile.CHUNKSIZE = 10
self.orig_chunksize = Store.READ_CHUNKSIZE
Store.READ_CHUNKSIZE = 10
self.store = Store(self.conf)
self.config(filesystem_store_datadir=self.test_dir,
group="glance_store")

2
tests/unit/test_http_store.py
View File

@ -29,7 +29,7 @@ class TestHttpStore(base.StoreBaseTest):
def setUp(self):
super(TestHttpStore, self).setUp()
self.config(default_store='http', group='glance_store')
http.Store.CHUNKSIZE = 2
http.Store.READ_CHUNKSIZE = 2
self.store = http.Store(self.conf)
response = mock.patch('httplib.HTTPConnection.getresponse')