Split CHUNKSIZE into WRITE/READ_CHUNKSIZE
See I4e0c563b8f3a5ced8f65fcca83d341a97729a5d4 for a detailed explanation. Change-Id: I7d366f324b2699be0be610c1dde1db7fc4045dfd
This commit is contained in:
parent
732d5e0251
commit
7b858be0f9
@ -88,10 +88,9 @@ class ChunkedFile(object):
|
||||
something that can iterate over a large file
|
||||
"""
|
||||
|
||||
CHUNKSIZE = 65536
|
||||
|
||||
def __init__(self, filepath):
|
||||
def __init__(self, filepath, chunk_size=None):
|
||||
self.filepath = filepath
|
||||
self.chunk_size = chunk_size
|
||||
self.fp = open(self.filepath, 'rb')
|
||||
|
||||
def __iter__(self):
|
||||
@ -99,7 +98,7 @@ class ChunkedFile(object):
|
||||
try:
|
||||
if self.fp:
|
||||
while True:
|
||||
chunk = self.fp.read(ChunkedFile.CHUNKSIZE)
|
||||
chunk = self.fp.read(self.chunk_size)
|
||||
if chunk:
|
||||
yield chunk
|
||||
else:
|
||||
@ -117,6 +116,8 @@ class ChunkedFile(object):
|
||||
class Store(glance.store.driver.Store):
|
||||
|
||||
OPTIONS = _FILESYSTEM_CONFIGS
|
||||
READ_CHUNKSIZE = 64 * units.Ki
|
||||
WRITE_CHUNKSIZE = READ_CHUNKSIZE
|
||||
|
||||
def get_schemes(self):
|
||||
return ('file', 'filesystem')
|
||||
@ -313,7 +314,9 @@ class Store(glance.store.driver.Store):
|
||||
filepath, filesize = self._resolve_location(location)
|
||||
msg = _("Found image at %s. Returning in ChunkedFile.") % filepath
|
||||
LOG.debug(msg)
|
||||
return (ChunkedFile(filepath), filesize)
|
||||
return (ChunkedFile(filepath,
|
||||
chunk_size=chunk_size or self.READ_CHUNKSIZE),
|
||||
filesize)
|
||||
|
||||
def get_size(self, location, context=None):
|
||||
"""
|
||||
@ -434,7 +437,7 @@ class Store(glance.store.driver.Store):
|
||||
try:
|
||||
with open(filepath, 'wb') as f:
|
||||
for buf in utils.chunkreadable(image_file,
|
||||
ChunkedFile.CHUNKSIZE):
|
||||
self.WRITE_CHUNKSIZE):
|
||||
bytes_written += len(buf)
|
||||
checksum.update(buf)
|
||||
f.write(buf)
|
||||
|
@ -121,7 +121,8 @@ class Store(glance.store.driver.Store):
|
||||
"""
|
||||
conn, resp, content_length = self._query(location, 'GET')
|
||||
|
||||
iterator = http_response_iterator(conn, resp, self.CHUNKSIZE)
|
||||
cs = chunk_size or self.READ_CHUNKSIZE
|
||||
iterator = http_response_iterator(conn, resp, cs)
|
||||
|
||||
class ResponseIndexable(glance.store.Indexable):
|
||||
def another(self):
|
||||
|
@ -141,12 +141,12 @@ class ImageIterator(object):
|
||||
Reads data from an RBD image, one chunk at a time.
|
||||
"""
|
||||
|
||||
def __init__(self, name, store):
|
||||
def __init__(self, name, store, chunk_size=None):
|
||||
self.name = name
|
||||
self.pool = store.pool
|
||||
self.user = store.user
|
||||
self.conf_file = store.conf_file
|
||||
self.chunk_size = store.chunk_size
|
||||
self.chunk_size = chunk_size or store.chunk_size
|
||||
|
||||
def __iter__(self):
|
||||
try:
|
||||
@ -188,6 +188,8 @@ class Store(driver.Store):
|
||||
try:
|
||||
chunk = self.conf.glance_store.rbd_store_chunk_size
|
||||
self.chunk_size = chunk * (1024 ^ 2)
|
||||
self.READ_CHUNKSIZE = self.chunk_size
|
||||
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
|
||||
|
||||
# these must not be unicode since they will be passed to a
|
||||
# non-unicode-aware C library
|
||||
@ -211,7 +213,8 @@ class Store(driver.Store):
|
||||
:raises `glance.store.exceptions.NotFound` if image does not exist
|
||||
"""
|
||||
loc = location.store_location
|
||||
return (ImageIterator(loc.image, self), self.get_size(location))
|
||||
return (ImageIterator(loc.image, self),
|
||||
self.get_size(location), chunk_size)
|
||||
|
||||
def get_size(self, location, context=None):
|
||||
"""
|
||||
@ -323,7 +326,7 @@ class Store(driver.Store):
|
||||
if hasattr(conn, 'get_fsid'):
|
||||
fsid = conn.get_fsid()
|
||||
with conn.open_ioctx(self.pool) as ioctx:
|
||||
order = int(math.log(self.chunk_size, 2))
|
||||
order = int(math.log(self.WRITE_CHUNKSIZE, 2))
|
||||
LOG.debug('creating image %s with order %d and size %d',
|
||||
image_name, order, image_size)
|
||||
if image_size == 0:
|
||||
@ -343,7 +346,7 @@ class Store(driver.Store):
|
||||
bytes_written = 0
|
||||
offset = 0
|
||||
chunks = utils.chunkreadable(image_file,
|
||||
self.chunk_size)
|
||||
self.WRITE_CHUNKSIZE)
|
||||
for chunk in chunks:
|
||||
# If the image size provided is zero we need to do
|
||||
# a resize for the amount we are writing. This will
|
||||
|
@ -30,6 +30,7 @@ import glance.store.driver
|
||||
from glance.store import exceptions
|
||||
from glance.store.i18n import _
|
||||
import glance.store.location
|
||||
from glance.store.openstack.common import units
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
@ -173,17 +174,16 @@ class ChunkedFile(object):
|
||||
something that can iterate over a ``boto.s3.key.Key``
|
||||
"""
|
||||
|
||||
CHUNKSIZE = 65536
|
||||
|
||||
def __init__(self, fp):
|
||||
def __init__(self, fp, chunk_size):
|
||||
self.fp = fp
|
||||
self.chunk_size = chunk_size
|
||||
|
||||
def __iter__(self):
|
||||
"""Return an iterator over the image file"""
|
||||
try:
|
||||
if self.fp:
|
||||
while True:
|
||||
chunk = self.fp.read(ChunkedFile.CHUNKSIZE)
|
||||
chunk = self.fp.read(self.chunk_size)
|
||||
if chunk:
|
||||
yield chunk
|
||||
else:
|
||||
@ -214,6 +214,9 @@ class Store(glance.store.driver.Store):
|
||||
OPTIONS = _S3_OPTS
|
||||
EXAMPLE_URL = "s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ>"
|
||||
|
||||
READ_CHUNKSIZE = 64 * units.Ki
|
||||
WRITE_CHUNKSIZE = READ_CHUNKSIZE
|
||||
|
||||
def get_schemes(self):
|
||||
return ('s3', 's3+http', 's3+https')
|
||||
|
||||
@ -268,15 +271,15 @@ class Store(glance.store.driver.Store):
|
||||
:raises `glance.store.exceptions.NotFound` if image does not exist
|
||||
"""
|
||||
key = self._retrieve_key(location)
|
||||
|
||||
key.BufferSize = self.CHUNKSIZE
|
||||
cs = chunk_size or self.READ_CHUNKSIZE
|
||||
key.BufferSize = cs
|
||||
|
||||
class ChunkedIndexable(glance.store.Indexable):
|
||||
def another(self):
|
||||
return (self.wrapped.fp.read(ChunkedFile.CHUNKSIZE)
|
||||
return (self.wrapped.fp.read(cs)
|
||||
if self.wrapped.fp else None)
|
||||
|
||||
return (ChunkedIndexable(ChunkedFile(key), key.size), key.size)
|
||||
return (ChunkedIndexable(ChunkedFile(key, cs), key.size), key.size)
|
||||
|
||||
def get_size(self, location, context=None):
|
||||
"""
|
||||
@ -404,7 +407,7 @@ class Store(glance.store.driver.Store):
|
||||
tmpdir = self.s3_store_object_buffer_dir
|
||||
temp_file = tempfile.NamedTemporaryFile(dir=tmpdir)
|
||||
checksum = hashlib.md5()
|
||||
for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE):
|
||||
for chunk in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE):
|
||||
checksum.update(chunk)
|
||||
temp_file.write(chunk)
|
||||
temp_file.flush()
|
||||
|
@ -192,6 +192,9 @@ class Store(glance.store.driver.Store):
|
||||
|
||||
try:
|
||||
self.chunk_size = CONF.sheepdog_store_chunk_size * units.Mi
|
||||
self.READ_CHUNKSIZE = self.chunk_size
|
||||
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
|
||||
|
||||
self.addr = CONF.sheepdog_store_address
|
||||
self.port = CONF.sheepdog_store_port
|
||||
except cfg.ConfigFileValueError as e:
|
||||
@ -208,7 +211,7 @@ class Store(glance.store.driver.Store):
|
||||
raise exceptions.BadStoreConfiguration(store_name='sheepdog',
|
||||
reason=reason)
|
||||
|
||||
def get(self, location):
|
||||
def get(self, location, offset=0, chunk_size=None, context=None):
|
||||
"""
|
||||
Takes a `glance.store.location.Location` object that indicates
|
||||
where to find the image file, and returns a generator for reading
|
||||
@ -221,13 +224,13 @@ class Store(glance.store.driver.Store):
|
||||
|
||||
loc = location.store_location
|
||||
image = SheepdogImage(self.addr, self.port, loc.image,
|
||||
self.chunk_size)
|
||||
chunk_size or self.READ_CHUNKSIZE)
|
||||
if not image.exist():
|
||||
raise exceptions.NotFound(_("Sheepdog image %s does not exist")
|
||||
% image.name)
|
||||
return (ImageIterator(image), image.get_size())
|
||||
|
||||
def get_size(self, location):
|
||||
def get_size(self, location, context=None):
|
||||
"""
|
||||
Takes a `glance.store.location.Location` object that indicates
|
||||
where to find the image file and returns the image size
|
||||
@ -240,13 +243,13 @@ class Store(glance.store.driver.Store):
|
||||
|
||||
loc = location.store_location
|
||||
image = SheepdogImage(self.addr, self.port, loc.image,
|
||||
self.chunk_size)
|
||||
self.READ_CHUNKSIZE)
|
||||
if not image.exist():
|
||||
raise exceptions.NotFound(_("Sheepdog image %s does not exist")
|
||||
% image.name)
|
||||
return image.get_size()
|
||||
|
||||
def add(self, image_id, image_file, image_size):
|
||||
def add(self, image_id, image_file, image_size, context=None):
|
||||
"""
|
||||
Stores an image file with supplied identifier to the backend
|
||||
storage system and returns a tuple containing information
|
||||
@ -262,7 +265,7 @@ class Store(glance.store.driver.Store):
|
||||
"""
|
||||
|
||||
image = SheepdogImage(self.addr, self.port, image_id,
|
||||
self.chunk_size)
|
||||
self.WRITE_CHUNKSIZE)
|
||||
if image.exist():
|
||||
raise exceptions.Duplicate(_("Sheepdog image %s already exists")
|
||||
% image_id)
|
||||
@ -288,7 +291,7 @@ class Store(glance.store.driver.Store):
|
||||
|
||||
return (location.get_uri(), image_size, checksum.hexdigest(), {})
|
||||
|
||||
def delete(self, location):
|
||||
def delete(self, location, context=None):
|
||||
"""
|
||||
Takes a `glance.store.location.Location` object that indicates
|
||||
where to find the image file to delete
|
||||
@ -301,7 +304,7 @@ class Store(glance.store.driver.Store):
|
||||
|
||||
loc = location.store_location
|
||||
image = SheepdogImage(self.addr, self.port, loc.image,
|
||||
self.chunk_size)
|
||||
self.WRITE_CHUNKSIZE)
|
||||
if not image.exist():
|
||||
raise exceptions.NotFound(_("Sheepdog image %s does not exist") %
|
||||
loc.image)
|
||||
|
@ -43,7 +43,8 @@ def _exception_to_unicode(exc):
|
||||
class Store(object):
|
||||
|
||||
OPTIONS = None
|
||||
CHUNKSIZE = 16 * (1024 * 1024) # 16M
|
||||
READ_CHUNKSIZE = 16 * (1024 * 1024) # 16M
|
||||
WRITE_CHUNKSIZE = READ_CHUNKSIZE
|
||||
|
||||
def __init__(self, conf):
|
||||
"""
|
||||
|
@ -43,8 +43,8 @@ class TestStore(base.StoreBaseTest):
|
||||
def setUp(self):
|
||||
"""Establish a clean test environment."""
|
||||
super(TestStore, self).setUp()
|
||||
self.orig_chunksize = ChunkedFile.CHUNKSIZE
|
||||
ChunkedFile.CHUNKSIZE = 10
|
||||
self.orig_chunksize = Store.READ_CHUNKSIZE
|
||||
Store.READ_CHUNKSIZE = 10
|
||||
self.store = Store(self.conf)
|
||||
self.config(filesystem_store_datadir=self.test_dir,
|
||||
group="glance_store")
|
||||
|
@ -29,7 +29,7 @@ class TestHttpStore(base.StoreBaseTest):
|
||||
def setUp(self):
|
||||
super(TestHttpStore, self).setUp()
|
||||
self.config(default_store='http', group='glance_store')
|
||||
http.Store.CHUNKSIZE = 2
|
||||
http.Store.READ_CHUNKSIZE = 2
|
||||
self.store = http.Store(self.conf)
|
||||
|
||||
response = mock.patch('httplib.HTTPConnection.getresponse')
|
||||
|
Loading…
Reference in New Issue
Block a user