openstack/glance_store
Code Issues Proposed changes

Handle sparse images in glance_store

Browse Source 
Add new configuration option ``rbd_thin_provisioning`` and
``filesystem_thin_provisioning`` to rbd and filesystem
store to enable or not sparse upload.

A sparse file means that we do not actually write null byte
sequences but only the data itself at a given offset, the
"holes" which can appear will automatically be interpreted by
the storage backend as null bytes, and do not really consume
your storage.

Change-Id: I129e30f490e3920e9093c2b793f89b70ce310a50
Co-Authored-By: Grégoire Unbekandt <gregoire.unbekandt@gmail.com>
Partially Implements: blueprint handle-sparse-image
This commit is contained in:
Victor Coutellier 2020-05-29 13:54:26 +02:00 committed by Dan Smith
parent c43f19e845
commit 201d85b4ea
7 changed files with 243 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
glance_store/_drivers/filesystem.py
View File

@ -165,7 +165,29 @@ Possible Values:
Related options:
* None
""")]
"""),
cfg.BoolOpt('filesystem_thin_provisioning',
default=False,
help="""
Enable or not thin provisioning in this backend.
This configuration option enable the feature of not really write null byte
sequences on the filesystem, the holes who can appear will automatically
be interpreted by the filesystem as null bytes, and do not really consume
your storage.
Enabling this feature will also speed up image upload and save network trafic
in addition to save space in the backend, as null bytes sequences are not
sent over the network.
Possible Values:
* True
* False
Related options:
* None
"""),
]
MULTI_FILESYSTEM_METADATA_SCHEMA = {
"type": "array",
@ -408,6 +430,8 @@ class Store(glance_store.driver.Store):
fstore_perm = store_conf.filesystem_store_file_perm
meta_file = store_conf.filesystem_store_metadata_file
self.thin_provisioning = store_conf.\
filesystem_thin_provisioning
self.chunk_size = store_conf.filesystem_store_chunk_size
self.READ_CHUNKSIZE = self.chunk_size
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
@ -725,7 +749,11 @@ class Store(glance_store.driver.Store):
checksum.update(buf)
if verifier:
verifier.update(buf)
f.write(buf)
if self.thin_provisioning and not any(buf):
f.truncate(bytes_written)
f.seek(0, os.SEEK_END)
else:
f.write(buf)
except IOError as e:
if e.errno != errno.EACCES:
self._delete_partial(filepath, image_id)

32
glance_store/_drivers/rbd.py
View File

@ -150,6 +150,26 @@ Possible Values:
Related options:
* None
"""),
cfg.BoolOpt('rbd_thin_provisioning',
default=False,
help="""
Enable or not thin provisioning in this backend.
This configuration option enable the feature of not really write null byte
sequences on the RBD backend, the holes who can appear will automatically
be interpreted by Ceph as null bytes, and do not really consume your storage.
Enabling this feature will also speed up image upload and save network trafic
in addition to save space in the backend, as null bytes sequences are not
sent over the network.
Possible Values:
* True
* False
Related options:
* None
"""),
]
@ -302,13 +322,19 @@ class Store(driver.Store):
self.backend_group).rbd_store_ceph_conf
connect_timeout = getattr(
self.conf, self.backend_group).rados_connect_timeout
thin_provisioning = getattr(self.conf,
self.backend_group).\
rbd_thin_provisioning
else:
chunk = self.conf.glance_store.rbd_store_chunk_size
pool = self.conf.glance_store.rbd_store_pool
user = self.conf.glance_store.rbd_store_user
conf_file = self.conf.glance_store.rbd_store_ceph_conf
connect_timeout = self.conf.glance_store.rados_connect_timeout
thin_provisioning = \
self.conf.glance_store.rbd_thin_provisioning
self.thin_provisioning = thin_provisioning
self.chunk_size = chunk * units.Mi
self.READ_CHUNKSIZE = self.chunk_size
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
@ -555,10 +581,10 @@ class Store(driver.Store):
image_size,
bytes_written,
chunk_length)
LOG.debug(_("writing chunk at offset %s") %
(offset))
offset += image.write(chunk, offset)
bytes_written += chunk_length
if not (self.thin_provisioning and not any(chunk)):
image.write(chunk, offset)
offset += chunk_length
os_hash_value.update(chunk)
checksum.update(chunk)
if verifier:

5
glance_store/multi_backend.py
View File

@ -143,7 +143,10 @@ def register_store_opts(conf, reserved_stores=None):
cfg.IntOpt('filesystem_store_chunk_size',
default=64 * units.Ki,
min=1,
help=FS_CONF_CHUNKSIZE_HELP.format(key))]
help=FS_CONF_CHUNKSIZE_HELP.format(key)),
cfg.BoolOpt('filesystem_thin_provisioning',
default=False,
help="""Not used""")]
LOG.debug("Registering options for reserved store: {}".format(key))
conf.register_opts(fs_conf_template, group=key)

87
glance_store/tests/unit/test_filesystem_store.py
View File

@ -143,8 +143,13 @@ class TestStore(base.StoreBaseTest,
self.store.get,
loc)
def test_add(self):
def _do_test_add(self, enable_thin_provisoning):
"""Test that we can add an image via the filesystem backend."""
self.config(filesystem_store_chunk_size=units.Ki,
filesystem_thin_provisioning=enable_thin_provisoning,
group='glance_store')
self.store.configure()
filesystem.ChunkedFile.CHUNKSIZE = units.Ki
expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K
@ -176,6 +181,86 @@ class TestStore(base.StoreBaseTest,
self.assertEqual(expected_file_contents, new_image_contents)
self.assertEqual(expected_file_size, new_image_file_size)
def test_thin_provisioning_is_disabled_by_default(self):
self.assertEqual(self.store.thin_provisioning, False)
def test_add_with_thick_provisioning(self):
self._do_test_add(enable_thin_provisoning=False)
def test_add_with_thin_provisioning(self):
self._do_test_add(enable_thin_provisoning=True)
def test_add_thick_provisioning_with_holes_in_file(self):
"""
Tests that a file which contains null bytes chunks is fully
written with a thick provisioning configuration.
"""
chunk_size = units.Ki # 1K
content = b"*" * chunk_size + b"\x00" * chunk_size + b"*" * chunk_size
self._do_test_thin_provisioning(content, 3 * chunk_size, 0, 3, False)
def test_add_thin_provisioning_with_holes_in_file(self):
"""
Tests that a file which contains null bytes chunks is sparsified
with a thin provisioning configuration.
"""
chunk_size = units.Ki # 1K
content = b"*" * chunk_size + b"\x00" * chunk_size + b"*" * chunk_size
self._do_test_thin_provisioning(content, 3 * chunk_size, 1, 2, True)
def test_add_thick_provisioning_without_holes_in_file(self):
"""
Tests that a file which not contain null bytes chunks is fully
written with a thick provisioning configuration.
"""
chunk_size = units.Ki # 1K
content = b"*" * 3 * chunk_size
self._do_test_thin_provisioning(content, 3 * chunk_size, 0, 3, False)
def test_add_thin_provisioning_without_holes_in_file(self):
"""
Tests that a file which not contain null bytes chunks is fully
written with a thin provisioning configuration.
"""
chunk_size = units.Ki # 1K
content = b"*" * 3 * chunk_size
self._do_test_thin_provisioning(content, 3 * chunk_size, 0, 3, True)
def test_add_thick_provisioning_with_partial_holes_in_file(self):
"""
Tests that a file which contains null bytes not aligned with
chunk size is fully written with a thick provisioning configuration.
"""
chunk_size = units.Ki # 1K
my_chunk = int(chunk_size * 1.5)
content = b"*" * my_chunk + b"\x00" * my_chunk + b"*" * my_chunk
self._do_test_thin_provisioning(content, 3 * my_chunk, 0, 5, False)
def test_add_thin_provisioning_with_partial_holes_in_file(self):
"""
Tests that a file which contains null bytes not aligned with
chunk size is sparsified with a thin provisioning configuration.
"""
chunk_size = units.Ki # 1K
my_chunk = int(chunk_size * 1.5)
content = b"*" * my_chunk + b"\x00" * my_chunk + b"*" * my_chunk
self._do_test_thin_provisioning(content, 3 * my_chunk, 1, 4, True)
def _do_test_thin_provisioning(self, content, size, truncate, write, thin):
self.config(filesystem_store_chunk_size=units.Ki,
filesystem_thin_provisioning=thin,
group='glance_store')
self.store.configure()
image_file = six.BytesIO(content)
image_id = str(uuid.uuid4())
with mock.patch.object(builtins, 'open') as popen:
self.store.add(image_id, image_file, size, self.hash_algo)
write_count = popen.return_value.__enter__().write.call_count
truncate_count = popen.return_value.__enter__().truncate.call_count
self.assertEqual(write_count, write)
self.assertEqual(truncate_count, truncate)
def test_add_with_verifier(self):
"""Test that 'verifier.update' is called when verifier is provided."""
verifier = mock.MagicMock(name='mock_verifier')

2
glance_store/tests/unit/test_opts.py
View File

@ -89,12 +89,14 @@ class OptsTestCase(base.StoreBaseTest):
'filesystem_store_datadirs',
'filesystem_store_file_perm',
'filesystem_store_metadata_file',
'filesystem_thin_provisioning',
'http_proxy_information',
'https_ca_certificates_file',
'rbd_store_ceph_conf',
'rbd_store_chunk_size',
'rbd_store_pool',
'rbd_store_user',
'rbd_thin_provisioning',
'rados_connect_timeout',
'rootwrap_config',
's3_store_access_key',

77
glance_store/tests/unit/test_rbd_store.py
View File

@ -247,6 +247,9 @@ class TestStore(base.StoreBaseTest,
self.data_iter = six.BytesIO(b'*' * self.data_len)
self.hash_algo = 'sha256'
def test_thin_provisioning_is_disabled_by_default(self):
self.assertEqual(self.store.thin_provisioning, False)
def test_add_w_image_size_zero(self):
"""Assert that correct size is returned even though 0 was provided."""
self.store.chunk_size = units.Ki
@ -359,6 +362,80 @@ class TestStore(base.StoreBaseTest,
self.assertEqual(expected_checksum, checksum)
self.assertEqual(expected_multihash, multihash)
def test_add_thick_provisioning_with_holes_in_file(self):
"""
Tests that a file which contains null bytes chunks is fully
written to rbd backend in a thick provisioning configuration.
"""
chunk_size = units.Mi
content = b"*" * chunk_size + b"\x00" * chunk_size + b"*" * chunk_size
self._do_test_thin_provisioning(content, 3 * chunk_size, 3, False)
def test_add_thin_provisioning_with_holes_in_file(self):
"""
Tests that a file which contains null bytes chunks is sparsified
in rbd backend with a thin provisioning configuration.
"""
chunk_size = units.Mi
content = b"*" * chunk_size + b"\x00" * chunk_size + b"*" * chunk_size
self._do_test_thin_provisioning(content, 3 * chunk_size, 2, True)
def test_add_thick_provisioning_without_holes_in_file(self):
"""
Tests that a file which not contain null bytes chunks is fully
written to rbd backend in a thick provisioning configuration.
"""
chunk_size = units.Mi
content = b"*" * 3 * chunk_size
self._do_test_thin_provisioning(content, 3 * chunk_size, 3, False)
def test_add_thin_provisioning_without_holes_in_file(self):
"""
Tests that a file which not contain null bytes chunks is fully
written to rbd backend in a thin provisioning configuration.
"""
chunk_size = units.Mi
content = b"*" * 3 * chunk_size
self._do_test_thin_provisioning(content, 3 * chunk_size, 3, True)
def test_add_thick_provisioning_with_partial_holes_in_file(self):
"""
Tests that a file which contains null bytes not aligned with
chunk size is fully written with a thick provisioning configuration.
"""
chunk_size = units.Mi
my_chunk = int(chunk_size * 1.5)
content = b"*" * my_chunk + b"\x00" * my_chunk + b"*" * my_chunk
self._do_test_thin_provisioning(content, 3 * my_chunk, 5, False)
def test_add_thin_provisioning_with_partial_holes_in_file(self):
"""
Tests that a file which contains null bytes not aligned with
chunk size is sparsified with a thin provisioning configuration.
"""
chunk_size = units.Mi
my_chunk = int(chunk_size * 1.5)
content = b"*" * my_chunk + b"\x00" * my_chunk + b"*" * my_chunk
self._do_test_thin_provisioning(content, 3 * my_chunk, 4, True)
def _do_test_thin_provisioning(self, content, size, write, thin):
self.config(rbd_store_chunk_size=1,
rbd_thin_provisioning=thin)
self.store.configure()
image_id = 'fake_image_id'
image_file = six.BytesIO(content)
expected_checksum = hashlib.md5(content).hexdigest()
expected_multihash = hashlib.sha256(content).hexdigest()
with mock.patch.object(rbd_store.rbd.Image, 'write') as mock_write:
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, size, self.hash_algo)
self.assertEqual(mock_write.call_count, write)
self.assertEqual(expected_checksum, checksum)
self.assertEqual(expected_multihash, multihash)
def test_delete(self):
def _fake_remove(*args, **kwargs):
self.called_commands_actual.append('remove')

15
releasenotes/notes/handle-sparse-image-a3ecfc4ae1c00d48.yaml Normal file
View File

@ -0,0 +1,15 @@
---
features:
- |
Add new configuration option ``rbd_thin_provisioning`` and
``filesystem_thin_provisioning`` to rbd and filesystem
store to enable or not sparse upload, default are False.
A sparse file means that we do not actually write null byte sequences
but only the data itself at a given offset, the "holes" which can
appear will automatically be interpreted by the storage backend as
null bytes, and do not really consume your storage.
Enabling this feature will also speed up image upload and save
network traffic in addition to save space in the backend, as null
bytes sequences are not sent over the network.