openstack
/
ironic
Code Issues Proposed changes

Enhanced checksum support 

This patch adds the ability to pass new hash fields os_hash_algo
and os_hash_value to IPA.

Glance will not re-compute hash values for existing images (they only
update these fields for newly uploaded images), so the os_hash_algo
and os_hash_value can be None.

In the case of direct interface with http provisioning, there
are two hash computing, one is for md5 checksum, and the other is
for new os_hash_algo (if it exists). If the os_hash_algo is
configured to md5, we'll bypass and wouldn't pass os_hash_algo to
IPA, because it's a waste of resource.

Change-Id: Iff72194787561b936efe3572e2b7a70217165a82
Story: 2003938
Task: 26845
This commit is contained in:
Kaifeng Wang 2018-10-16 15:34:12 +08:00
parent ab1b117ee4
commit 11811f2f18
7 changed files with 130 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ironic/common/glance_service/service_utils.py
View File

@ -39,7 +39,8 @@ _IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner',
'name', 'created_at', 'updated_at',
'deleted_at', 'deleted', 'status',
'min_disk', 'min_ram', 'tags', 'visibility',
'protected', 'file', 'schema']
'protected', 'file', 'schema', 'os_hash_algo',
'os_hash_value']
def _extract_attributes(image):

7
ironic/drivers/modules/agent.py
View File

@ -219,6 +219,13 @@ class AgentDeployMixin(agent_base_vendor.AgentDeployMixin):
'stream_raw_images': CONF.agent.stream_raw_images,
}
if (node.instance_info.get('image_os_hash_algo') and
node.instance_info.get('image_os_hash_value')):
image_info['os_hash_algo'] = node.instance_info[
'image_os_hash_algo']
image_info['os_hash_value'] = node.instance_info[
'image_os_hash_value']
proxies = {}
for scheme in ('http', 'https'):
proxy_param = 'image_%s_proxy' % scheme

46
ironic/drivers/modules/deploy_utils.py
View File

@ -1156,6 +1156,22 @@ def destroy_images(node_uuid):
InstanceImageCache().clean_up()
@METRICS.timer('compute_image_checksum')
def compute_image_checksum(image_path, algorithm='md5'):
"""Compute checksum by given image path and algorithm."""
time_start = time.time()
LOG.debug('Start computing %(algo)s checksum for image %(image)s.',
{'algo': algorithm, 'image': image_path})
checksum = fileutils.compute_file_checksum(image_path,
algorithm=algorithm)
time_elapsed = time.time() - time_start
LOG.debug('Computed %(algo)s checksum for image %(image)s in '
'%(delta).2f seconds, checksum value: %(checksum)s.',
{'algo': algorithm, 'image': image_path, 'delta': time_elapsed,
'checksum': checksum})
return checksum
def remove_http_instance_symlink(node_uuid):
symlink_path = _get_http_image_symlink_file_path(node_uuid)
il_utils.unlink_without_raise(symlink_path)
@ -1210,27 +1226,35 @@ def build_instance_info_for_deploy(task):
instance_info['image_url'] = swift_temp_url
instance_info['image_checksum'] = image_info['checksum']
instance_info['image_disk_format'] = image_info['disk_format']
instance_info['image_os_hash_algo'] = image_info['os_hash_algo']
instance_info['image_os_hash_value'] = image_info['os_hash_value']
else:
# Ironic cache and serve images from httpboot server
force_raw = direct_deploy_should_convert_raw_image(node)
_, image_path = cache_instance_image(task.context, node,
force_raw=force_raw)
if force_raw:
time_start = time.time()
LOG.debug('Start calculating checksum for image %(image)s.',
{'image': image_path})
checksum = fileutils.compute_file_checksum(image_path,
algorithm='md5')
time_elapsed = time.time() - time_start
LOG.debug('Recalculated checksum for image %(image)s in '
'%(delta).2f seconds, new checksum %(checksum)s ',
{'image': image_path, 'delta': time_elapsed,
'checksum': checksum})
instance_info['image_checksum'] = checksum
instance_info['image_disk_format'] = 'raw'
LOG.debug('Recalculating checksum for image %(image)s due to '
'image conversion.', {'image': image_path})
md5checksum = compute_image_checksum(image_path, 'md5')
instance_info['image_checksum'] = md5checksum
# Populate instance_info with os_hash_algo, os_hash_value
# if they exists and not md5
os_hash_algo = image_info['os_hash_algo']
if os_hash_algo and os_hash_algo != 'md5':
hash_value = compute_image_checksum(image_path,
os_hash_algo)
instance_info['image_os_hash_algo'] = os_hash_algo
instance_info['image_os_hash_value'] = hash_value
else:
instance_info['image_checksum'] = image_info['checksum']
instance_info['image_disk_format'] = image_info['disk_format']
instance_info['image_os_hash_algo'] = image_info[
'os_hash_algo']
instance_info['image_os_hash_value'] = image_info[
'os_hash_value']
# Create symlink and update image url
symlink_dir = _get_http_image_symlink_dir_path()

2
ironic/tests/unit/common/test_glance_service.py
View File

@ -146,6 +146,8 @@ class TestGlanceImageService(base.TestCase):
'tags': None,
'updated_at': None,
'visibility': None,
'os_hash_algo': None,
'os_hash_value': None,
}
with mock.patch.object(self.service, 'call', return_value=image,
autospec=True):

138
ironic/tests/unit/drivers/modules/test_deploy_utils.py
View File

@ -2247,6 +2247,7 @@ class TestBuildInstanceInfoForDeploy(db_base.DbTestCase):
self.node.save()
image_info = {'checksum': 'aa', 'disk_format': 'qcow2',
'os_hash_algo': 'sha512', 'os_hash_value': 'fake-sha512',
'container_format': 'bare', 'properties': {}}
glance_mock.return_value.show = mock.MagicMock(spec_set=[],
return_value=image_info)
@ -2287,6 +2288,7 @@ class TestBuildInstanceInfoForDeploy(db_base.DbTestCase):
self.node.save()
image_info = {'checksum': 'aa', 'disk_format': 'qcow2',
'os_hash_algo': 'sha512', 'os_hash_value': 'fake-sha512',
'container_format': 'bare',
'properties': {'kernel_id': 'kernel',
'ramdisk_id': 'ramdisk'}}
@ -2310,6 +2312,8 @@ class TestBuildInstanceInfoForDeploy(db_base.DbTestCase):
'image_properties': {'kernel_id': 'kernel',
'ramdisk_id': 'ramdisk'},
'image_checksum': 'aa',
'image_os_hash_algo': 'sha512',
'image_os_hash_value': 'fake-sha512',
'image_container_format': 'bare',
'image_disk_format': 'qcow2'}
with task_manager.acquire(
@ -2434,9 +2438,9 @@ class TestBuildInstanceInfoForHttpProvisioning(db_base.DbTestCase):
self.node.instance_info = i_info
self.node.save()
self.md5sum_mock = self.useFixture(fixtures.MockPatchObject(
self.checksum_mock = self.useFixture(fixtures.MockPatchObject(
fileutils, 'compute_file_checksum')).mock
self.md5sum_mock.return_value = 'fake md5'
self.checksum_mock.return_value = 'fake-checksum'
self.cache_image_mock = self.useFixture(fixtures.MockPatchObject(
utils, 'cache_instance_image', autospec=True)).mock
self.cache_image_mock.return_value = (
@ -2454,79 +2458,87 @@ class TestBuildInstanceInfoForHttpProvisioning(db_base.DbTestCase):
self.expected_url = '/'.join([cfg.CONF.deploy.http_url,
cfg.CONF.deploy.http_image_subdir,
self.node.uuid])
self.image_info = {'checksum': 'aa', 'disk_format': 'qcow2',
'os_hash_algo': 'sha512',
'os_hash_value': 'fake-sha512',
'container_format': 'bare', 'properties': {}}
@mock.patch.object(image_service.HttpImageService, 'validate_href',
autospec=True)
@mock.patch.object(image_service, 'GlanceImageService', autospec=True)
def test_build_instance_info_no_force_raw(self, glance_mock,
validate_mock):
def _test_build_instance_info(self, glance_mock, validate_mock,
image_info={}, expect_raw=False):
glance_mock.return_value.show = mock.MagicMock(spec_set=[],
return_value=image_info)
with task_manager.acquire(
self.context, self.node.uuid, shared=False) as task:
instance_info = utils.build_instance_info_for_deploy(task)
glance_mock.assert_called_once_with(context=task.context)
glance_mock.return_value.show.assert_called_once_with(
self.node.instance_info['image_source'])
self.cache_image_mock.assert_called_once_with(task.context,
task.node,
force_raw=expect_raw)
symlink_dir = utils._get_http_image_symlink_dir_path()
symlink_file = utils._get_http_image_symlink_file_path(
self.node.uuid)
image_path = utils._get_image_file_path(self.node.uuid)
self.ensure_tree_mock.assert_called_once_with(symlink_dir)
self.create_link_mock.assert_called_once_with(image_path,
symlink_file)
validate_mock.assert_called_once_with(mock.ANY, self.expected_url,
secret=True)
return image_path, instance_info
def test_build_instance_info_no_force_raw(self):
cfg.CONF.set_override('force_raw_images', False)
_, instance_info = self._test_build_instance_info(
image_info=self.image_info, expect_raw=False)
image_info = {'checksum': 'aa', 'disk_format': 'qcow2',
'container_format': 'bare', 'properties': {}}
glance_mock.return_value.show = mock.MagicMock(spec_set=[],
return_value=image_info)
self.assertEqual(instance_info['image_checksum'], 'aa')
self.assertEqual(instance_info['image_disk_format'], 'qcow2')
self.assertEqual(instance_info['image_os_hash_algo'], 'sha512')
self.assertEqual(instance_info['image_os_hash_value'],
'fake-sha512')
self.checksum_mock.assert_not_called()
with task_manager.acquire(
self.context, self.node.uuid, shared=False) as task:
instance_info = utils.build_instance_info_for_deploy(task)
glance_mock.assert_called_once_with(context=task.context)
glance_mock.return_value.show.assert_called_once_with(
self.node.instance_info['image_source'])
self.cache_image_mock.assert_called_once_with(task.context,
task.node,
force_raw=False)
symlink_dir = utils._get_http_image_symlink_dir_path()
symlink_file = utils._get_http_image_symlink_file_path(
self.node.uuid)
image_path = utils._get_image_file_path(self.node.uuid)
self.ensure_tree_mock.assert_called_once_with(symlink_dir)
self.create_link_mock.assert_called_once_with(image_path,
symlink_file)
self.assertEqual(instance_info['image_checksum'], 'aa')
self.assertEqual(instance_info['image_disk_format'], 'qcow2')
self.md5sum_mock.assert_not_called()
validate_mock.assert_called_once_with(mock.ANY, self.expected_url,
secret=True)
@mock.patch.object(image_service.HttpImageService, 'validate_href',
autospec=True)
@mock.patch.object(image_service, 'GlanceImageService', autospec=True)
def test_build_instance_info_force_raw(self, glance_mock,
validate_mock):
def test_build_instance_info_force_raw(self):
cfg.CONF.set_override('force_raw_images', True)
image_path, instance_info = self._test_build_instance_info(
image_info=self.image_info, expect_raw=True)
image_info = {'checksum': 'aa', 'disk_format': 'qcow2',
'container_format': 'bare', 'properties': {}}
glance_mock.return_value.show = mock.MagicMock(spec_set=[],
return_value=image_info)
self.assertEqual(instance_info['image_checksum'], 'fake-checksum')
self.assertEqual(instance_info['image_disk_format'], 'raw')
calls = [mock.call(image_path, algorithm='md5'),
mock.call(image_path, algorithm='sha512')]
self.checksum_mock.assert_has_calls(calls)
with task_manager.acquire(
self.context, self.node.uuid, shared=False) as task:
def test_build_instance_info_force_raw_new_fields_none(self):
cfg.CONF.set_override('force_raw_images', True)
self.image_info['os_hash_algo'] = None
self.image_info['os_hash_value'] = None
image_path, instance_info = self._test_build_instance_info(
image_info=self.image_info, expect_raw=True)
instance_info = utils.build_instance_info_for_deploy(task)
self.assertEqual(instance_info['image_checksum'], 'fake-checksum')
self.assertEqual(instance_info['image_disk_format'], 'raw')
self.assertNotIn('image_os_hash_algo', instance_info.keys())
self.assertNotIn('image_os_hash_value', instance_info.keys())
self.checksum_mock.assert_called_once_with(image_path, algorithm='md5')
glance_mock.assert_called_once_with(context=task.context)
glance_mock.return_value.show.assert_called_once_with(
self.node.instance_info['image_source'])
self.cache_image_mock.assert_called_once_with(task.context,
task.node,
force_raw=True)
symlink_dir = utils._get_http_image_symlink_dir_path()
symlink_file = utils._get_http_image_symlink_file_path(
self.node.uuid)
image_path = utils._get_image_file_path(self.node.uuid)
self.ensure_tree_mock.assert_called_once_with(symlink_dir)
self.create_link_mock.assert_called_once_with(image_path,
symlink_file)
self.assertEqual(instance_info['image_checksum'], 'fake md5')
self.assertEqual(instance_info['image_disk_format'], 'raw')
self.md5sum_mock.assert_called_once_with(image_path,
algorithm='md5')
validate_mock.assert_called_once_with(mock.ANY, self.expected_url,
secret=True)
def test_build_instance_info_force_raw_new_fields_is_md5(self):
cfg.CONF.set_override('force_raw_images', True)
self.image_info['os_hash_algo'] = 'md5'
self.image_info['os_hash_value'] = 'fake-md5'
image_path, instance_info = self._test_build_instance_info(
image_info=self.image_info, expect_raw=True)
self.assertEqual(instance_info['image_checksum'], 'fake-checksum')
self.assertEqual(instance_info['image_disk_format'], 'raw')
self.assertNotIn('image_os_hash_algo', instance_info.keys())
self.assertNotIn('image_os_hash_value', instance_info.keys())
self.checksum_mock.assert_called_once_with(image_path, algorithm='md5')
class TestStorageInterfaceUtils(db_base.DbTestCase):

3
ironic/tests/unit/stubs.py
View File

@ -57,7 +57,8 @@ class FakeImage(dict):
'container_format', 'checksum', 'id',
'name', 'deleted', 'status',
'min_disk', 'min_ram', 'tags', 'visibility',
'protected', 'file', 'schema']
'protected', 'file', 'schema', 'os_hash_algo',
'os_hash_value']
raw = dict.fromkeys(IMAGE_ATTRIBUTES)
raw.update(metadata)
# raw['created_at'] = NOW_GLANCE_FORMAT

7
releasenotes/notes/enhanced-checksum-f5a2b7aa8632b88f.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- In accordance with the `multihash support
<https://specs.openstack.org/openstack/glance-specs/specs/rocky/approved/glance/multihash.html>`_
provided by glance, ironic now supports using the new ``os_hash_algo``
and ``os_hash_value`` fields to computes and validates image checksum
when deploying instance images by the ``direct`` deploy interface.