openstack/ironic-python-agent
Code Issues Proposed changes

Allow image checksum to be a URL

Browse Source 
We allow image_source to be a URL, let us also support URLs for checksums.
This change copies handling of multi-file checksum files from metalsmith.

Change-Id: Ie4d7e5c79b76bdd72d50eeb384cf10519278a80c
Story: #2005061
Task: #29605
This commit is contained in:
Dmitry Tantsur 2019-02-25 13:33:00 +01:00
parent c121bbd247
commit f821db3a54
4 changed files with 186 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ironic_python_agent/errors.py
View File

@ -142,7 +142,7 @@ class ImageDownloadError(RESTError):
message = 'Error downloading image'
def __init__(self, image_id, msg):
details = 'Download of image id {} failed: {}'.format(image_id, msg)
details = 'Download of image {} failed: {}'.format(image_id, msg)
self.secondary_message = msg
super(ImageDownloadError, self).__init__(details)

83
ironic_python_agent/extensions/standby.py
View File

@ -23,6 +23,7 @@ from oslo_config import cfg
from oslo_log import log
import requests
import six
from six.moves.urllib import parse as urlparse
from ironic_python_agent import errors
from ironic_python_agent.extensions import base
@ -54,6 +55,60 @@ def _path_to_script(script):
return os.path.join(cwd, '..', script)
def _download_with_proxy(image_info, url, image_id):
"""Opens a download stream for the given URL.
:param image_info: Image information dictionary.
:param url: The URL string to request the image from.
:param image_id: Image ID or URL for logging.
:raises: ImageDownloadError if the download stream was not started
properly.
"""
no_proxy = image_info.get('no_proxy')
if no_proxy:
os.environ['no_proxy'] = no_proxy
proxies = image_info.get('proxies', {})
verify, cert = utils.get_ssl_client_options(CONF)
resp = requests.get(url, stream=True, proxies=proxies,
verify=verify, cert=cert)
if resp.status_code != 200:
msg = ('Received status code {} from {}, expected 200. Response '
'body: {}').format(resp.status_code, url, resp.text)
raise errors.ImageDownloadError(image_id, msg)
return resp
def _fetch_checksum(checksum, image_info):
"""Fetch checksum from remote location, if needed."""
if not (checksum.startswith('http://') or checksum.startswith('https://')):
# Not a remote checksum, return as it is.
return checksum
LOG.debug('Downloading checksums file from %s', checksum)
resp = _download_with_proxy(image_info, checksum, checksum).text
lines = [line.strip() for line in resp.split('\n') if line.strip()]
if not lines:
raise errors.ImageDownloadError(checksum, "Empty checksum file")
elif len(lines) == 1:
# Special case - checksums file with only the checksum itself
if ' ' not in lines[0]:
return lines[0]
# FIXME(dtantsur): can we assume the same name for all images?
expected_fname = os.path.basename(
urlparse.urlparse(image_info['urls'][0]).path)
for line in lines:
checksum, fname = line.strip().split(None, 1)
# The star symbol designates binary mode, which is the same as text
# mode on GNU systems.
if fname.strip().lstrip('*') == expected_fname:
return checksum.strip()
raise errors.ImageDownloadError(
checksum, "Checksum file does not contain name %s" % expected_fname)
def _write_partition_image(image, image_info, device):
"""Call disk_util to create partition and write the partition image.
@ -217,11 +272,15 @@ class ImageDownload(object):
self._hash_algo = hashlib.md5()
self._expected_hash_value = image_info['checksum']
self._expected_hash_value = _fetch_checksum(self._expected_hash_value,
image_info)
details = []
for url in image_info['urls']:
try:
LOG.info("Attempting to download image from {}".format(url))
self._request = self._download_file(image_info, url)
self._request = _download_with_proxy(image_info, url,
image_info['id'])
except errors.ImageDownloadError as e:
failtime = time.time() - self._time
log_msg = ('URL: {}; time: {} '
@ -236,28 +295,6 @@ class ImageDownload(object):
details = '\n '.join(details)
raise errors.ImageDownloadError(image_info['id'], details)
def _download_file(self, image_info, url):
"""Opens a download stream for the given URL.
:param image_info: Image information dictionary.
:param url: The URL string to request the image from.
:raises: ImageDownloadError if the download stream was not started
properly.
"""
no_proxy = image_info.get('no_proxy')
if no_proxy:
os.environ['no_proxy'] = no_proxy
proxies = image_info.get('proxies', {})
verify, cert = utils.get_ssl_client_options(CONF)
resp = requests.get(url, stream=True, proxies=proxies,
verify=verify, cert=cert)
if resp.status_code != 200:
msg = ('Received status code {} from {}, expected 200. Response '
'body: {}').format(resp.status_code, url, resp.text)
raise errors.ImageDownloadError(image_info['id'], msg)
return resp
def __iter__(self):
"""Downloads and returns the next chunk of the image.

130
ironic_python_agent/tests/unit/extensions/test_standby.py
View File

@ -23,13 +23,11 @@ from ironic_python_agent import hardware
from ironic_python_agent.tests.unit import base
def _build_fake_image_info():
def _build_fake_image_info(url='http://example.org'):
return {
'id': 'fake_id',
'node_uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c123',
'urls': [
'http://example.org',
],
'urls': [url],
'checksum': 'abc123',
'image_type': 'whole-disk-image',
}
@ -1119,10 +1117,10 @@ class TestStandbyExtension(base.IronicAgentTest):
self.assertEqual(expected_msg, result_msg)
@mock.patch('hashlib.md5', autospec=True)
@mock.patch('requests.get', autospec=True)
class TestImageDownload(base.IronicAgentTest):
@mock.patch('hashlib.md5', autospec=True)
@mock.patch('requests.get', autospec=True)
def test_download_image(self, requests_mock, md5_mock):
content = ['SpongeBob', 'SquarePants']
response = requests_mock.return_value
@ -1140,16 +1138,14 @@ class TestImageDownload(base.IronicAgentTest):
self.assertEqual(image_info['checksum'],
image_download._hash_algo.hexdigest())
@mock.patch('time.time', autospec=True)
@mock.patch('requests.get', autospec=True)
def test_download_image_fail(self, requests_mock, time_mock):
response = requests_mock.return_value
response.status_code = 401
response.text = 'Unauthorized'
time_mock.return_value = 0.0
image_info = _build_fake_image_info()
msg = ('Error downloading image: Download of image id fake_id failed: '
'URL: http://example.org; time: 0.0 seconds. Error: '
msg = ('Error downloading image: Download of image fake_id failed: '
'URL: http://example.org; time: .* seconds. Error: '
'Received status code 401 from http://example.org, expected '
'200. Response body: Unauthorized')
self.assertRaisesRegex(errors.ImageDownloadError, msg,
@ -1157,3 +1153,117 @@ class TestImageDownload(base.IronicAgentTest):
requests_mock.assert_called_once_with(image_info['urls'][0],
cert=None, verify=True,
stream=True, proxies={})
def test_download_image_and_checksum(self, requests_mock, md5_mock):
content = ['SpongeBob', 'SquarePants']
fake_cs = "019fe036425da1c562f2e9f5299820bf"
cs_response = mock.Mock()
cs_response.status_code = 200
cs_response.text = fake_cs + '\n'
response = mock.Mock()
response.status_code = 200
response.iter_content.return_value = content
requests_mock.side_effect = [cs_response, response]
image_info = _build_fake_image_info()
image_info['checksum'] = 'http://example.com/checksum'
md5_mock.return_value.hexdigest.return_value = fake_cs
image_download = standby.ImageDownload(image_info)
self.assertEqual(content, list(image_download))
requests_mock.assert_has_calls([
mock.call('http://example.com/checksum', cert=None, verify=True,
stream=True, proxies={}),
mock.call(image_info['urls'][0], cert=None, verify=True,
stream=True, proxies={}),
])
self.assertEqual(fake_cs, image_download._hash_algo.hexdigest())
def test_download_image_and_checksum_multiple(self, requests_mock,
md5_mock):
content = ['SpongeBob', 'SquarePants']
fake_cs = "019fe036425da1c562f2e9f5299820bf"
cs_response = mock.Mock()
cs_response.status_code = 200
cs_response.text = """
foobar irrelevant file.img
%s image.img
""" % fake_cs
response = mock.Mock()
response.status_code = 200
response.iter_content.return_value = content
requests_mock.side_effect = [cs_response, response]
image_info = _build_fake_image_info(
'http://example.com/path/image.img')
image_info['checksum'] = 'http://example.com/checksum'
md5_mock.return_value.hexdigest.return_value = fake_cs
image_download = standby.ImageDownload(image_info)
self.assertEqual(content, list(image_download))
requests_mock.assert_has_calls([
mock.call('http://example.com/checksum', cert=None, verify=True,
stream=True, proxies={}),
mock.call(image_info['urls'][0], cert=None, verify=True,
stream=True, proxies={}),
])
self.assertEqual(fake_cs, image_download._hash_algo.hexdigest())
def test_download_image_and_checksum_unknown_file(self, requests_mock,
md5_mock):
content = ['SpongeBob', 'SquarePants']
fake_cs = "019fe036425da1c562f2e9f5299820bf"
cs_response = mock.Mock()
cs_response.status_code = 200
cs_response.text = """
foobar irrelevant file.img
%s not-my-image.img
""" % fake_cs
response = mock.Mock()
response.status_code = 200
response.iter_content.return_value = content
requests_mock.side_effect = [cs_response, response]
image_info = _build_fake_image_info(
'http://example.com/path/image.img')
image_info['checksum'] = 'http://example.com/checksum'
md5_mock.return_value.hexdigest.return_value = fake_cs
self.assertRaisesRegex(errors.ImageDownloadError,
'Checksum file does not contain name image.img',
standby.ImageDownload, image_info)
def test_download_image_and_checksum_empty_file(self, requests_mock,
md5_mock):
content = ['SpongeBob', 'SquarePants']
cs_response = mock.Mock()
cs_response.status_code = 200
cs_response.text = " "
response = mock.Mock()
response.status_code = 200
response.iter_content.return_value = content
requests_mock.side_effect = [cs_response, response]
image_info = _build_fake_image_info(
'http://example.com/path/image.img')
image_info['checksum'] = 'http://example.com/checksum'
self.assertRaisesRegex(errors.ImageDownloadError,
'Empty checksum file',
standby.ImageDownload, image_info)
def test_download_image_and_checksum_failed(self, requests_mock, md5_mock):
content = ['SpongeBob', 'SquarePants']
cs_response = mock.Mock()
cs_response.status_code = 400
cs_response.text = " "
response = mock.Mock()
response.status_code = 200
response.iter_content.return_value = content
requests_mock.side_effect = [cs_response, response]
image_info = _build_fake_image_info(
'http://example.com/path/image.img')
image_info['checksum'] = 'http://example.com/checksum'
self.assertRaisesRegex(errors.ImageDownloadError,
'Received status code 400 from '
'http://example.com/checksum',
standby.ImageDownload, image_info)

5
releasenotes/notes/image-checksum-39b2ceef40933c28.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
Allows image checksum to be a URL pointing at a file with the image
checksum or several checksums.