From 137b3cf975d73437943e100065c76b83acfa7dd3 Mon Sep 17 00:00:00 2001 From: Lars Gellrich Date: Wed, 1 Aug 2012 16:04:37 +0000 Subject: [PATCH] Enable client V2 to download images Added the CLI option image-download to download an image via API V2. Added utility function to save an image. Added common iterator to validate the checksum. Related to bp glance-client-v2 Change-Id: I0247f5a3462142dc5e9f3dc16cbe00c8e3d42f42 --- glanceclient/common/utils.py | 38 +++++++++++++ glanceclient/v1/images.py | 16 ++++-- glanceclient/v2/images.py | 17 ++++++ glanceclient/v2/shell.py | 11 ++++ tests/test_utils.py | 45 +++++++++++++++ tests/utils.py | 3 + tests/v1/test_images.py | 106 ++++++++++++++++++++++++++++++++++- tests/v2/test_images.py | 58 +++++++++++++++++++ 8 files changed, 287 insertions(+), 7 deletions(-) create mode 100644 tests/test_utils.py diff --git a/glanceclient/common/utils.py b/glanceclient/common/utils.py index 957f713c..31b85300 100644 --- a/glanceclient/common/utils.py +++ b/glanceclient/common/utils.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import errno +import hashlib import os import sys import uuid @@ -130,3 +132,39 @@ def exit(msg=''): if msg: print >> sys.stderr, msg sys.exit(1) + + +def save_image(data, path): + """ + Save an image to the specified path. + + :param data: binary data of the image + :param path: path to save the image to + """ + if path is None: + image = sys.stdout + else: + image = open(path, 'wb') + try: + for chunk in data: + image.write(chunk) + finally: + if path is not None: + image.close() + + +def integrity_iter(iter, checksum): + """ + Check image data integrity. + + :raises: IOError + """ + md5sum = hashlib.md5() + for chunk in iter: + yield chunk + md5sum.update(chunk) + md5sum = md5sum.hexdigest() + if md5sum != checksum: + raise IOError(errno.EPIPE, + 'Corrupt image download. Checksum was %s expected %s' % + (md5sum, checksum)) diff --git a/glanceclient/v1/images.py b/glanceclient/v1/images.py index 16249232..37a565a6 100644 --- a/glanceclient/v1/images.py +++ b/glanceclient/v1/images.py @@ -20,6 +20,7 @@ import os import urllib from glanceclient.common import base +from glanceclient.common import utils UPDATE_PARAMS = ('name', 'disk_format', 'container_format', 'min_disk', 'min_ram', 'owner', 'size', 'is_public', 'protected', @@ -40,8 +41,8 @@ class Image(base.Resource): def delete(self): return self.manager.delete(self) - def data(self): - return self.manager.data(self) + def data(self, **kwargs): + return self.manager.data(self, **kwargs) class ImageManager(base.Manager): @@ -77,15 +78,20 @@ class ImageManager(base.Manager): meta = self._image_meta_from_headers(dict(resp.getheaders())) return Image(self, meta) - def data(self, image): + def data(self, image, do_checksum=True): """Get the raw data for a specific image. :param image: image object or id to look up + :param do_checksum: Enable/disable checksum validation :rtype: iterable containing image data """ image_id = base.getid(image) - _, body = self.api.raw_request('GET', '/v1/images/%s' % image_id) - return body + resp, body = self.api.raw_request('GET', '/v1/images/%s' % image_id) + checksum = resp.getheader('x-image-meta-checksum', None) + if do_checksum and checksum is not None: + return utils.integrity_iter(body, checksum) + else: + return body def list(self, **kwargs): """Get a list of images. diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index 09532f12..939e8d81 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +from glanceclient.common import utils + DEFAULT_PAGE_SIZE = 20 @@ -54,3 +56,18 @@ class Controller(object): # way to pass it into the model constructor without conflict body.pop('self', None) return self.model(**body) + + def data(self, image_id, do_checksum=True): + """ + Retrieve data of an image. + + :param image_id: ID of the image to download. + :param do_checksum: Enable/disable checksum validation. + """ + url = '/v2/images/%s/file' % image_id + resp, body = self.http_client.raw_request('GET', url) + checksum = resp.getheader('content-md5', None) + if do_checksum and checksum is not None: + return utils.integrity_iter(body, checksum) + else: + return body diff --git a/glanceclient/v2/shell.py b/glanceclient/v2/shell.py index 8ad4d993..63bcd131 100644 --- a/glanceclient/v2/shell.py +++ b/glanceclient/v2/shell.py @@ -49,3 +49,14 @@ def do_explain(gc, args): formatters = {'Attribute': lambda m: m.name} columns = ['Attribute', 'Description'] utils.print_list(schema.properties, columns, formatters) + + +@utils.arg('--file', metavar='', + help='Local file to save downloaded image data to. ' + 'If this is not specified the image data will be ' + 'written to stdout.') +@utils.arg('id', metavar='', help='ID of image to download.') +def do_image_download(gc, args): + """Download a specific image.""" + body = gc.images.data(args.id) + utils.save_image(body, args.file) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..cf388bfe --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,45 @@ +# Copyright 2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import errno +import unittest + +from glanceclient.common import utils + + +class TestUtils(unittest.TestCase): + + def test_integrity_iter_without_checksum(self): + try: + data = ''.join([f for f in utils.integrity_iter('A', None)]) + self.fail('integrity checked passed without checksum.') + except IOError, e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was 7fc56270e7a70fa81a5935b72eacbe29 expected None' + self.assertTrue(msg in str(e)) + + def test_integrity_iter_with_wrong_checksum(self): + try: + data = ''.join([f for f in utils.integrity_iter('BB', 'wrong')]) + self.fail('integrity checked passed with wrong checksum') + except IOError, e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was 9d3d9048db16a7eee539e93e3618cbe7 expected wrong' + self.assertTrue('expected wrong' in str(e)) + + def test_integrity_iter_with_checksum(self): + fixture = 'CCC' + checksum = 'defb99e69a9f1f6e06f15006b1f166ae' + data = ''.join([f for f in utils.integrity_iter(fixture, checksum)]) diff --git a/tests/utils.py b/tests/utils.py index 0df9f19b..95a0f0a9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -45,3 +45,6 @@ class FakeResponse(object): def getheaders(self): return copy.deepcopy(self.headers).items() + + def getheader(self, key, default): + return self.headers.get(key, default) diff --git a/tests/v1/test_images.py b/tests/v1/test_images.py index bc3a1f20..7a99f375 100644 --- a/tests/v1/test_images.py +++ b/tests/v1/test_images.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import errno import json import StringIO import unittest @@ -164,7 +165,34 @@ fixtures = { ), 'DELETE': ({}, None), }, - + '/v1/images/2': { + 'HEAD': ( + { + 'x-image-meta-id': '2' + }, + None, + ), + 'GET': ( + { + 'x-image-meta-checksum': 'wrong' + }, + 'YYY', + ), + }, + '/v1/images/3': { + 'HEAD': ( + { + 'x-image-meta-id': '3' + }, + None, + ), + 'GET': ( + { + 'x-image-meta-checksum': '0745064918b49693cca64d6b6a13d28a' + }, + 'ZZZ', + ), + }, } @@ -220,11 +248,44 @@ class ImageManagerTest(unittest.TestCase): self.assertEqual(image.name, 'image-1') def test_data(self): - data = ''.join([b for b in self.mgr.data('1')]) + data = ''.join([b for b in self.mgr.data('1', do_checksum=False)]) expect = [('GET', '/v1/images/1', {}, None)] self.assertEqual(self.api.calls, expect) self.assertEqual(data, 'XXX') + expect += [('GET', '/v1/images/1', {}, None)] + data = ''.join([b for b in self.mgr.data('1')]) + self.assertEqual(self.api.calls, expect) + self.assertEqual(data, 'XXX') + + def test_data_with_wrong_checksum(self): + data = ''.join([b for b in self.mgr.data('2', do_checksum=False)]) + expect = [('GET', '/v1/images/2', {}, None)] + self.assertEqual(self.api.calls, expect) + self.assertEqual(data, 'YYY') + + expect += [('GET', '/v1/images/2', {}, None)] + data = self.mgr.data('2') + self.assertEqual(self.api.calls, expect) + try: + data = ''.join([b for b in data]) + self.fail('data did not raise an error.') + except IOError, e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was fd7c5c4fdaa97163ee4ba8842baa537a expected wrong' + self.assertTrue(msg in str(e)) + + def test_data_with_checksum(self): + data = ''.join([b for b in self.mgr.data('3', do_checksum=False)]) + expect = [('GET', '/v1/images/3', {}, None)] + self.assertEqual(self.api.calls, expect) + self.assertEqual(data, 'ZZZ') + + expect += [('GET', '/v1/images/3', {}, None)] + data = ''.join([b for b in self.mgr.data('3')]) + self.assertEqual(self.api.calls, expect) + self.assertEqual(data, 'ZZZ') + def test_delete(self): self.mgr.delete('1') expect = [('DELETE', '/v1/images/1', {}, None)] @@ -352,3 +413,44 @@ class ImageTest(unittest.TestCase): ] self.assertEqual(self.api.calls, expect) self.assertEqual(data, 'XXX') + + data = ''.join([b for b in image.data(do_checksum=False)]) + expect += [('GET', '/v1/images/1', {}, None)] + self.assertEqual(self.api.calls, expect) + self.assertEqual(data, 'XXX') + + def test_data_with_wrong_checksum(self): + image = self.mgr.get('2') + data = ''.join([b for b in image.data(do_checksum=False)]) + expect = [ + ('HEAD', '/v1/images/2', {}, None), + ('GET', '/v1/images/2', {}, None), + ] + self.assertEqual(self.api.calls, expect) + self.assertEqual(data, 'YYY') + + data = image.data() + expect += [('GET', '/v1/images/2', {}, None)] + self.assertEqual(self.api.calls, expect) + try: + data = ''.join([b for b in image.data()]) + self.fail('data did not raise an error.') + except IOError, e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was fd7c5c4fdaa97163ee4ba8842baa537a expected wrong' + self.assertTrue(msg in str(e)) + + def test_data_with_checksum(self): + image = self.mgr.get('3') + data = ''.join([b for b in image.data(do_checksum=False)]) + expect = [ + ('HEAD', '/v1/images/3', {}, None), + ('GET', '/v1/images/3', {}, None), + ] + self.assertEqual(self.api.calls, expect) + self.assertEqual(data, 'ZZZ') + + data = ''.join([b for b in image.data()]) + expect += [('GET', '/v1/images/3', {}, None)] + self.assertEqual(self.api.calls, expect) + self.assertEqual(data, 'ZZZ') diff --git a/tests/v2/test_images.py b/tests/v2/test_images.py index 71821799..ef3869e9 100644 --- a/tests/v2/test_images.py +++ b/tests/v2/test_images.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import errno import unittest import warlock @@ -72,6 +73,28 @@ fixtures = { }, ), }, + '/v2/images/5cc4bebc-db27-11e1-a1eb-080027cbe205/file': { + 'GET': ( + {}, + 'A', + ), + }, + '/v2/images/66fb18d6-db27-11e1-a1eb-080027cbe205/file': { + 'GET': ( + { + 'content-md5': 'wrong' + }, + 'BB', + ), + }, + '/v2/images/1b1c6366-dd57-11e1-af0f-02163e68b1d8/file': { + 'GET': ( + { + 'content-md5': 'defb99e69a9f1f6e06f15006b1f166ae' + }, + 'CCC', + ), + } } @@ -105,3 +128,38 @@ class TestController(unittest.TestCase): image = self.controller.get('3a4560a1-e585-443e-9b39-553b46ec92d1') self.assertEqual(image.id, '3a4560a1-e585-443e-9b39-553b46ec92d1') self.assertEqual(image.name, 'image-1') + + def test_data_without_checksum(self): + body = self.controller.data('5cc4bebc-db27-11e1-a1eb-080027cbe205', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual(body, 'A') + + body = self.controller.data('5cc4bebc-db27-11e1-a1eb-080027cbe205') + body = ''.join([b for b in body]) + self.assertEqual(body, 'A') + + def test_data_with_wrong_checksum(self): + body = self.controller.data('66fb18d6-db27-11e1-a1eb-080027cbe205', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual(body, 'BB') + + body = self.controller.data('66fb18d6-db27-11e1-a1eb-080027cbe205') + try: + body = ''.join([b for b in body]) + self.fail('data did not raise an error.') + except IOError, e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was 9d3d9048db16a7eee539e93e3618cbe7 expected wrong' + self.assertTrue(msg in str(e)) + + def test_data_with_checksum(self): + body = self.controller.data('1b1c6366-dd57-11e1-af0f-02163e68b1d8', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual(body, 'CCC') + + body = self.controller.data('1b1c6366-dd57-11e1-af0f-02163e68b1d8') + body = ''.join([b for b in body]) + self.assertEqual(body, 'CCC')