diff --git a/doc/source/configuring.rst b/doc/source/configuring.rst index 1b00431e8c..cdbb497188 100644 --- a/doc/source/configuring.rst +++ b/doc/source/configuring.rst @@ -225,6 +225,31 @@ Can only be specified in configuration files. If true, Glance will attempt to create the container ``swift_store_container`` if it does not exist. +* ``swift_store_large_object_size=SIZE_IN_MB`` + +Optional. Default: ``5120`` + +Can only be specified in configuration files. + +`This option is specific to the Swift storage backend.` + +What size, in MB, should Glance start chunking image files +and do a large object manifest in Swift? By default, this is +the maximum object size in Swift, which is 5GB + +* ``swift_store_large_object_chunk_size=SIZE_IN_MB`` + +Optional. Default: ``200`` + +Can only be specified in configuration files. + +`This option is specific to the Swift storage backend.` + +When doing a large object manifest, what size, in MB, should +Glance write chunks to Swift? This amount of data is written +to a temporary disk buffer during the process of chunking +the image file, and the default is 200MB + Configuring the S3 Storage Backend ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/etc/glance-api.conf b/etc/glance-api.conf index a920ea523d..5d65809d74 100644 --- a/etc/glance-api.conf +++ b/etc/glance-api.conf @@ -29,6 +29,8 @@ log_file = /var/log/glance/api.log # Send logs to syslog (/dev/log) instead of to file specified by `log_file` use_syslog = False +# ============ Notification System Options ===================== + # Notifications can be sent when images are create, updated or deleted. # There are three methods of sending notifications, logging (via the # log_file directive), rabbit (via a rabbitmq queue) or noop (no @@ -70,6 +72,17 @@ swift_store_container = glance # Do we create the container if it does not exist? swift_store_create_container_on_put = False +# What size, in MB, should Glance start chunking image files +# and do a large object manifest in Swift? By default, this is +# the maximum object size in Swift, which is 5GB +swift_store_large_object_size = 5120 + +# When doing a large object manifest, what size, in MB, should +# Glance write chunks to Swift? This amount of data is written +# to a temporary disk buffer during the process of chunking +# the image file, and the default is 200MB +swift_store_large_object_chunk_size = 200 + # Whether to use ServiceNET to communicate with the Swift storage servers. # (If you aren't RACKSPACE, leave this False!) # diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index 9b12a8acdc..10e3c87466 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -343,8 +343,17 @@ class Controller(api.BaseController): try: logger.debug(_("Uploading image data for image %(image_id)s " "to %(store_name)s store"), locals()) + if req.content_length: + image_size = int(req.content_length) + elif 'x-image-meta-size' in req.headers: + image_size = int(req.headers['x-image-meta-size']) + else: + logger.debug(_("Got request with no content-length and no " + "x-image-meta-size header")) + image_size = 0 location, size, checksum = store.add(image_meta['id'], - req.body_file) + req.body_file, + image_size) # Verify any supplied checksum value matches checksum # returned from store when adding image diff --git a/glance/client.py b/glance/client.py index a0024ca2d8..44fb048d35 100644 --- a/glance/client.py +++ b/glance/client.py @@ -19,7 +19,9 @@ Client classes for callers of a Glance system """ +import errno import json +import os from glance.api.v1 import images as v1_images from glance.common import client as base_client @@ -131,6 +133,24 @@ class V1Client(base_client.BaseClient): if image_data: body = image_data headers['content-type'] = 'application/octet-stream' + # For large images, we need to supply the size of the + # image file. See LP Bug #827660. + if hasattr(image_data, 'seek') and hasattr(image_data, 'tell'): + try: + image_data.seek(0, os.SEEK_END) + image_size = image_data.tell() + image_data.seek(0) + headers['x-image-meta-size'] = image_size + headers['content-length'] = image_size + except IOError, e: + if e.errno == errno.ESPIPE: + # Illegal seek. This means the user is trying + # to pipe image data to the client, e.g. + # echo testdata | bin/glance add blah..., or + # that stdin is empty + pass + else: + raise else: body = None diff --git a/glance/store/base.py b/glance/store/base.py index 0a8a92fbc6..f7310fd9a2 100644 --- a/glance/store/base.py +++ b/glance/store/base.py @@ -72,7 +72,7 @@ class Store(object): """ raise exception.StoreAddDisabled - def add(self, image_id, image_file): + def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object @@ -80,6 +80,7 @@ class Store(object): :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object + :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py index e7fa5f2c24..475e0c6950 100644 --- a/glance/store/filesystem.py +++ b/glance/store/filesystem.py @@ -165,7 +165,7 @@ class Store(glance.store.base.Store): else: raise exception.NotFound(_("Image file %s does not exist") % fn) - def add(self, image_id, image_file): + def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object @@ -173,6 +173,7 @@ class Store(glance.store.base.Store): :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object + :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already diff --git a/glance/store/s3.py b/glance/store/s3.py index 668c28446b..7f791dc592 100644 --- a/glance/store/s3.py +++ b/glance/store/s3.py @@ -255,7 +255,7 @@ class Store(glance.store.base.Store): key.BufferSize = self.CHUNKSIZE return ChunkedFile(key) - def add(self, image_id, image_file): + def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object @@ -263,6 +263,7 @@ class Store(glance.store.base.Store): :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object + :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already diff --git a/glance/store/swift.py b/glance/store/swift.py index f9810ab900..d1e5d23305 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -19,8 +19,11 @@ from __future__ import absolute_import +import hashlib import httplib import logging +import math +import tempfile import urlparse from glance.common import config @@ -34,7 +37,9 @@ try: except ImportError: pass -DEFAULT_SWIFT_CONTAINER = 'glance' +DEFAULT_CONTAINER = 'glance' +DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 # 5GB +DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 * 1024 * 1024 # 200M logger = logging.getLogger('glance.store.swift') @@ -189,7 +194,26 @@ class Store(glance.store.base.Store): self.user = self._option_get('swift_store_user') self.key = self._option_get('swift_store_key') self.container = self.options.get('swift_store_container', - DEFAULT_SWIFT_CONTAINER) + DEFAULT_CONTAINER) + try: + if self.options.get('swift_store_large_object_size'): + self.large_object_size = int( + self.options.get('swift_store_large_object_size') + ) * (1024 * 1024) # Size specified in MB in conf files + else: + self.large_object_size = DEFAULT_LARGE_OBJECT_SIZE + + if self.options.get('swift_store_large_object_chunk_size'): + self.large_object_chunk_size = int( + self.options.get('swift_store_large_object_chunk_size') + ) * (1024 * 1024) # Size specified in MB in conf files + else: + self.large_object_chunk_size = DEFAULT_LARGE_OBJECT_CHUNK_SIZE + except Exception, e: + reason = _("Error in configuration options: %s") % e + logger.error(reason) + raise exception.BadStoreConfiguration(store_name="swift", + reason=reason) self.scheme = 'swift+https' if self.auth_address.startswith('http://'): @@ -259,7 +283,7 @@ class Store(glance.store.base.Store): reason=reason) return result - def add(self, image_id, image_file): + def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object @@ -267,6 +291,7 @@ class Store(glance.store.base.Store): :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object + :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already @@ -284,6 +309,11 @@ class Store(glance.store.base.Store): :note Swift auth URLs by default use HTTPS. To specify an HTTP auth URL, you can specify http://someurl.com for the swift_store_auth_address config option + + :note Swift cannot natively/transparently handle objects >5GB + in size. So, if the image is greater than 5GB, we write + chunks of image data to Swift and then write an manifest + to Swift that contains information about the chunks. """ swift_conn = self._make_swift_connection( auth_url=self.full_auth_address, user=self.user, key=self.key) @@ -301,8 +331,67 @@ class Store(glance.store.base.Store): logger.debug(_("Adding image object '%(obj_name)s' " "to Swift") % locals()) try: - obj_etag = swift_conn.put_object(self.container, obj_name, - image_file) + if image_size < self.large_object_size: + # image_size == 0 is when we don't know the size + # of the image. This can occur with older clients + # that don't inspect the payload size, and we simply + # try to put the object into Swift and hope for the + # best... + obj_etag = swift_conn.put_object(self.container, obj_name, + image_file) + else: + # Write the image into Swift in chunks. We cannot + # stream chunks of the webob.Request.body_file, unfortunately, + # so we must write chunks of the body_file into a temporary + # disk buffer, and then pass this disk buffer to Swift. + bytes_left = image_size + chunk_id = 1 + total_chunks = int(math.ceil( + float(image_size) / float(self.large_object_chunk_size))) + checksum = hashlib.md5() + while bytes_left > 0: + with tempfile.NamedTemporaryFile() as disk_buffer: + chunk_size = min(self.large_object_chunk_size, + bytes_left) + logger.debug(_("Writing %(chunk_size)d bytes for " + "chunk %(chunk_id)d/" + "%(total_chunks)d to disk buffer " + "for image %(image_id)s") + % locals()) + chunk = image_file.read(chunk_size) + checksum.update(chunk) + disk_buffer.write(chunk) + disk_buffer.flush() + logger.debug(_("Writing chunk %(chunk_id)d/" + "%(total_chunks)d to Swift " + "for image %(image_id)s") + % locals()) + chunk_etag = swift_conn.put_object( + self.container, + "%s-%05d" % (obj_name, chunk_id), + open(disk_buffer.name, 'rb')) + logger.debug(_("Wrote chunk %(chunk_id)d/" + "%(total_chunks)d to Swift " + "returning MD5 of content: " + "%(chunk_etag)s") + % locals()) + bytes_left -= self.large_object_chunk_size + chunk_id += 1 + + # Now we write the object manifest and return the + # manifest's etag... + manifest = "%s/%s" % (self.container, obj_name) + headers = {'ETag': hashlib.md5("").hexdigest(), + 'X-Object-Manifest': manifest} + + # The ETag returned for the manifest is actually the + # MD5 hash of the concatenated checksums of the strings + # of each chunk...so we ignore this result in favour of + # the MD5 of the entire image file contents, so that + # users can verify the image file contents accordingly + _ignored = swift_conn.put_object(self.container, obj_name, + None, headers=headers) + obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual @@ -310,15 +399,7 @@ class Store(glance.store.base.Store): # the location attribute from GET /images/ and # GET /images/details - # We do a HEAD on the newly-added image to determine the size - # of the image. A bit slow, but better than taking the word - # of the user adding the image with size attribute in the metadata - resp_headers = swift_conn.head_object(self.container, obj_name) - size = 0 - # header keys are lowercased by Swift - if 'content-length' in resp_headers: - size = int(resp_headers['content-length']) - return (location.get_uri(), size, obj_etag) + return (location.get_uri(), image_size, obj_etag) except swift_client.ClientException, e: if e.http_status == httplib.CONFLICT: raise exception.Duplicate(_("Swift already has an image at " diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index 1c558b1d52..d6d998958e 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -141,6 +141,12 @@ class ApiServer(Server): self.s3_store_access_key = "" self.s3_store_secret_key = "" self.s3_store_bucket = "" + self.swift_store_auth_address = "" + self.swift_store_user = "" + self.swift_store_key = "" + self.swift_store_container = "" + self.swift_store_large_object_size = 5 * 1024 + self.swift_store_large_object_chunk_size = 200 self.delayed_delete = delayed_delete self.conf_base = """[DEFAULT] verbose = %(verbose)s @@ -156,6 +162,12 @@ s3_store_host = %(s3_store_host)s s3_store_access_key = %(s3_store_access_key)s s3_store_secret_key = %(s3_store_secret_key)s s3_store_bucket = %(s3_store_bucket)s +swift_store_auth_address = %(swift_store_auth_address)s +swift_store_user = %(swift_store_user)s +swift_store_key = %(swift_store_key)s +swift_store_container = %(swift_store_container)s +swift_store_large_object_size = %(swift_store_large_object_size)s +swift_store_large_object_chunk_size = %(swift_store_large_object_chunk_size)s delayed_delete = %(delayed_delete)s [pipeline:glance-api] diff --git a/glance/tests/functional/test_bin_glance.py b/glance/tests/functional/test_bin_glance.py index 25474c2e35..251da69fbb 100644 --- a/glance/tests/functional/test_bin_glance.py +++ b/glance/tests/functional/test_bin_glance.py @@ -55,10 +55,14 @@ class TestBinGlance(functional.FunctionalTest): self.assertEqual('', out.strip()) # 1. Add public image - cmd = "echo testdata | bin/glance --port=%d add is_public=True"\ - " name=MyImage" % api_port + with tempfile.NamedTemporaryFile() as image_file: + image_file.write("XXX") + image_file.flush() + image_file_name = image_file.name + cmd = "bin/glance --port=%d add is_public=True"\ + " name=MyImage < %s" % (api_port, image_file_name) - exitcode, out, err = execute(cmd) + exitcode, out, err = execute(cmd) self.assertEqual(0, exitcode) self.assertEqual('Added new image with ID: 1', out.strip()) @@ -78,9 +82,10 @@ class TestBinGlance(functional.FunctionalTest): [c.strip() for c in line.split()] self.assertEqual('MyImage', name) - self.assertEqual('9', size, - "Expected image to be 9 bytes in size. Make sure" - " you're running the correct version of webob.") + self.assertEqual('3', size, + "Expected image to be 3 bytes in size, but got %s. " + "Make sure you're running the correct version " + "of webob." % size) # 3. Delete the image cmd = "bin/glance --port=%d --force delete 1" % api_port diff --git a/glance/tests/functional/test_swift.py b/glance/tests/functional/test_swift.py index 69c38a69fd..064f6c4dd1 100644 --- a/glance/tests/functional/test_swift.py +++ b/glance/tests/functional/test_swift.py @@ -31,6 +31,9 @@ skipped. """ import ConfigParser +import hashlib +import httplib +import httplib2 import json import os import tempfile @@ -39,6 +42,8 @@ import unittest from glance.tests.functional import test_api from glance.tests.utils import execute, skip_if_disabled +FIVE_MB = 5 * 1024 * 1024 + class TestSwift(test_api.TestApi): @@ -137,5 +142,192 @@ class TestSwift(test_api.TestApi): super(TestSwift, self).tearDown() def clear_container(self): - self.swift_conn.delete_container(self.swift_store_container) + from swift.common import client as swift_client + try: + self.swift_conn.delete_container(self.swift_store_container) + except swift_client.ClientException, e: + if e.http_status == httplib.CONFLICT: + pass + else: + raise self.swift_conn.put_container(self.swift_store_container) + + @skip_if_disabled + def test_add_large_object_manifest(self): + """ + We test the large object manifest code path in the Swift driver. + In the case where an image file is bigger than the config variable + swift_store_large_object_size, then we chunk the image into + Swift, and add a manifest put_object at the end. + """ + self.cleanup() + + self.swift_store_large_object_size = 2 # In MB + self.swift_store_large_object_chunk_size = 1 # In MB + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # 0. GET /images + # Verify no public images + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(content, '{"images": []}') + + # 1. POST /images with public image named Image1 + # attribute and no custom properties. Verify a 200 OK is returned + image_data = "*" * FIVE_MB + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image1', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + self.assertEqual(data['image']['checksum'], + hashlib.md5(image_data).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_MB) + self.assertEqual(data['image']['name'], "Image1") + self.assertEqual(data['image']['is_public'], True) + + # 4. HEAD /images/1 + # Verify image found now + path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'HEAD') + self.assertEqual(response.status, 200) + self.assertEqual(response['x-image-meta-name'], "Image1") + + # 5. GET /images/1 + # Verify all information on image we just added is correct + path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + + expected_image_headers = { + 'x-image-meta-id': '1', + 'x-image-meta-name': 'Image1', + 'x-image-meta-is_public': 'True', + 'x-image-meta-status': 'active', + 'x-image-meta-disk_format': '', + 'x-image-meta-container_format': '', + 'x-image-meta-size': str(FIVE_MB) + } + + expected_std_headers = { + 'content-length': str(FIVE_MB), + 'content-type': 'application/octet-stream'} + + for expected_key, expected_value in expected_image_headers.items(): + self.assertEqual(response[expected_key], expected_value, + "For key '%s' expected header value '%s'. Got '%s'" + % (expected_key, expected_value, + response[expected_key])) + + for expected_key, expected_value in expected_std_headers.items(): + self.assertEqual(response[expected_key], expected_value, + "For key '%s' expected header value '%s'. Got '%s'" + % (expected_key, + expected_value, + response[expected_key])) + + self.assertEqual(content, "*" * FIVE_MB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_MB).hexdigest()) + + self.stop_servers() + + @skip_if_disabled + def test_add_large_object_manifest_uneven_size(self): + """ + Test when large object manifest in scenario where + image size % chunk size != 0 + """ + self.cleanup() + + self.swift_store_large_object_size = 3 # In MB + self.swift_store_large_object_chunk_size = 2 # In MB + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # 0. GET /images + # Verify no public images + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(content, '{"images": []}') + + # 1. POST /images with public image named Image1 + # attribute and no custom properties. Verify a 200 OK is returned + image_data = "*" * FIVE_MB + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image1', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + self.assertEqual(data['image']['checksum'], + hashlib.md5(image_data).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_MB) + self.assertEqual(data['image']['name'], "Image1") + self.assertEqual(data['image']['is_public'], True) + + # 4. HEAD /images/1 + # Verify image found now + path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'HEAD') + self.assertEqual(response.status, 200) + self.assertEqual(response['x-image-meta-name'], "Image1") + + # 5. GET /images/1 + # Verify all information on image we just added is correct + path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + + expected_image_headers = { + 'x-image-meta-id': '1', + 'x-image-meta-name': 'Image1', + 'x-image-meta-is_public': 'True', + 'x-image-meta-status': 'active', + 'x-image-meta-disk_format': '', + 'x-image-meta-container_format': '', + 'x-image-meta-size': str(FIVE_MB) + } + + expected_std_headers = { + 'content-length': str(FIVE_MB), + 'content-type': 'application/octet-stream'} + + for expected_key, expected_value in expected_image_headers.items(): + self.assertEqual(response[expected_key], expected_value, + "For key '%s' expected header value '%s'. Got '%s'" + % (expected_key, expected_value, + response[expected_key])) + + for expected_key, expected_value in expected_std_headers.items(): + self.assertEqual(response[expected_key], expected_value, + "For key '%s' expected header value '%s'. Got '%s'" + % (expected_key, + expected_value, + response[expected_key])) + + self.assertEqual(content, "*" * FIVE_MB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_MB).hexdigest()) + + self.stop_servers() diff --git a/glance/tests/unit/test_filesystem_store.py b/glance/tests/unit/test_filesystem_store.py index d7ce4d830f..f854e82d40 100644 --- a/glance/tests/unit/test_filesystem_store.py +++ b/glance/tests/unit/test_filesystem_store.py @@ -87,7 +87,8 @@ class TestStore(unittest.TestCase): expected_image_id) image_file = StringIO.StringIO(expected_file_contents) - location, size, checksum = self.store.add(42, image_file) + location, size, checksum = self.store.add(42, image_file, + expected_file_size) self.assertEquals(expected_location, location) self.assertEquals(expected_file_size, size) @@ -116,7 +117,7 @@ class TestStore(unittest.TestCase): 'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR} self.assertRaises(exception.Duplicate, self.store.add, - 2, image_file) + 2, image_file, 0) def test_delete(self): """ diff --git a/glance/tests/unit/test_s3_store.py b/glance/tests/unit/test_s3_store.py index def76dfb68..3e86e45927 100644 --- a/glance/tests/unit/test_s3_store.py +++ b/glance/tests/unit/test_s3_store.py @@ -209,7 +209,8 @@ class TestStore(unittest.TestCase): expected_image_id) image_s3 = StringIO.StringIO(expected_s3_contents) - location, size, checksum = self.store.add(42, image_s3) + location, size, checksum = self.store.add(42, image_s3, + expected_s3_size) self.assertEquals(expected_location, location) self.assertEquals(expected_s3_size, size) @@ -258,7 +259,8 @@ class TestStore(unittest.TestCase): image_s3 = StringIO.StringIO(expected_s3_contents) self.store = Store(new_options) - location, size, checksum = self.store.add(i, image_s3) + location, size, checksum = self.store.add(i, image_s3, + expected_s3_size) self.assertEquals(expected_location, location) self.assertEquals(expected_s3_size, size) @@ -281,7 +283,7 @@ class TestStore(unittest.TestCase): image_s3 = StringIO.StringIO("nevergonnamakeit") self.assertRaises(exception.Duplicate, self.store.add, - 2, image_s3) + 2, image_s3, 0) def _option_required(self, key): options = S3_OPTIONS.copy() diff --git a/glance/tests/unit/test_swift_store.py b/glance/tests/unit/test_swift_store.py index b31bd2d917..a2ea8a75d7 100644 --- a/glance/tests/unit/test_swift_store.py +++ b/glance/tests/unit/test_swift_store.py @@ -29,9 +29,10 @@ import swift.common.client from glance.common import exception from glance.store import BackendException -from glance.store.swift import Store +import glance.store.swift from glance.store.location import get_location_from_uri +Store = glance.store.swift.Store FIVE_KB = (5 * 1024) SWIFT_OPTIONS = {'verbose': True, 'debug': True, @@ -64,7 +65,13 @@ def stub_out_swift_common_client(stubs): def fake_put_object(url, token, container, name, contents, **kwargs): # PUT returns the ETag header for the newly-added object + # Large object manifest... fixture_key = "%s/%s" % (container, name) + if kwargs.get('headers'): + etag = kwargs['headers']['ETag'] + fixture_headers[fixture_key] = {'manifest': True, + 'etag': etag} + return etag if not fixture_key in fixture_headers.keys(): if hasattr(contents, 'read'): fixture_object = StringIO.StringIO() @@ -83,7 +90,7 @@ def stub_out_swift_common_client(stubs): fixture_headers[fixture_key] = { 'content-length': read_len, 'etag': etag} - return fixture_headers[fixture_key]['etag'] + return etag else: msg = ("Object PUT failed - Object with key %s already exists" % fixture_key) @@ -92,14 +99,27 @@ def stub_out_swift_common_client(stubs): def fake_get_object(url, token, container, name, **kwargs): # GET returns the tuple (list of headers, file object) - try: - fixture_key = "%s/%s" % (container, name) - return fixture_headers[fixture_key], fixture_objects[fixture_key] - except KeyError: + fixture_key = "%s/%s" % (container, name) + if not fixture_key in fixture_headers: msg = "Object GET failed" raise swift.common.client.ClientException(msg, http_status=httplib.NOT_FOUND) + fixture = fixture_headers[fixture_key] + if 'manifest' in fixture: + # Large object manifest... we return a file containing + # all objects with prefix of this fixture key + chunk_keys = sorted([k for k in fixture_headers.keys() + if k.startswith(fixture_key) and + k != fixture_key]) + result = StringIO.StringIO() + for key in chunk_keys: + result.write(fixture_objects[key].getvalue()) + return fixture_headers[fixture_key], result + + else: + return fixture_headers[fixture_key], fixture_objects[fixture_key] + def fake_head_object(url, token, container, name, **kwargs): # HEAD returns the list of headers for an object try: @@ -227,7 +247,8 @@ class TestStore(unittest.TestCase): expected_image_id) image_swift = StringIO.StringIO(expected_swift_contents) - location, size, checksum = self.store.add(42, image_swift) + location, size, checksum = self.store.add(42, image_swift, + expected_swift_size) self.assertEquals(expected_location, location) self.assertEquals(expected_swift_size, size) @@ -274,7 +295,8 @@ class TestStore(unittest.TestCase): image_swift = StringIO.StringIO(expected_swift_contents) self.store = Store(new_options) - location, size, checksum = self.store.add(i, image_swift) + location, size, checksum = self.store.add(i, image_swift, + expected_swift_size) self.assertEquals(expected_location, location) self.assertEquals(expected_swift_size, size) @@ -305,7 +327,7 @@ class TestStore(unittest.TestCase): # simply used self.assertRaises here exception_caught = False try: - self.store.add(3, image_swift) + self.store.add(3, image_swift, 0) except BackendException, e: exception_caught = True self.assertTrue("container noexist does not exist " @@ -333,7 +355,53 @@ class TestStore(unittest.TestCase): image_swift = StringIO.StringIO(expected_swift_contents) self.store = Store(options) - location, size, checksum = self.store.add(42, image_swift) + location, size, checksum = self.store.add(42, image_swift, + expected_swift_size) + + self.assertEquals(expected_location, location) + self.assertEquals(expected_swift_size, size) + self.assertEquals(expected_checksum, checksum) + + loc = get_location_from_uri(expected_location) + new_image_swift = self.store.get(loc) + new_image_contents = new_image_swift.getvalue() + new_image_swift_size = new_image_swift.len + + self.assertEquals(expected_swift_contents, new_image_contents) + self.assertEquals(expected_swift_size, new_image_swift_size) + + def test_add_large_object(self): + """ + Tests that adding a very large image. We simulate the large + object by setting the DEFAULT_LARGE_OBJECT_SIZE to a small number + and then verify that there have been a number of calls to + put_object()... + """ + options = SWIFT_OPTIONS.copy() + options['swift_store_container'] = 'glance' + expected_image_id = 42 + expected_swift_size = FIVE_KB + expected_swift_contents = "*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_location = format_swift_location( + options['swift_store_user'], + options['swift_store_key'], + options['swift_store_auth_address'], + options['swift_store_container'], + expected_image_id) + image_swift = StringIO.StringIO(expected_swift_contents) + + orig_max_size = glance.store.swift.DEFAULT_LARGE_OBJECT_SIZE + orig_temp_size = glance.store.swift.DEFAULT_LARGE_OBJECT_CHUNK_SIZE + try: + glance.store.swift.DEFAULT_LARGE_OBJECT_SIZE = 1024 + glance.store.swift.DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 1024 + self.store = Store(options) + location, size, checksum = self.store.add(42, image_swift, + expected_swift_size) + finally: + swift.DEFAULT_LARGE_OBJECT_CHUNK_SIZE = orig_temp_size + swift.DEFAULT_LARGE_OBJECT_SIZE = orig_max_size self.assertEquals(expected_location, location) self.assertEquals(expected_swift_size, size) @@ -355,7 +423,7 @@ class TestStore(unittest.TestCase): image_swift = StringIO.StringIO("nevergonnamakeit") self.assertRaises(exception.Duplicate, self.store.add, - 2, image_swift) + 2, image_swift, 0) def _option_required(self, key): options = SWIFT_OPTIONS.copy()