openstack/glance
Code Issues Proposed changes

Fixes LP Bug #827660 - Swift driver fail 5G upload

Browse Source 
Fixes LP Bug #827660 - Swift driver fails to
upload image files greater than 5GB. The swift
store driver was not using the large object
manifest methods required to upload objects
greater than 5GB into Swift.

Adds configuration options to specify the
size in MB that an image file should be considered
for large object manifest, and a configurable
chunk size. We use a disk buffer of this size,
adding chunks of the image file into Swift and
then after all chunks are saved, add the manifest
to Swift.

Change-Id: I4b4421aa1bba584ab242016a041e864ef19d0214
This commit is contained in:
Jay Pipes
2011-08-22 14:43:27 -04:00
parent bcc7ae3b73
commit 501b14bf94
14 changed files with 472 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
doc/source/configuring.rst
View File

@@ -225,6 +225,31 @@ Can only be specified in configuration files.
If true, Glance will attempt to create the container ``swift_store_container``
if it does not exist.
* ``swift_store_large_object_size=SIZE_IN_MB``
Optional. Default: ``5120``
Can only be specified in configuration files.
`This option is specific to the Swift storage backend.`
What size, in MB, should Glance start chunking image files
and do a large object manifest in Swift? By default, this is
the maximum object size in Swift, which is 5GB
* ``swift_store_large_object_chunk_size=SIZE_IN_MB``
Optional. Default: ``200``
Can only be specified in configuration files.
`This option is specific to the Swift storage backend.`
When doing a large object manifest, what size, in MB, should
Glance write chunks to Swift? This amount of data is written
to a temporary disk buffer during the process of chunking
the image file, and the default is 200MB
Configuring the S3 Storage Backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

13
etc/glance-api.conf
View File

@@ -29,6 +29,8 @@ log_file = /var/log/glance/api.log
# Send logs to syslog (/dev/log) instead of to file specified by `log_file`
use_syslog = False
# ============ Notification System Options =====================
# Notifications can be sent when images are create, updated or deleted.
# There are three methods of sending notifications, logging (via the
# log_file directive), rabbit (via a rabbitmq queue) or noop (no
@@ -70,6 +72,17 @@ swift_store_container = glance
# Do we create the container if it does not exist?
swift_store_create_container_on_put = False
# What size, in MB, should Glance start chunking image files
# and do a large object manifest in Swift? By default, this is
# the maximum object size in Swift, which is 5GB
swift_store_large_object_size = 5120
# When doing a large object manifest, what size, in MB, should
# Glance write chunks to Swift? This amount of data is written
# to a temporary disk buffer during the process of chunking
# the image file, and the default is 200MB
swift_store_large_object_chunk_size = 200
# Whether to use ServiceNET to communicate with the Swift storage servers.
# (If you aren't RACKSPACE, leave this False!)
#

11
glance/api/v1/images.py
View File

@@ -343,8 +343,17 @@ class Controller(api.BaseController):
try:
logger.debug(_("Uploading image data for image %(image_id)s "
"to %(store_name)s store"), locals())
if req.content_length:
image_size = int(req.content_length)
elif 'x-image-meta-size' in req.headers:
image_size = int(req.headers['x-image-meta-size'])
else:
logger.debug(_("Got request with no content-length and no "
"x-image-meta-size header"))
image_size = 0
location, size, checksum = store.add(image_meta['id'],
req.body_file)
req.body_file,
image_size)
# Verify any supplied checksum value matches checksum
# returned from store when adding image

20
glance/client.py
View File

@@ -19,7 +19,9 @@
Client classes for callers of a Glance system
"""
import errno
import json
import os
from glance.api.v1 import images as v1_images
from glance.common import client as base_client
@@ -131,6 +133,24 @@ class V1Client(base_client.BaseClient):
if image_data:
body = image_data
headers['content-type'] = 'application/octet-stream'
# For large images, we need to supply the size of the
# image file. See LP Bug #827660.
if hasattr(image_data, 'seek') and hasattr(image_data, 'tell'):
try:
image_data.seek(0, os.SEEK_END)
image_size = image_data.tell()
image_data.seek(0)
headers['x-image-meta-size'] = image_size
headers['content-length'] = image_size
except IOError, e:
if e.errno == errno.ESPIPE:
# Illegal seek. This means the user is trying
# to pipe image data to the client, e.g.
# echo testdata | bin/glance add blah..., or
# that stdin is empty
pass
else:
raise
else:
body = None

3
glance/store/base.py
View File

@@ -72,7 +72,7 @@ class Store(object):
"""
raise exception.StoreAddDisabled
def add(self, image_id, image_file):
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
@@ -80,6 +80,7 @@ class Store(object):
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already

3
glance/store/filesystem.py
View File

@@ -165,7 +165,7 @@ class Store(glance.store.base.Store):
else:
raise exception.NotFound(_("Image file %s does not exist") % fn)
def add(self, image_id, image_file):
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
@@ -173,6 +173,7 @@ class Store(glance.store.base.Store):
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already

3
glance/store/s3.py
View File

@@ -255,7 +255,7 @@ class Store(glance.store.base.Store):
key.BufferSize = self.CHUNKSIZE
return ChunkedFile(key)
def add(self, image_id, image_file):
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
@@ -263,6 +263,7 @@ class Store(glance.store.base.Store):
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already

109
glance/store/swift.py
View File

@@ -19,8 +19,11 @@
from __future__ import absolute_import
import hashlib
import httplib
import logging
import math
import tempfile
import urlparse
from glance.common import config
@@ -34,7 +37,9 @@ try:
except ImportError:
pass
DEFAULT_SWIFT_CONTAINER = 'glance'
DEFAULT_CONTAINER = 'glance'
DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 # 5GB
DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 * 1024 * 1024 # 200M
logger = logging.getLogger('glance.store.swift')
@@ -189,7 +194,26 @@ class Store(glance.store.base.Store):
self.user = self._option_get('swift_store_user')
self.key = self._option_get('swift_store_key')
self.container = self.options.get('swift_store_container',
DEFAULT_SWIFT_CONTAINER)
DEFAULT_CONTAINER)
try:
if self.options.get('swift_store_large_object_size'):
self.large_object_size = int(
self.options.get('swift_store_large_object_size')
) * (1024 * 1024) # Size specified in MB in conf files
else:
self.large_object_size = DEFAULT_LARGE_OBJECT_SIZE
if self.options.get('swift_store_large_object_chunk_size'):
self.large_object_chunk_size = int(
self.options.get('swift_store_large_object_chunk_size')
) * (1024 * 1024) # Size specified in MB in conf files
else:
self.large_object_chunk_size = DEFAULT_LARGE_OBJECT_CHUNK_SIZE
except Exception, e:
reason = _("Error in configuration options: %s") % e
logger.error(reason)
raise exception.BadStoreConfiguration(store_name="swift",
reason=reason)
self.scheme = 'swift+https'
if self.auth_address.startswith('http://'):
@@ -259,7 +283,7 @@ class Store(glance.store.base.Store):
reason=reason)
return result
def add(self, image_id, image_file):
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
@@ -267,6 +291,7 @@ class Store(glance.store.base.Store):
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
@@ -284,6 +309,11 @@ class Store(glance.store.base.Store):
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:note Swift cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Swift and then write an manifest
to Swift that contains information about the chunks.
"""
swift_conn = self._make_swift_connection(
auth_url=self.full_auth_address, user=self.user, key=self.key)
@@ -301,8 +331,67 @@ class Store(glance.store.base.Store):
logger.debug(_("Adding image object '%(obj_name)s' "
"to Swift") % locals())
try:
obj_etag = swift_conn.put_object(self.container, obj_name,
image_file)
if image_size < self.large_object_size:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size, and we simply
# try to put the object into Swift and hope for the
# best...
obj_etag = swift_conn.put_object(self.container, obj_name,
image_file)
else:
# Write the image into Swift in chunks. We cannot
# stream chunks of the webob.Request.body_file, unfortunately,
# so we must write chunks of the body_file into a temporary
# disk buffer, and then pass this disk buffer to Swift.
bytes_left = image_size
chunk_id = 1
total_chunks = int(math.ceil(
float(image_size) / float(self.large_object_chunk_size)))
checksum = hashlib.md5()
while bytes_left > 0:
with tempfile.NamedTemporaryFile() as disk_buffer:
chunk_size = min(self.large_object_chunk_size,
bytes_left)
logger.debug(_("Writing %(chunk_size)d bytes for "
"chunk %(chunk_id)d/"
"%(total_chunks)d to disk buffer "
"for image %(image_id)s")
% locals())
chunk = image_file.read(chunk_size)
checksum.update(chunk)
disk_buffer.write(chunk)
disk_buffer.flush()
logger.debug(_("Writing chunk %(chunk_id)d/"
"%(total_chunks)d to Swift "
"for image %(image_id)s")
% locals())
chunk_etag = swift_conn.put_object(
self.container,
"%s-%05d" % (obj_name, chunk_id),
open(disk_buffer.name, 'rb'))
logger.debug(_("Wrote chunk %(chunk_id)d/"
"%(total_chunks)d to Swift "
"returning MD5 of content: "
"%(chunk_etag)s")
% locals())
bytes_left -= self.large_object_chunk_size
chunk_id += 1
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (self.container, obj_name)
headers = {'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
_ignored = swift_conn.put_object(self.container, obj_name,
None, headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
@@ -310,15 +399,7 @@ class Store(glance.store.base.Store):
# the location attribute from GET /images/<ID> and
# GET /images/details
# We do a HEAD on the newly-added image to determine the size
# of the image. A bit slow, but better than taking the word
# of the user adding the image with size attribute in the metadata
resp_headers = swift_conn.head_object(self.container, obj_name)
size = 0
# header keys are lowercased by Swift
if 'content-length' in resp_headers:
size = int(resp_headers['content-length'])
return (location.get_uri(), size, obj_etag)
return (location.get_uri(), image_size, obj_etag)
except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Swift already has an image at "

12
glance/tests/functional/__init__.py
View File

@@ -141,6 +141,12 @@ class ApiServer(Server):
self.s3_store_access_key = ""
self.s3_store_secret_key = ""
self.s3_store_bucket = ""
self.swift_store_auth_address = ""
self.swift_store_user = ""
self.swift_store_key = ""
self.swift_store_container = ""
self.swift_store_large_object_size = 5 * 1024
self.swift_store_large_object_chunk_size = 200
self.delayed_delete = delayed_delete
self.conf_base = """[DEFAULT]
verbose = %(verbose)s
@@ -156,6 +162,12 @@ s3_store_host = %(s3_store_host)s
s3_store_access_key = %(s3_store_access_key)s
s3_store_secret_key = %(s3_store_secret_key)s
s3_store_bucket = %(s3_store_bucket)s
swift_store_auth_address = %(swift_store_auth_address)s
swift_store_user = %(swift_store_user)s
swift_store_key = %(swift_store_key)s
swift_store_container = %(swift_store_container)s
swift_store_large_object_size = %(swift_store_large_object_size)s
swift_store_large_object_chunk_size = %(swift_store_large_object_chunk_size)s
delayed_delete = %(delayed_delete)s
[pipeline:glance-api]

17
glance/tests/functional/test_bin_glance.py
View File

@@ -55,10 +55,14 @@ class TestBinGlance(functional.FunctionalTest):
self.assertEqual('', out.strip())
# 1. Add public image
cmd = "echo testdata | bin/glance --port=%d add is_public=True"\
" name=MyImage" % api_port
with tempfile.NamedTemporaryFile() as image_file:
image_file.write("XXX")
image_file.flush()
image_file_name = image_file.name
cmd = "bin/glance --port=%d add is_public=True"\
" name=MyImage < %s" % (api_port, image_file_name)
exitcode, out, err = execute(cmd)
exitcode, out, err = execute(cmd)
self.assertEqual(0, exitcode)
self.assertEqual('Added new image with ID: 1', out.strip())
@@ -78,9 +82,10 @@ class TestBinGlance(functional.FunctionalTest):
[c.strip() for c in line.split()]
self.assertEqual('MyImage', name)
self.assertEqual('9', size,
"Expected image to be 9 bytes in size. Make sure"
" you're running the correct version of webob.")
self.assertEqual('3', size,
"Expected image to be 3 bytes in size, but got %s. "
"Make sure you're running the correct version "
"of webob." % size)
# 3. Delete the image
cmd = "bin/glance --port=%d --force delete 1" % api_port

194
glance/tests/functional/test_swift.py
View File

@@ -31,6 +31,9 @@ skipped.
"""
import ConfigParser
import hashlib
import httplib
import httplib2
import json
import os
import tempfile
@@ -39,6 +42,8 @@ import unittest
from glance.tests.functional import test_api
from glance.tests.utils import execute, skip_if_disabled
FIVE_MB = 5 * 1024 * 1024
class TestSwift(test_api.TestApi):
@@ -137,5 +142,192 @@ class TestSwift(test_api.TestApi):
super(TestSwift, self).tearDown()
def clear_container(self):
self.swift_conn.delete_container(self.swift_store_container)
from swift.common import client as swift_client
try:
self.swift_conn.delete_container(self.swift_store_container)
except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
pass
else:
raise
self.swift_conn.put_container(self.swift_store_container)
@skip_if_disabled
def test_add_large_object_manifest(self):
"""
We test the large object manifest code path in the Swift driver.
In the case where an image file is bigger than the config variable
swift_store_large_object_size, then we chunk the image into
Swift, and add a manifest put_object at the end.
"""
self.cleanup()
self.swift_store_large_object_size = 2 # In MB
self.swift_store_large_object_chunk_size = 1 # In MB
self.start_servers(**self.__dict__.copy())
api_port = self.api_port
registry_port = self.registry_port
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with public image named Image1
# attribute and no custom properties. Verify a 200 OK is returned
image_data = "*" * FIVE_MB
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers,
body=image_data)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_MB)
self.assertEqual(data['image']['name'], "Image1")
self.assertEqual(data['image']['is_public'], True)
# 4. HEAD /images/1
# Verify image found now
path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'HEAD')
self.assertEqual(response.status, 200)
self.assertEqual(response['x-image-meta-name'], "Image1")
# 5. GET /images/1
# Verify all information on image we just added is correct
path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
expected_image_headers = {
'x-image-meta-id': '1',
'x-image-meta-name': 'Image1',
'x-image-meta-is_public': 'True',
'x-image-meta-status': 'active',
'x-image-meta-disk_format': '',
'x-image-meta-container_format': '',
'x-image-meta-size': str(FIVE_MB)
}
expected_std_headers = {
'content-length': str(FIVE_MB),
'content-type': 'application/octet-stream'}
for expected_key, expected_value in expected_image_headers.items():
self.assertEqual(response[expected_key], expected_value,
"For key '%s' expected header value '%s'. Got '%s'"
% (expected_key, expected_value,
response[expected_key]))
for expected_key, expected_value in expected_std_headers.items():
self.assertEqual(response[expected_key], expected_value,
"For key '%s' expected header value '%s'. Got '%s'"
% (expected_key,
expected_value,
response[expected_key]))
self.assertEqual(content, "*" * FIVE_MB)
self.assertEqual(hashlib.md5(content).hexdigest(),
hashlib.md5("*" * FIVE_MB).hexdigest())
self.stop_servers()
@skip_if_disabled
def test_add_large_object_manifest_uneven_size(self):
"""
Test when large object manifest in scenario where
image size % chunk size != 0
"""
self.cleanup()
self.swift_store_large_object_size = 3 # In MB
self.swift_store_large_object_chunk_size = 2 # In MB
self.start_servers(**self.__dict__.copy())
api_port = self.api_port
registry_port = self.registry_port
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with public image named Image1
# attribute and no custom properties. Verify a 200 OK is returned
image_data = "*" * FIVE_MB
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers,
body=image_data)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_MB)
self.assertEqual(data['image']['name'], "Image1")
self.assertEqual(data['image']['is_public'], True)
# 4. HEAD /images/1
# Verify image found now
path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'HEAD')
self.assertEqual(response.status, 200)
self.assertEqual(response['x-image-meta-name'], "Image1")
# 5. GET /images/1
# Verify all information on image we just added is correct
path = "http://%s:%d/v1/images/1" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
expected_image_headers = {
'x-image-meta-id': '1',
'x-image-meta-name': 'Image1',
'x-image-meta-is_public': 'True',
'x-image-meta-status': 'active',
'x-image-meta-disk_format': '',
'x-image-meta-container_format': '',
'x-image-meta-size': str(FIVE_MB)
}
expected_std_headers = {
'content-length': str(FIVE_MB),
'content-type': 'application/octet-stream'}
for expected_key, expected_value in expected_image_headers.items():
self.assertEqual(response[expected_key], expected_value,
"For key '%s' expected header value '%s'. Got '%s'"
% (expected_key, expected_value,
response[expected_key]))
for expected_key, expected_value in expected_std_headers.items():
self.assertEqual(response[expected_key], expected_value,
"For key '%s' expected header value '%s'. Got '%s'"
% (expected_key,
expected_value,
response[expected_key]))
self.assertEqual(content, "*" * FIVE_MB)
self.assertEqual(hashlib.md5(content).hexdigest(),
hashlib.md5("*" * FIVE_MB).hexdigest())
self.stop_servers()

5
glance/tests/unit/test_filesystem_store.py
View File

@@ -87,7 +87,8 @@ class TestStore(unittest.TestCase):
expected_image_id)
image_file = StringIO.StringIO(expected_file_contents)
location, size, checksum = self.store.add(42, image_file)
location, size, checksum = self.store.add(42, image_file,
expected_file_size)
self.assertEquals(expected_location, location)
self.assertEquals(expected_file_size, size)
@@ -116,7 +117,7 @@ class TestStore(unittest.TestCase):
'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR}
self.assertRaises(exception.Duplicate,
self.store.add,
2, image_file)
2, image_file, 0)
def test_delete(self):
"""

8
glance/tests/unit/test_s3_store.py
View File

@@ -209,7 +209,8 @@ class TestStore(unittest.TestCase):
expected_image_id)
image_s3 = StringIO.StringIO(expected_s3_contents)
location, size, checksum = self.store.add(42, image_s3)
location, size, checksum = self.store.add(42, image_s3,
expected_s3_size)
self.assertEquals(expected_location, location)
self.assertEquals(expected_s3_size, size)
@@ -258,7 +259,8 @@ class TestStore(unittest.TestCase):
image_s3 = StringIO.StringIO(expected_s3_contents)
self.store = Store(new_options)
location, size, checksum = self.store.add(i, image_s3)
location, size, checksum = self.store.add(i, image_s3,
expected_s3_size)
self.assertEquals(expected_location, location)
self.assertEquals(expected_s3_size, size)
@@ -281,7 +283,7 @@ class TestStore(unittest.TestCase):
image_s3 = StringIO.StringIO("nevergonnamakeit")
self.assertRaises(exception.Duplicate,
self.store.add,
2, image_s3)
2, image_s3, 0)
def _option_required(self, key):
options = S3_OPTIONS.copy()

90
glance/tests/unit/test_swift_store.py
View File

@@ -29,9 +29,10 @@ import swift.common.client
from glance.common import exception
from glance.store import BackendException
from glance.store.swift import Store
import glance.store.swift
from glance.store.location import get_location_from_uri
Store = glance.store.swift.Store
FIVE_KB = (5 * 1024)
SWIFT_OPTIONS = {'verbose': True,
'debug': True,
@@ -64,7 +65,13 @@ def stub_out_swift_common_client(stubs):
def fake_put_object(url, token, container, name, contents, **kwargs):
# PUT returns the ETag header for the newly-added object
# Large object manifest...
fixture_key = "%s/%s" % (container, name)
if kwargs.get('headers'):
etag = kwargs['headers']['ETag']
fixture_headers[fixture_key] = {'manifest': True,
'etag': etag}
return etag
if not fixture_key in fixture_headers.keys():
if hasattr(contents, 'read'):
fixture_object = StringIO.StringIO()
@@ -83,7 +90,7 @@ def stub_out_swift_common_client(stubs):
fixture_headers[fixture_key] = {
'content-length': read_len,
'etag': etag}
return fixture_headers[fixture_key]['etag']
return etag
else:
msg = ("Object PUT failed - Object with key %s already exists"
% fixture_key)
@@ -92,14 +99,27 @@ def stub_out_swift_common_client(stubs):
def fake_get_object(url, token, container, name, **kwargs):
# GET returns the tuple (list of headers, file object)
try:
fixture_key = "%s/%s" % (container, name)
return fixture_headers[fixture_key], fixture_objects[fixture_key]
except KeyError:
fixture_key = "%s/%s" % (container, name)
if not fixture_key in fixture_headers:
msg = "Object GET failed"
raise swift.common.client.ClientException(msg,
http_status=httplib.NOT_FOUND)
fixture = fixture_headers[fixture_key]
if 'manifest' in fixture:
# Large object manifest... we return a file containing
# all objects with prefix of this fixture key
chunk_keys = sorted([k for k in fixture_headers.keys()
if k.startswith(fixture_key) and
k != fixture_key])
result = StringIO.StringIO()
for key in chunk_keys:
result.write(fixture_objects[key].getvalue())
return fixture_headers[fixture_key], result
else:
return fixture_headers[fixture_key], fixture_objects[fixture_key]
def fake_head_object(url, token, container, name, **kwargs):
# HEAD returns the list of headers for an object
try:
@@ -227,7 +247,8 @@ class TestStore(unittest.TestCase):
expected_image_id)
image_swift = StringIO.StringIO(expected_swift_contents)
location, size, checksum = self.store.add(42, image_swift)
location, size, checksum = self.store.add(42, image_swift,
expected_swift_size)
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
@@ -274,7 +295,8 @@ class TestStore(unittest.TestCase):
image_swift = StringIO.StringIO(expected_swift_contents)
self.store = Store(new_options)
location, size, checksum = self.store.add(i, image_swift)
location, size, checksum = self.store.add(i, image_swift,
expected_swift_size)
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
@@ -305,7 +327,7 @@ class TestStore(unittest.TestCase):
# simply used self.assertRaises here
exception_caught = False
try:
self.store.add(3, image_swift)
self.store.add(3, image_swift, 0)
except BackendException, e:
exception_caught = True
self.assertTrue("container noexist does not exist "
@@ -333,7 +355,53 @@ class TestStore(unittest.TestCase):
image_swift = StringIO.StringIO(expected_swift_contents)
self.store = Store(options)
location, size, checksum = self.store.add(42, image_swift)
location, size, checksum = self.store.add(42, image_swift,
expected_swift_size)
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
self.assertEquals(expected_swift_contents, new_image_contents)
self.assertEquals(expected_swift_size, new_image_swift_size)
def test_add_large_object(self):
"""
Tests that adding a very large image. We simulate the large
object by setting the DEFAULT_LARGE_OBJECT_SIZE to a small number
and then verify that there have been a number of calls to
put_object()...
"""
options = SWIFT_OPTIONS.copy()
options['swift_store_container'] = 'glance'
expected_image_id = 42
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_location = format_swift_location(
options['swift_store_user'],
options['swift_store_key'],
options['swift_store_auth_address'],
options['swift_store_container'],
expected_image_id)
image_swift = StringIO.StringIO(expected_swift_contents)
orig_max_size = glance.store.swift.DEFAULT_LARGE_OBJECT_SIZE
orig_temp_size = glance.store.swift.DEFAULT_LARGE_OBJECT_CHUNK_SIZE
try:
glance.store.swift.DEFAULT_LARGE_OBJECT_SIZE = 1024
glance.store.swift.DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 1024
self.store = Store(options)
location, size, checksum = self.store.add(42, image_swift,
expected_swift_size)
finally:
swift.DEFAULT_LARGE_OBJECT_CHUNK_SIZE = orig_temp_size
swift.DEFAULT_LARGE_OBJECT_SIZE = orig_max_size
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
@@ -355,7 +423,7 @@ class TestStore(unittest.TestCase):
image_swift = StringIO.StringIO("nevergonnamakeit")
self.assertRaises(exception.Duplicate,
self.store.add,
2, image_swift)
2, image_swift, 0)
def _option_required(self, key):
options = SWIFT_OPTIONS.copy()