openstack/glance
Code Issues Proposed changes

Adds ability for Swift to be used as a full-fledged backend.

Browse Source 
Adds POST/PUT capabilities to the SwiftBackend
Adds lots of unit tests for both FilesystemBackend and SwiftBackend
Removes now-unused tests.unit.fakeswifthttp module
This commit is contained in:
jaypipes@gmail.com 2011-02-27 15:54:29 -05:00
parent ed1b5758e0
commit 2cf64655da
10 changed files with 582 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
etc/glance.conf.sample
View File

@ -8,10 +8,6 @@ debug = False
[app:glance-api]
paste.app_factory = glance.server:app_factory
# Directory that the Filesystem backend store
# writes image data to
filesystem_store_datadir=/var/lib/glance/images/
# Which backend store should Glance use by default is not specified
# in a request to add a new image to Glance? Default: 'file'
# Available choices are 'file', 'swift', and 's3'
@ -29,6 +25,32 @@ registry_host = 0.0.0.0
# Port the registry server is listening on
registry_port = 9191
# ============ Filesystem Store Options ========================
# Directory that the Filesystem backend store
# writes image data to
filesystem_store_datadir=/var/lib/glance/images/
# ============ Swift Store Options =============================
# Address where the Swift authentication service lives
swift_store_auth_address = 127.0.0.1:8080
# User to authenticate against the Swift authentication service
swift_store_user = jdoe
# Auth key for the user authenticating against the
# Swift authentication service
swift_store_key = a86850deb2742ec3cb41518e26aa2d89
# Account to use for the user:key Swift auth combination
# for storing images in Swift
swift_store_account = glance
# Container within the account that the account should use
# for storing images in Swift
swift_store_container = glance
[app:glance-registry]
paste.app_factory = glance.registry.server:app_factory

3
glance/server.py
View File

@ -154,7 +154,8 @@ class Controller(wsgi.Controller):
def image_iterator():
chunks = get_from_backend(image['location'],
expected_size=image['size'])
expected_size=image['size'],
options=self.options)
for chunk in chunks:
yield chunk

13
glance/store/filesystem.py
View File

@ -19,12 +19,15 @@
A simple filesystem-backed store
"""
import logging
import os
import urlparse
from glance.common import exception
import glance.store
logger = logging.getLogger('glance.store.filesystem')
class ChunkedFile(object):
@ -60,8 +63,7 @@ class ChunkedFile(object):
class FilesystemBackend(glance.store.Backend):
@classmethod
def get(cls, parsed_uri, opener=lambda p: open(p, "rb"),
expected_size=None):
def get(cls, parsed_uri, expected_size=None, options=None):
""" Filesystem-based backend
file:///path/to/file.tar.gz.0
@ -71,6 +73,8 @@ class FilesystemBackend(glance.store.Backend):
if not os.path.exists(filepath):
raise exception.NotFound("Image file %s not found" % filepath)
else:
logger.debug("Found image at %s. Returning in ChunkedFile.",
filepath)
return ChunkedFile(filepath)
@classmethod
@ -87,6 +91,7 @@ class FilesystemBackend(glance.store.Backend):
fn = parsed_uri.path
if os.path.exists(fn):
try:
logger.debug("Deleting image at %s", fn)
os.unlink(fn)
except OSError:
raise exception.NotAuthorized("You cannot delete file %s" % fn)
@ -112,6 +117,8 @@ class FilesystemBackend(glance.store.Backend):
datadir = options['filesystem_store_datadir']
if not os.path.exists(datadir):
logger.info("Directory to write image files does not exist "
"(%s). Creating.", datadir)
os.makedirs(datadir)
filepath = os.path.join(datadir, str(id))
@ -129,6 +136,8 @@ class FilesystemBackend(glance.store.Backend):
bytes_written += len(buf)
f.write(buf)
logger.debug("Wrote %(bytes_written)d bytes to %(filepath)s"
% locals())
return ('file://%s' % filepath, bytes_written)
@classmethod

149
glance/store/swift.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 OpenStack, LLC
# Copyright 2010-2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -15,20 +15,33 @@
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
"""Storage backend for SWIFT"""
import httplib
import logging
import urllib
from glance.common import config
from glance.common import exception
import glance.store
DEFAULT_SWIFT_ACCOUNT = 'glance'
DEFAULT_SWIFT_CONTAINER = 'glance'
logger = logging.getLogger('glance.store.swift')
class SwiftBackend(glance.store.Backend):
"""
An implementation of the swift backend adapter.
"""
EXAMPLE_URL = "swift://user:password@auth_url/container/file.gz.0"
EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"
CHUNKSIZE = 65536
@classmethod
def get(cls, parsed_uri, expected_size, conn_class=None):
def get(cls, parsed_uri, expected_size=None, options=None,
conn_class=None):
"""
Takes a parsed_uri in the format of:
swift://user:password@auth_url/container/file.gz.0, connects to the
@ -43,12 +56,23 @@ class SwiftBackend(glance.store.Backend):
# snet=True
connection_class = get_connection_class(conn_class)
swift_conn = conn_class(
swift_conn = connection_class(
authurl=authurl, user=user, key=key, snet=False)
try:
(resp_headers, resp_body) = swift_conn.get_object(
container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE)
# TODO(jaypipes) use real exceptions when remove all the cruft
# around importing Swift stuff...
except Exception, e:
if e.http_status == httplib.NOT_FOUND:
location = "swift://%s:%s@%s/%s/%s" % (user, key, authurl,
container, obj)
raise exception.NotFound("Swift could not find image at "
"location %(location)s" % locals())
if expected_size:
obj_size = int(resp_headers['content-length'])
if obj_size != expected_size:
raise glance.store.BackendException(
@ -57,6 +81,105 @@ class SwiftBackend(glance.store.Backend):
return resp_body
@classmethod
def add(cls, id, data, options, conn_class=None):
"""
Stores image data to Swift and returns a location that the image was
written to.
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:param id: The opaque image identifier
:param data: The image data to write, as a file-like object
:param options: Conf mapping
:retval Tuple with (location, size)
The location that was written,
and the size in bytes of the data written
"""
account = options.get('swift_store_account',
DEFAULT_SWIFT_ACCOUNT)
container = options.get('swift_store_container',
DEFAULT_SWIFT_CONTAINER)
auth_address = options.get('swift_store_auth_address')
user = options.get('swift_store_user')
key = options.get('swift_store_key')
# TODO(jaypipes): This needs to be checked every time
# because of the decision to make glance.store.Backend's
# interface all @classmethods. This is inefficient. Backend
# should be a stateful object with options parsed once in
# a constructor.
if not auth_address:
logger.error(msg)
msg = ("Could not find swift_store_auth_address in configuration "
"options.")
raise glance.store.BackendException(msg)
else:
full_auth_address = auth_address
if not full_auth_address.startswith('http'):
full_auth_address = 'https://' + full_auth_address
if not user:
logger.error(msg)
msg = ("Could not find swift_store_user in configuration "
"options.")
raise glance.store.BackendException(msg)
if not key:
logger.error(msg)
msg = ("Could not find swift_store_key in configuration "
"options.")
raise glance.store.BackendException(msg)
connection_class = get_connection_class(conn_class)
swift_conn = connection_class(authurl=full_auth_address, user=user,
key=key, snet=False)
logger.debug("Adding image object to Swift using "
"(auth_address=%(auth_address)s, user=%(user)s, "
"key=%(key)s)")
try:
obj_etag = swift_conn.put_object(container, id, data)
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
location = "swift://%(user)s:%(key)s@%(auth_address)s/"\
"%(container)s/%(id)s" % locals()
# We do a HEAD on the newly-added image to determine the size
# of the image. A bit slow, but better than taking the word
# of the user adding the image with size attribute in the metadata
resp_headers = swift_conn.head_object(container, id)
size = 0
# header keys are lowercased by Swift
if 'content-length' in resp_headers:
size = int(resp_headers['content-length'])
return (location, size)
# TODO(jaypipes) use real exceptions when remove all the cruft
# around importing Swift stuff...
except Exception, e:
if e.http_status == httplib.CONFLICT:
location = "swift://%s:%s@%s/%s/%s" % (user, key, auth_address,
container, id)
raise exception.Duplicate("Swift already has an image at "
"location %(location)s" % locals())
msg = ("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s" % locals())
raise glance.store.BackendException(msg)
@classmethod
def delete(cls, parsed_uri, conn_class=None):
"""
@ -70,14 +193,20 @@ class SwiftBackend(glance.store.Backend):
# snet=True
connection_class = get_connection_class(conn_class)
swift_conn = conn_class(
swift_conn = connection_class(
authurl=authurl, user=user, key=key, snet=False)
(resp_headers, resp_body) = swift_conn.delete_object(
container=container, obj=obj)
try:
swift_conn.delete_object(container, obj)
# TODO(jaypipes): What to return here? After reading the docs
# at swift.common.client, I'm not sure what to check for...
# TODO(jaypipes) use real exceptions when remove all the cruft
# around importing Swift stuff...
except Exception, e:
if e.http_status == httplib.NOT_FOUND:
location = "swift://%s:%s@%s/%s/%s" % (user, key, authurl,
container, obj)
raise exception.NotFound("Swift could not find image at "
"location %(location)s" % locals())
@classmethod
def _parse_swift_tokens(cls, parsed_uri):

29
tests/stubs.py
View File

@ -184,35 +184,6 @@ def stub_out_swift_backend(stubs):
fake_swift_backend.get)
def stub_out_registry(stubs):
"""Stubs out the Registry registry with fake data returns.
The stubbed Registry always returns the following fixture::
{'files': [
{'location': 'file:///chunk0', 'size': 12345},
{'location': 'file:///chunk1', 'size': 1235}
]}
:param stubs: Set of stubout stubs
"""
class FakeRegistry(object):
DATA = \
{'files': [
{'location': 'file:///chunk0', 'size': 12345},
{'location': 'file:///chunk1', 'size': 1235}]}
@classmethod
def lookup(cls, _parsed_uri):
return cls.DATA
fake_registry_registry = FakeRegistry()
stubs.Set(glance.store.registries.Registry, 'lookup',
fake_registry_registry.lookup)
def stub_out_registry_and_store_server(stubs):
"""
Mocks calls to 127.0.0.1 on 9191 and 9292 for testing so

294
tests/unit/swiftfakehttp.py
View File

@ -1,294 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
fakehttp/socket implementation
- TrackerSocket: an object which masquerades as a socket and responds to
requests in a manner consistent with a *very* stupid CloudFS tracker.
- CustomHTTPConnection: an object which subclasses httplib.HTTPConnection
in order to replace it's socket with a TrackerSocket instance.
The unittests each have setup methods which create freerange connection
instances that have had their HTTPConnection instances replaced by
intances of CustomHTTPConnection.
"""
from httplib import HTTPConnection as connbase
import StringIO
class FakeSocket(object):
def __init__(self):
self._rbuffer = StringIO.StringIO()
self._wbuffer = StringIO.StringIO()
def close(self):
pass
def send(self, data, flags=0):
self._rbuffer.write(data)
sendall = send
def recv(self, len=1024, flags=0):
return self._wbuffer(len)
def connect(self):
pass
def makefile(self, mode, flags):
return self._wbuffer
class TrackerSocket(FakeSocket):
def write(self, data):
self._wbuffer.write(data)
def read(self, length=-1):
return self._rbuffer.read(length)
def _create_GET_account_content(self, path, args):
if 'format' in args and args['format'] == 'json':
containers = []
containers.append('[\n')
containers.append('{"name":"container1","count":2,"bytes":78},\n')
containers.append('{"name":"container2","count":1,"bytes":39},\n')
containers.append('{"name":"container3","count":3,"bytes":117}\n')
containers.append(']\n')
elif 'format' in args and args['format'] == 'xml':
containers = []
containers.append('<?xml version="1.0" encoding="UTF-8"?>\n')
containers.append('<account name="FakeAccount">\n')
containers.append('<container><name>container1</name>'
'<count>2</count>'
'<bytes>78</bytes></container>\n')
containers.append('<container><name>container2</name>'
'<count>1</count>'
'<bytes>39</bytes></container>\n')
containers.append('<container><name>container3</name>'
'<count>3</count>'
'<bytes>117</bytes></container>\n')
containers.append('</account>\n')
else:
containers = ['container%s\n' % i for i in range(1, 4)]
return ''.join(containers)
def _create_GET_container_content(self, path, args):
left = 0
right = 9
if 'offset' in args:
left = int(args['offset'])
if 'limit' in args:
right = left + int(args['limit'])
if 'format' in args and args['format'] == 'json':
objects = []
objects.append('{"name":"object1",'
'"hash":"4281c348eaf83e70ddce0e07221c3d28",'
'"bytes":14,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object2",'
'"hash":"b039efe731ad111bc1b0ef221c3849d0",'
'"bytes":64,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object3",'
'"hash":"4281c348eaf83e70ddce0e07221c3d28",'
'"bytes":14,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object4",'
'"hash":"b039efe731ad111bc1b0ef221c3849d0",'
'"bytes":64,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object5",'
'"hash":"4281c348eaf83e70ddce0e07221c3d28",'
'"bytes":14,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object6",'
'"hash":"b039efe731ad111bc1b0ef221c3849d0",'
'"bytes":64,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object7",'
'"hash":"4281c348eaf83e70ddce0e07221c3d28",'
'"bytes":14,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object8",'
'"hash":"b039efe731ad111bc1b0ef221c3849d0",'
'"bytes":64,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
output = '[\n%s\n]\n' % (',\n'.join(objects[left:right]))
elif 'format' in args and args['format'] == 'xml':
objects = []
objects.append('<object><name>object1</name>'
'<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
'<bytes>14</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object2</name>'
'<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
'<bytes>64</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object3</name>'
'<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
'<bytes>14</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object4</name>'
'<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
'<bytes>64</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object5</name>'
'<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
'<bytes>14</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object6</name>'
'<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
'<bytes>64</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object7</name>'
'<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
'<bytes>14</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object8</name>'
'<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
'<bytes>64</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects = objects[left:right]
objects.insert(0, '<?xml version="1.0" encoding="UTF-8"?>\n')
objects.insert(1, '<container name="test_container_1"\n')
objects.append('</container>\n')
output = ''.join(objects)
else:
objects = ['object%s\n' % i for i in range(1, 9)]
objects = objects[left:right]
output = ''.join(objects)
# prefix/path don't make much sense given our test data
if 'prefix' in args or 'path' in args:
pass
return output
def render_GET(self, path, args):
# Special path that returns 404 Not Found
if (len(path) == 4) and (path[3] == 'bogus'):
self.write('HTTP/1.1 404 Not Found\n')
self.write('Content-Type: text/plain\n')
self.write('Content-Length: 0\n')
self.write('Connection: close\n\n')
return
self.write('HTTP/1.1 200 Ok\n')
self.write('Content-Type: text/plain\n')
if len(path) == 2:
content = self._create_GET_account_content(path, args)
elif len(path) == 3:
content = self._create_GET_container_content(path, args)
# Object
elif len(path) == 4:
content = 'I am a teapot, short and stout\n'
self.write('Content-Length: %d\n' % len(content))
self.write('Connection: close\n\n')
self.write(content)
def render_HEAD(self, path, args):
# Account
if len(path) == 2:
self.write('HTTP/1.1 204 No Content\n')
self.write('Content-Type: text/plain\n')
self.write('Connection: close\n')
self.write('X-Account-Container-Count: 3\n')
self.write('X-Account-Bytes-Used: 234\n\n')
else:
self.write('HTTP/1.1 200 Ok\n')
self.write('Content-Type: text/plain\n')
self.write('ETag: d5c7f3babf6c602a8da902fb301a9f27\n')
self.write('Content-Length: 21\n')
self.write('Connection: close\n\n')
def render_POST(self, path, args):
self.write('HTTP/1.1 202 Ok\n')
self.write('Connection: close\n\n')
def render_PUT(self, path, args):
self.write('HTTP/1.1 200 Ok\n')
self.write('Content-Type: text/plain\n')
self.write('Connection: close\n\n')
render_DELETE = render_PUT
def render(self, method, uri):
if '?' in uri:
parts = uri.split('?')
query = parts[1].strip('&').split('&')
args = dict([tuple(i.split('=', 1)) for i in query])
path = parts[0].strip('/').split('/')
else:
args = {}
path = uri.strip('/').split('/')
if hasattr(self, 'render_%s' % method):
getattr(self, 'render_%s' % method)(path, args)
else:
self.write('HTTP/1.1 406 Not Acceptable\n')
self.write('Content-Type: text/plain\n')
self.write('Connection: close\n')
def makefile(self, mode, flags):
self._rbuffer.seek(0)
lines = self.read().splitlines()
(method, uri, version) = lines[0].split()
self.render(method, uri)
self._wbuffer.seek(0)
return self._wbuffer
class CustomHTTPConnection(connbase):
def connect(self):
self.sock = TrackerSocket()
if __name__ == '__main__':
conn = CustomHTTPConnection('localhost', 8000)
conn.request('HEAD', '/v1/account/container/object')
response = conn.getresponse()
print "Status:", response.status, response.reason
for (key, value) in response.getheaders():
print "%s: %s" % (key, value)
print response.read()

135
tests/unit/test_filesystem_store.py Normal file
View File

@ -0,0 +1,135 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests the filesystem backend store"""
import StringIO
import unittest
import urlparse
import stubout
from glance.common import exception
from glance.store.filesystem import FilesystemBackend, ChunkedFile
from tests import stubs
class TestFilesystemBackend(unittest.TestCase):
def setUp(self):
"""Establish a clean test environment"""
self.stubs = stubout.StubOutForTesting()
stubs.stub_out_filesystem_backend()
self.orig_chunksize = ChunkedFile.CHUNKSIZE
ChunkedFile.CHUNKSIZE = 10
def tearDown(self):
"""Clear the test environment"""
stubs.clean_out_fake_filesystem_backend()
self.stubs.UnsetAll()
ChunkedFile.CHUNKSIZE = self.orig_chunksize
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2")
image_file = FilesystemBackend.get(url_pieces)
expected_data = "chunk00000remainder"
expected_num_chunks = 2
data = ""
num_chunks = 0
for chunk in image_file:
num_chunks += 1
data += chunk
self.assertEqual(expected_data, data)
self.assertEqual(expected_num_chunks, num_chunks)
def test_get_non_existing(self):
"""
Test that trying to retrieve a file that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing")
self.assertRaises(exception.NotFound,
FilesystemBackend.get,
url_pieces)
def test_add(self):
"""Test that we can add an image via the filesystem backend"""
ChunkedFile.CHUNKSIZE = 1024
expected_image_id = 42
expected_file_size = 1024 * 5 # 5K
expected_file_contents = "*" * expected_file_size
expected_location = "file://%s/%s" % (stubs.FAKE_FILESYSTEM_ROOTDIR,
expected_image_id)
image_file = StringIO.StringIO(expected_file_contents)
options = {'verbose': True,
'debug': True,
'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR}
location, size = FilesystemBackend.add(42, image_file, options)
self.assertEquals(expected_location, location)
self.assertEquals(expected_file_size, size)
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/42")
new_image_file = FilesystemBackend.get(url_pieces)
new_image_contents = ""
new_image_file_size = 0
for chunk in new_image_file:
new_image_file_size += len(chunk)
new_image_contents += chunk
self.assertEquals(expected_file_contents, new_image_contents)
self.assertEquals(expected_file_size, new_image_file_size)
def test_add_already_existing(self):
"""
Tests that adding an image with an existing identifier
raises an appropriate exception
"""
image_file = StringIO.StringIO("nevergonnamakeit")
options = {'verbose': True,
'debug': True,
'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR}
self.assertRaises(exception.Duplicate,
FilesystemBackend.add,
2, image_file, options)
def test_delete(self):
"""
Test we can delete an existing image in the filesystem store
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2")
FilesystemBackend.delete(url_pieces)
self.assertRaises(exception.NotFound,
FilesystemBackend.get,
url_pieces)
def test_delete_non_existing(self):
"""
Test that trying to delete a file that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing")
self.assertRaises(exception.NotFound,
FilesystemBackend.delete,
url_pieces)

63
tests/unit/test_stores.py
View File

@ -40,27 +40,6 @@ class TestBackend(unittest.TestCase):
self.stubs.UnsetAll()
class TestFilesystemBackend(TestBackend):
def setUp(self):
"""Establish a clean test environment"""
stubs.stub_out_filesystem_backend()
def tearDown(self):
"""Clear the test environment"""
stubs.clean_out_fake_filesystem_backend()
def test_get(self):
fetcher = get_from_backend("file:///tmp/glance-tests/2",
expected_size=19)
data = ""
for chunk in fetcher:
data += chunk
self.assertEqual(data, "chunk00000remainder")
class TestHTTPBackend(TestBackend):
def setUp(self):
@ -104,45 +83,3 @@ class TestS3Backend(TestBackend):
chunks = [c for c in fetcher]
self.assertEqual(chunks, expected_returns)
class TestSwiftBackend(TestBackend):
def setUp(self):
super(TestSwiftBackend, self).setUp()
stubs.stub_out_swift_backend(self.stubs)
def test_get(self):
swift_uri = "swift://user:pass@localhost/container1/file.tar.gz"
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
fetcher = get_from_backend(swift_uri,
expected_size=21,
conn_class=SwiftBackend)
chunks = [c for c in fetcher]
self.assertEqual(chunks, expected_returns)
def test_get_bad_uri(self):
swift_url = "swift://localhost/container1/file.tar.gz"
self.assertRaises(BackendException, get_from_backend,
swift_url, expected_size=21)
def test_url_parsing(self):
swift_uri = "swift://user:pass@localhost/v1.0/container1/file.tar.gz"
parsed_uri = urlparse.urlparse(swift_uri)
(user, key, authurl, container, obj) = \
SwiftBackend._parse_swift_tokens(parsed_uri)
self.assertEqual(user, 'user')
self.assertEqual(key, 'pass')
self.assertEqual(authurl, 'https://localhost/v1.0')
self.assertEqual(container, 'container1')
self.assertEqual(obj, 'file.tar.gz')

261
tests/unit/test_swift_store.py Normal file
View File

@ -0,0 +1,261 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this swift except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests the Swift backend store"""
import StringIO
import hashlib
import httplib
import sys
import unittest
import urlparse
import stubout
from glance.common import exception
import glance.store.swift
SwiftBackend = glance.store.swift.SwiftBackend
SWIFT_INSTALLED = False
try:
import swift.common.client
SWIFT_INSTALLED = True
except ImportError:
print "Skipping Swift store tests since Swift is not installed."
FIVE_KB = (5 * 1024)
SWIFT_OPTIONS = {'verbose': True,
'debug': True,
'swift_store_user': 'glance',
'swift_store_key': 'key',
'swift_store_auth_address': 'localhost:8080',
'swift_store_container': 'glance'}
# We stub out as little as possible to ensure that the code paths
# between glance.store.swift and swift.common.client are tested
# thoroughly
def stub_out_swift_common_client(stubs):
fixture_headers = {'glance/2':
{'content-length': FIVE_KB,
'etag': 'c2e5db72bd7fd153f53ede5da5a06de3'}}
fixture_objects = {'glance/2':
StringIO.StringIO("*" * FIVE_KB)}
def fake_put_object(url, token, container, name, contents, **kwargs):
# PUT returns the ETag header for the newly-added object
fixture_key = "%s/%s" % (container, name)
if not fixture_key in fixture_headers.keys():
if hasattr(contents, 'read'):
fixture_object = StringIO.StringIO()
chunk = contents.read(SwiftBackend.CHUNKSIZE)
while chunk:
fixture_object.write(chunk)
chunk = contents.read(SwiftBackend.CHUNKSIZE)
else:
fixture_object = StringIO.StringIO(contents)
fixture_objects[fixture_key] = fixture_object
fixture_headers[fixture_key] = {
'content-length': fixture_object.len,
'etag': hashlib.md5(fixture_object.read()).hexdigest()}
return fixture_headers[fixture_key]['etag']
else:
msg = ("Object PUT failed - Object with key %s already exists"
% fixture_key)
raise swift.common.client.ClientException(msg,
http_status=httplib.CONFLICT)
def fake_get_object(url, token, container, name, **kwargs):
# GET returns the tuple (list of headers, file object)
try:
fixture_key = "%s/%s" % (container, name)
return fixture_headers[fixture_key], fixture_objects[fixture_key]
except KeyError:
msg = "Object GET failed"
raise swift.common.client.ClientException(msg,
http_status=httplib.NOT_FOUND)
def fake_head_object(url, token, container, name, **kwargs):
# HEAD returns the list of headers for an object
try:
fixture_key = "%s/%s" % (container, name)
return fixture_headers[fixture_key]
except KeyError:
msg = "Object HEAD failed - Object does not exist"
raise swift.common.client.ClientException(msg,
http_status=httplib.NOT_FOUND)
def fake_delete_object(url, token, container, name, **kwargs):
# DELETE returns nothing
fixture_key = "%s/%s" % (container, name)
if fixture_key not in fixture_headers.keys():
msg = "Object DELETE failed - Object does not exist"
raise swift.common.client.ClientException(msg,
http_status=httplib.NOT_FOUND)
else:
del fixture_headers[fixture_key]
del fixture_objects[fixture_key]
def fake_get_connection_class(*args):
return swift.common.client.Connection
def fake_http_connection(self):
return None
def fake_get_auth(self):
return None, None
stubs.Set(swift.common.client,
'put_object', fake_put_object)
stubs.Set(swift.common.client,
'delete_object', fake_delete_object)
stubs.Set(swift.common.client,
'head_object', fake_head_object)
stubs.Set(swift.common.client,
'get_object', fake_get_object)
stubs.Set(swift.common.client.Connection,
'get_auth', fake_get_auth)
stubs.Set(swift.common.client.Connection,
'http_connection', fake_http_connection)
stubs.Set(glance.store.swift,
'get_connection_class', fake_get_connection_class)
class TestSwiftBackend(unittest.TestCase):
def setUp(self):
"""Establish a clean test environment"""
self.stubs = stubout.StubOutForTesting()
if SWIFT_INSTALLED:
stub_out_swift_common_client(self.stubs)
def tearDown(self):
"""Clear the test environment"""
self.stubs.UnsetAll()
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
if not SWIFT_INSTALLED:
return
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
image_swift = SwiftBackend.get(url_pieces)
expected_data = "*" * FIVE_KB
data = ""
for chunk in image_swift:
data += chunk
self.assertEqual(expected_data, data)
def test_get_mismatched_expected_size(self):
"""
Test retrieval of an image with wrong expected_size param
raises an exception
"""
if not SWIFT_INSTALLED:
return
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
self.assertRaises(glance.store.BackendException,
SwiftBackend.get,
url_pieces,
{'expected_size': 42})
def test_get_non_existing(self):
"""
Test that trying to retrieve a swift that doesn't exist
raises an error
"""
if not SWIFT_INSTALLED:
return
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/noexist")
self.assertRaises(exception.NotFound,
SwiftBackend.get,
url_pieces)
def test_add(self):
"""Test that we can add an image via the swift backend"""
if not SWIFT_INSTALLED:
return
expected_image_id = 42
expected_swift_size = 1024 * 5 # 5K
expected_swift_contents = "*" * expected_swift_size
expected_location = "swift://%s:%s@%s/%s/%s" % (
SWIFT_OPTIONS['swift_store_user'],
SWIFT_OPTIONS['swift_store_key'],
SWIFT_OPTIONS['swift_store_auth_address'],
SWIFT_OPTIONS['swift_store_container'],
expected_image_id)
image_swift = StringIO.StringIO(expected_swift_contents)
location, size = SwiftBackend.add(42, image_swift, SWIFT_OPTIONS)
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/42")
new_image_swift = SwiftBackend.get(url_pieces)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
self.assertEquals(expected_swift_contents, new_image_contents)
self.assertEquals(expected_swift_size, new_image_swift_size)
def test_add_already_existing(self):
"""
Tests that adding an image with an existing identifier
raises an appropriate exception
"""
if not SWIFT_INSTALLED:
return
image_swift = StringIO.StringIO("nevergonnamakeit")
self.assertRaises(exception.Duplicate,
SwiftBackend.add,
2, image_swift, SWIFT_OPTIONS)
def test_delete(self):
"""
Test we can delete an existing image in the swift store
"""
if not SWIFT_INSTALLED:
return
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
SwiftBackend.delete(url_pieces)
self.assertRaises(exception.NotFound,
SwiftBackend.get,
url_pieces)
def test_delete_non_existing(self):
"""
Test that trying to delete a swift that doesn't exist
raises an error
"""
if not SWIFT_INSTALLED:
return
url_pieces = urlparse.urlparse("swift://user:key@auth_address/noexist")
self.assertRaises(exception.NotFound,
SwiftBackend.delete,
url_pieces)

1
tools/pip-requires
View File

@ -12,5 +12,6 @@ nose
sphinx
argparse
mox==0.5.0
swift
-f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz
sqlalchemy-migrate>=0.6