From 2cf64655da38d228b8892a86f005353c84ab212b Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Sun, 27 Feb 2011 15:54:29 -0500 Subject: [PATCH] Adds ability for Swift to be used as a full-fledged backend. Adds POST/PUT capabilities to the SwiftBackend Adds lots of unit tests for both FilesystemBackend and SwiftBackend Removes now-unused tests.unit.fakeswifthttp module --- etc/glance.conf.sample | 30 ++- glance/server.py | 3 +- glance/store/filesystem.py | 13 +- glance/store/swift.py | 163 +++++++++++++-- tests/stubs.py | 29 --- tests/unit/swiftfakehttp.py | 294 ---------------------------- tests/unit/test_filesystem_store.py | 135 +++++++++++++ tests/unit/test_stores.py | 63 ------ tests/unit/test_swift_store.py | 261 ++++++++++++++++++++++++ tools/pip-requires | 1 + 10 files changed, 582 insertions(+), 410 deletions(-) delete mode 100644 tests/unit/swiftfakehttp.py create mode 100644 tests/unit/test_filesystem_store.py create mode 100644 tests/unit/test_swift_store.py diff --git a/etc/glance.conf.sample b/etc/glance.conf.sample index d2eafce629..b9f7fdd48d 100644 --- a/etc/glance.conf.sample +++ b/etc/glance.conf.sample @@ -8,10 +8,6 @@ debug = False [app:glance-api] paste.app_factory = glance.server:app_factory -# Directory that the Filesystem backend store -# writes image data to -filesystem_store_datadir=/var/lib/glance/images/ - # Which backend store should Glance use by default is not specified # in a request to add a new image to Glance? Default: 'file' # Available choices are 'file', 'swift', and 's3' @@ -29,6 +25,32 @@ registry_host = 0.0.0.0 # Port the registry server is listening on registry_port = 9191 +# ============ Filesystem Store Options ======================== + +# Directory that the Filesystem backend store +# writes image data to +filesystem_store_datadir=/var/lib/glance/images/ + +# ============ Swift Store Options ============================= + +# Address where the Swift authentication service lives +swift_store_auth_address = 127.0.0.1:8080 + +# User to authenticate against the Swift authentication service +swift_store_user = jdoe + +# Auth key for the user authenticating against the +# Swift authentication service +swift_store_key = a86850deb2742ec3cb41518e26aa2d89 + +# Account to use for the user:key Swift auth combination +# for storing images in Swift +swift_store_account = glance + +# Container within the account that the account should use +# for storing images in Swift +swift_store_container = glance + [app:glance-registry] paste.app_factory = glance.registry.server:app_factory diff --git a/glance/server.py b/glance/server.py index 35ee5ca309..2bd5726aee 100644 --- a/glance/server.py +++ b/glance/server.py @@ -154,7 +154,8 @@ class Controller(wsgi.Controller): def image_iterator(): chunks = get_from_backend(image['location'], - expected_size=image['size']) + expected_size=image['size'], + options=self.options) for chunk in chunks: yield chunk diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py index 3608bef04c..1619c45629 100644 --- a/glance/store/filesystem.py +++ b/glance/store/filesystem.py @@ -19,12 +19,15 @@ A simple filesystem-backed store """ +import logging import os import urlparse from glance.common import exception import glance.store +logger = logging.getLogger('glance.store.filesystem') + class ChunkedFile(object): @@ -60,8 +63,7 @@ class ChunkedFile(object): class FilesystemBackend(glance.store.Backend): @classmethod - def get(cls, parsed_uri, opener=lambda p: open(p, "rb"), - expected_size=None): + def get(cls, parsed_uri, expected_size=None, options=None): """ Filesystem-based backend file:///path/to/file.tar.gz.0 @@ -71,6 +73,8 @@ class FilesystemBackend(glance.store.Backend): if not os.path.exists(filepath): raise exception.NotFound("Image file %s not found" % filepath) else: + logger.debug("Found image at %s. Returning in ChunkedFile.", + filepath) return ChunkedFile(filepath) @classmethod @@ -87,6 +91,7 @@ class FilesystemBackend(glance.store.Backend): fn = parsed_uri.path if os.path.exists(fn): try: + logger.debug("Deleting image at %s", fn) os.unlink(fn) except OSError: raise exception.NotAuthorized("You cannot delete file %s" % fn) @@ -112,6 +117,8 @@ class FilesystemBackend(glance.store.Backend): datadir = options['filesystem_store_datadir'] if not os.path.exists(datadir): + logger.info("Directory to write image files does not exist " + "(%s). Creating.", datadir) os.makedirs(datadir) filepath = os.path.join(datadir, str(id)) @@ -129,6 +136,8 @@ class FilesystemBackend(glance.store.Backend): bytes_written += len(buf) f.write(buf) + logger.debug("Wrote %(bytes_written)d bytes to %(filepath)s" + % locals()) return ('file://%s' % filepath, bytes_written) @classmethod diff --git a/glance/store/swift.py b/glance/store/swift.py index 68655e78bc..5e6098b701 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 OpenStack, LLC +# Copyright 2010-2011 OpenStack, LLC # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,20 +15,33 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import +"""Storage backend for SWIFT""" + +import httplib +import logging +import urllib + +from glance.common import config +from glance.common import exception import glance.store +DEFAULT_SWIFT_ACCOUNT = 'glance' +DEFAULT_SWIFT_CONTAINER = 'glance' + +logger = logging.getLogger('glance.store.swift') + class SwiftBackend(glance.store.Backend): """ An implementation of the swift backend adapter. """ - EXAMPLE_URL = "swift://user:password@auth_url/container/file.gz.0" + EXAMPLE_URL = "swift://:@//" CHUNKSIZE = 65536 @classmethod - def get(cls, parsed_uri, expected_size, conn_class=None): + def get(cls, parsed_uri, expected_size=None, options=None, + conn_class=None): """ Takes a parsed_uri in the format of: swift://user:password@auth_url/container/file.gz.0, connects to the @@ -43,20 +56,130 @@ class SwiftBackend(glance.store.Backend): # snet=True connection_class = get_connection_class(conn_class) - swift_conn = conn_class( + swift_conn = connection_class( authurl=authurl, user=user, key=key, snet=False) - (resp_headers, resp_body) = swift_conn.get_object( - container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE) + try: + (resp_headers, resp_body) = swift_conn.get_object( + container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE) - obj_size = int(resp_headers['content-length']) - if obj_size != expected_size: - raise glance.store.BackendException( - "Expected %s byte file, Swift has %s bytes" % - (expected_size, obj_size)) + # TODO(jaypipes) use real exceptions when remove all the cruft + # around importing Swift stuff... + except Exception, e: + if e.http_status == httplib.NOT_FOUND: + location = "swift://%s:%s@%s/%s/%s" % (user, key, authurl, + container, obj) + raise exception.NotFound("Swift could not find image at " + "location %(location)s" % locals()) + + if expected_size: + obj_size = int(resp_headers['content-length']) + if obj_size != expected_size: + raise glance.store.BackendException( + "Expected %s byte file, Swift has %s bytes" % + (expected_size, obj_size)) return resp_body + @classmethod + def add(cls, id, data, options, conn_class=None): + """ + Stores image data to Swift and returns a location that the image was + written to. + + Swift writes the image data using the scheme: + ``swift://:@//` + where: + = ``swift_store_user`` + = ``swift_store_key`` + = ``swift_store_auth_address`` + = ``swift_store_container`` + = The id of the image being added + + :param id: The opaque image identifier + :param data: The image data to write, as a file-like object + :param options: Conf mapping + + :retval Tuple with (location, size) + The location that was written, + and the size in bytes of the data written + """ + account = options.get('swift_store_account', + DEFAULT_SWIFT_ACCOUNT) + container = options.get('swift_store_container', + DEFAULT_SWIFT_CONTAINER) + auth_address = options.get('swift_store_auth_address') + user = options.get('swift_store_user') + key = options.get('swift_store_key') + + # TODO(jaypipes): This needs to be checked every time + # because of the decision to make glance.store.Backend's + # interface all @classmethods. This is inefficient. Backend + # should be a stateful object with options parsed once in + # a constructor. + if not auth_address: + logger.error(msg) + msg = ("Could not find swift_store_auth_address in configuration " + "options.") + raise glance.store.BackendException(msg) + else: + full_auth_address = auth_address + if not full_auth_address.startswith('http'): + full_auth_address = 'https://' + full_auth_address + + if not user: + logger.error(msg) + msg = ("Could not find swift_store_user in configuration " + "options.") + raise glance.store.BackendException(msg) + + if not key: + logger.error(msg) + msg = ("Could not find swift_store_key in configuration " + "options.") + raise glance.store.BackendException(msg) + + connection_class = get_connection_class(conn_class) + swift_conn = connection_class(authurl=full_auth_address, user=user, + key=key, snet=False) + + logger.debug("Adding image object to Swift using " + "(auth_address=%(auth_address)s, user=%(user)s, " + "key=%(key)s)") + + try: + obj_etag = swift_conn.put_object(container, id, data) + + # NOTE: We return the user and key here! Have to because + # location is used by the API server to return the actual + # image data. We *really* should consider NOT returning + # the location attribute from GET /images/ and + # GET /images/details + location = "swift://%(user)s:%(key)s@%(auth_address)s/"\ + "%(container)s/%(id)s" % locals() + + # We do a HEAD on the newly-added image to determine the size + # of the image. A bit slow, but better than taking the word + # of the user adding the image with size attribute in the metadata + resp_headers = swift_conn.head_object(container, id) + size = 0 + # header keys are lowercased by Swift + if 'content-length' in resp_headers: + size = int(resp_headers['content-length']) + return (location, size) + + # TODO(jaypipes) use real exceptions when remove all the cruft + # around importing Swift stuff... + except Exception, e: + if e.http_status == httplib.CONFLICT: + location = "swift://%s:%s@%s/%s/%s" % (user, key, auth_address, + container, id) + raise exception.Duplicate("Swift already has an image at " + "location %(location)s" % locals()) + msg = ("Failed to add object to Swift.\n" + "Got error from Swift: %(e)s" % locals()) + raise glance.store.BackendException(msg) + @classmethod def delete(cls, parsed_uri, conn_class=None): """ @@ -70,14 +193,20 @@ class SwiftBackend(glance.store.Backend): # snet=True connection_class = get_connection_class(conn_class) - swift_conn = conn_class( + swift_conn = connection_class( authurl=authurl, user=user, key=key, snet=False) - (resp_headers, resp_body) = swift_conn.delete_object( - container=container, obj=obj) + try: + swift_conn.delete_object(container, obj) - # TODO(jaypipes): What to return here? After reading the docs - # at swift.common.client, I'm not sure what to check for... + # TODO(jaypipes) use real exceptions when remove all the cruft + # around importing Swift stuff... + except Exception, e: + if e.http_status == httplib.NOT_FOUND: + location = "swift://%s:%s@%s/%s/%s" % (user, key, authurl, + container, obj) + raise exception.NotFound("Swift could not find image at " + "location %(location)s" % locals()) @classmethod def _parse_swift_tokens(cls, parsed_uri): diff --git a/tests/stubs.py b/tests/stubs.py index 931957e0a4..da9fc5d85d 100644 --- a/tests/stubs.py +++ b/tests/stubs.py @@ -184,35 +184,6 @@ def stub_out_swift_backend(stubs): fake_swift_backend.get) -def stub_out_registry(stubs): - """Stubs out the Registry registry with fake data returns. - - The stubbed Registry always returns the following fixture:: - - {'files': [ - {'location': 'file:///chunk0', 'size': 12345}, - {'location': 'file:///chunk1', 'size': 1235} - ]} - - :param stubs: Set of stubout stubs - - """ - class FakeRegistry(object): - - DATA = \ - {'files': [ - {'location': 'file:///chunk0', 'size': 12345}, - {'location': 'file:///chunk1', 'size': 1235}]} - - @classmethod - def lookup(cls, _parsed_uri): - return cls.DATA - - fake_registry_registry = FakeRegistry() - stubs.Set(glance.store.registries.Registry, 'lookup', - fake_registry_registry.lookup) - - def stub_out_registry_and_store_server(stubs): """ Mocks calls to 127.0.0.1 on 9191 and 9292 for testing so diff --git a/tests/unit/swiftfakehttp.py b/tests/unit/swiftfakehttp.py deleted file mode 100644 index 995ea8d7cc..0000000000 --- a/tests/unit/swiftfakehttp.py +++ /dev/null @@ -1,294 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack, LLC -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -fakehttp/socket implementation - -- TrackerSocket: an object which masquerades as a socket and responds to - requests in a manner consistent with a *very* stupid CloudFS tracker. - -- CustomHTTPConnection: an object which subclasses httplib.HTTPConnection - in order to replace it's socket with a TrackerSocket instance. - -The unittests each have setup methods which create freerange connection -instances that have had their HTTPConnection instances replaced by -intances of CustomHTTPConnection. -""" - -from httplib import HTTPConnection as connbase -import StringIO - - -class FakeSocket(object): - def __init__(self): - self._rbuffer = StringIO.StringIO() - self._wbuffer = StringIO.StringIO() - - def close(self): - pass - - def send(self, data, flags=0): - self._rbuffer.write(data) - sendall = send - - def recv(self, len=1024, flags=0): - return self._wbuffer(len) - - def connect(self): - pass - - def makefile(self, mode, flags): - return self._wbuffer - - -class TrackerSocket(FakeSocket): - def write(self, data): - self._wbuffer.write(data) - - def read(self, length=-1): - return self._rbuffer.read(length) - - def _create_GET_account_content(self, path, args): - if 'format' in args and args['format'] == 'json': - containers = [] - containers.append('[\n') - containers.append('{"name":"container1","count":2,"bytes":78},\n') - containers.append('{"name":"container2","count":1,"bytes":39},\n') - containers.append('{"name":"container3","count":3,"bytes":117}\n') - containers.append(']\n') - elif 'format' in args and args['format'] == 'xml': - containers = [] - containers.append('\n') - containers.append('\n') - containers.append('container1' - '2' - '78\n') - containers.append('container2' - '1' - '39\n') - containers.append('container3' - '3' - '117\n') - containers.append('\n') - else: - containers = ['container%s\n' % i for i in range(1, 4)] - return ''.join(containers) - - def _create_GET_container_content(self, path, args): - left = 0 - right = 9 - if 'offset' in args: - left = int(args['offset']) - if 'limit' in args: - right = left + int(args['limit']) - - if 'format' in args and args['format'] == 'json': - objects = [] - objects.append('{"name":"object1",' - '"hash":"4281c348eaf83e70ddce0e07221c3d28",' - '"bytes":14,' - '"content_type":"application\/octet-stream",' - '"last_modified":"2007-03-04 20:32:17"}') - objects.append('{"name":"object2",' - '"hash":"b039efe731ad111bc1b0ef221c3849d0",' - '"bytes":64,' - '"content_type":"application\/octet-stream",' - '"last_modified":"2007-03-04 20:32:17"}') - objects.append('{"name":"object3",' - '"hash":"4281c348eaf83e70ddce0e07221c3d28",' - '"bytes":14,' - '"content_type":"application\/octet-stream",' - '"last_modified":"2007-03-04 20:32:17"}') - objects.append('{"name":"object4",' - '"hash":"b039efe731ad111bc1b0ef221c3849d0",' - '"bytes":64,' - '"content_type":"application\/octet-stream",' - '"last_modified":"2007-03-04 20:32:17"}') - objects.append('{"name":"object5",' - '"hash":"4281c348eaf83e70ddce0e07221c3d28",' - '"bytes":14,' - '"content_type":"application\/octet-stream",' - '"last_modified":"2007-03-04 20:32:17"}') - objects.append('{"name":"object6",' - '"hash":"b039efe731ad111bc1b0ef221c3849d0",' - '"bytes":64,' - '"content_type":"application\/octet-stream",' - '"last_modified":"2007-03-04 20:32:17"}') - objects.append('{"name":"object7",' - '"hash":"4281c348eaf83e70ddce0e07221c3d28",' - '"bytes":14,' - '"content_type":"application\/octet-stream",' - '"last_modified":"2007-03-04 20:32:17"}') - objects.append('{"name":"object8",' - '"hash":"b039efe731ad111bc1b0ef221c3849d0",' - '"bytes":64,' - '"content_type":"application\/octet-stream",' - '"last_modified":"2007-03-04 20:32:17"}') - output = '[\n%s\n]\n' % (',\n'.join(objects[left:right])) - elif 'format' in args and args['format'] == 'xml': - objects = [] - objects.append('object1' - '4281c348eaf83e70ddce0e07221c3d28' - '14' - 'application/octet-stream' - '2007-03-04 20:32:17' - '\n') - objects.append('object2' - 'b039efe731ad111bc1b0ef221c3849d0' - '64' - 'application/octet-stream' - '2007-03-04 20:32:17' - '\n') - objects.append('object3' - '4281c348eaf83e70ddce0e07221c3d28' - '14' - 'application/octet-stream' - '2007-03-04 20:32:17' - '\n') - objects.append('object4' - 'b039efe731ad111bc1b0ef221c3849d0' - '64' - 'application/octet-stream' - '2007-03-04 20:32:17' - '\n') - objects.append('object5' - '4281c348eaf83e70ddce0e07221c3d28' - '14' - 'application/octet-stream' - '2007-03-04 20:32:17' - '\n') - objects.append('object6' - 'b039efe731ad111bc1b0ef221c3849d0' - '64' - 'application/octet-stream' - '2007-03-04 20:32:17' - '\n') - objects.append('object7' - '4281c348eaf83e70ddce0e07221c3d28' - '14' - 'application/octet-stream' - '2007-03-04 20:32:17' - '\n') - objects.append('object8' - 'b039efe731ad111bc1b0ef221c3849d0' - '64' - 'application/octet-stream' - '2007-03-04 20:32:17' - '\n') - objects = objects[left:right] - objects.insert(0, '\n') - objects.insert(1, '\n') - output = ''.join(objects) - else: - objects = ['object%s\n' % i for i in range(1, 9)] - objects = objects[left:right] - output = ''.join(objects) - - # prefix/path don't make much sense given our test data - if 'prefix' in args or 'path' in args: - pass - return output - - def render_GET(self, path, args): - # Special path that returns 404 Not Found - if (len(path) == 4) and (path[3] == 'bogus'): - self.write('HTTP/1.1 404 Not Found\n') - self.write('Content-Type: text/plain\n') - self.write('Content-Length: 0\n') - self.write('Connection: close\n\n') - return - - self.write('HTTP/1.1 200 Ok\n') - self.write('Content-Type: text/plain\n') - if len(path) == 2: - content = self._create_GET_account_content(path, args) - elif len(path) == 3: - content = self._create_GET_container_content(path, args) - # Object - elif len(path) == 4: - content = 'I am a teapot, short and stout\n' - self.write('Content-Length: %d\n' % len(content)) - self.write('Connection: close\n\n') - self.write(content) - - def render_HEAD(self, path, args): - # Account - if len(path) == 2: - self.write('HTTP/1.1 204 No Content\n') - self.write('Content-Type: text/plain\n') - self.write('Connection: close\n') - self.write('X-Account-Container-Count: 3\n') - self.write('X-Account-Bytes-Used: 234\n\n') - else: - self.write('HTTP/1.1 200 Ok\n') - self.write('Content-Type: text/plain\n') - self.write('ETag: d5c7f3babf6c602a8da902fb301a9f27\n') - self.write('Content-Length: 21\n') - self.write('Connection: close\n\n') - - def render_POST(self, path, args): - self.write('HTTP/1.1 202 Ok\n') - self.write('Connection: close\n\n') - - def render_PUT(self, path, args): - self.write('HTTP/1.1 200 Ok\n') - self.write('Content-Type: text/plain\n') - self.write('Connection: close\n\n') - render_DELETE = render_PUT - - def render(self, method, uri): - if '?' in uri: - parts = uri.split('?') - query = parts[1].strip('&').split('&') - args = dict([tuple(i.split('=', 1)) for i in query]) - path = parts[0].strip('/').split('/') - else: - args = {} - path = uri.strip('/').split('/') - - if hasattr(self, 'render_%s' % method): - getattr(self, 'render_%s' % method)(path, args) - else: - self.write('HTTP/1.1 406 Not Acceptable\n') - self.write('Content-Type: text/plain\n') - self.write('Connection: close\n') - - def makefile(self, mode, flags): - self._rbuffer.seek(0) - lines = self.read().splitlines() - (method, uri, version) = lines[0].split() - - self.render(method, uri) - - self._wbuffer.seek(0) - return self._wbuffer - - -class CustomHTTPConnection(connbase): - def connect(self): - self.sock = TrackerSocket() - - -if __name__ == '__main__': - conn = CustomHTTPConnection('localhost', 8000) - conn.request('HEAD', '/v1/account/container/object') - response = conn.getresponse() - print "Status:", response.status, response.reason - for (key, value) in response.getheaders(): - print "%s: %s" % (key, value) - print response.read() diff --git a/tests/unit/test_filesystem_store.py b/tests/unit/test_filesystem_store.py new file mode 100644 index 0000000000..94bc3bfc4a --- /dev/null +++ b/tests/unit/test_filesystem_store.py @@ -0,0 +1,135 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests the filesystem backend store""" + +import StringIO +import unittest +import urlparse + +import stubout + +from glance.common import exception +from glance.store.filesystem import FilesystemBackend, ChunkedFile +from tests import stubs + + +class TestFilesystemBackend(unittest.TestCase): + + def setUp(self): + """Establish a clean test environment""" + self.stubs = stubout.StubOutForTesting() + stubs.stub_out_filesystem_backend() + self.orig_chunksize = ChunkedFile.CHUNKSIZE + ChunkedFile.CHUNKSIZE = 10 + + def tearDown(self): + """Clear the test environment""" + stubs.clean_out_fake_filesystem_backend() + self.stubs.UnsetAll() + ChunkedFile.CHUNKSIZE = self.orig_chunksize + + def test_get(self): + """Test a "normal" retrieval of an image in chunks""" + url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2") + image_file = FilesystemBackend.get(url_pieces) + + expected_data = "chunk00000remainder" + expected_num_chunks = 2 + data = "" + num_chunks = 0 + + for chunk in image_file: + num_chunks += 1 + data += chunk + self.assertEqual(expected_data, data) + self.assertEqual(expected_num_chunks, num_chunks) + + def test_get_non_existing(self): + """ + Test that trying to retrieve a file that doesn't exist + raises an error + """ + url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing") + self.assertRaises(exception.NotFound, + FilesystemBackend.get, + url_pieces) + + def test_add(self): + """Test that we can add an image via the filesystem backend""" + ChunkedFile.CHUNKSIZE = 1024 + expected_image_id = 42 + expected_file_size = 1024 * 5 # 5K + expected_file_contents = "*" * expected_file_size + expected_location = "file://%s/%s" % (stubs.FAKE_FILESYSTEM_ROOTDIR, + expected_image_id) + image_file = StringIO.StringIO(expected_file_contents) + options = {'verbose': True, + 'debug': True, + 'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR} + + location, size = FilesystemBackend.add(42, image_file, options) + + self.assertEquals(expected_location, location) + self.assertEquals(expected_file_size, size) + + url_pieces = urlparse.urlparse("file:///tmp/glance-tests/42") + new_image_file = FilesystemBackend.get(url_pieces) + new_image_contents = "" + new_image_file_size = 0 + + for chunk in new_image_file: + new_image_file_size += len(chunk) + new_image_contents += chunk + + self.assertEquals(expected_file_contents, new_image_contents) + self.assertEquals(expected_file_size, new_image_file_size) + + def test_add_already_existing(self): + """ + Tests that adding an image with an existing identifier + raises an appropriate exception + """ + image_file = StringIO.StringIO("nevergonnamakeit") + options = {'verbose': True, + 'debug': True, + 'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR} + self.assertRaises(exception.Duplicate, + FilesystemBackend.add, + 2, image_file, options) + + def test_delete(self): + """ + Test we can delete an existing image in the filesystem store + """ + url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2") + + FilesystemBackend.delete(url_pieces) + + self.assertRaises(exception.NotFound, + FilesystemBackend.get, + url_pieces) + + def test_delete_non_existing(self): + """ + Test that trying to delete a file that doesn't exist + raises an error + """ + url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing") + self.assertRaises(exception.NotFound, + FilesystemBackend.delete, + url_pieces) diff --git a/tests/unit/test_stores.py b/tests/unit/test_stores.py index 599ec4465a..1b5028c544 100644 --- a/tests/unit/test_stores.py +++ b/tests/unit/test_stores.py @@ -40,27 +40,6 @@ class TestBackend(unittest.TestCase): self.stubs.UnsetAll() -class TestFilesystemBackend(TestBackend): - - def setUp(self): - """Establish a clean test environment""" - stubs.stub_out_filesystem_backend() - - def tearDown(self): - """Clear the test environment""" - stubs.clean_out_fake_filesystem_backend() - - def test_get(self): - - fetcher = get_from_backend("file:///tmp/glance-tests/2", - expected_size=19) - - data = "" - for chunk in fetcher: - data += chunk - self.assertEqual(data, "chunk00000remainder") - - class TestHTTPBackend(TestBackend): def setUp(self): @@ -104,45 +83,3 @@ class TestS3Backend(TestBackend): chunks = [c for c in fetcher] self.assertEqual(chunks, expected_returns) - - -class TestSwiftBackend(TestBackend): - def setUp(self): - super(TestSwiftBackend, self).setUp() - stubs.stub_out_swift_backend(self.stubs) - - def test_get(self): - - swift_uri = "swift://user:pass@localhost/container1/file.tar.gz" - expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', - 'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n'] - - fetcher = get_from_backend(swift_uri, - expected_size=21, - conn_class=SwiftBackend) - - chunks = [c for c in fetcher] - - self.assertEqual(chunks, expected_returns) - - def test_get_bad_uri(self): - - swift_url = "swift://localhost/container1/file.tar.gz" - - self.assertRaises(BackendException, get_from_backend, - swift_url, expected_size=21) - - def test_url_parsing(self): - - swift_uri = "swift://user:pass@localhost/v1.0/container1/file.tar.gz" - - parsed_uri = urlparse.urlparse(swift_uri) - - (user, key, authurl, container, obj) = \ - SwiftBackend._parse_swift_tokens(parsed_uri) - - self.assertEqual(user, 'user') - self.assertEqual(key, 'pass') - self.assertEqual(authurl, 'https://localhost/v1.0') - self.assertEqual(container, 'container1') - self.assertEqual(obj, 'file.tar.gz') diff --git a/tests/unit/test_swift_store.py b/tests/unit/test_swift_store.py new file mode 100644 index 0000000000..d301580e6b --- /dev/null +++ b/tests/unit/test_swift_store.py @@ -0,0 +1,261 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this swift except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests the Swift backend store""" + +import StringIO +import hashlib +import httplib +import sys +import unittest +import urlparse + +import stubout + +from glance.common import exception +import glance.store.swift + +SwiftBackend = glance.store.swift.SwiftBackend + +SWIFT_INSTALLED = False + +try: + import swift.common.client + SWIFT_INSTALLED = True +except ImportError: + print "Skipping Swift store tests since Swift is not installed." + +FIVE_KB = (5 * 1024) +SWIFT_OPTIONS = {'verbose': True, + 'debug': True, + 'swift_store_user': 'glance', + 'swift_store_key': 'key', + 'swift_store_auth_address': 'localhost:8080', + 'swift_store_container': 'glance'} + + +# We stub out as little as possible to ensure that the code paths +# between glance.store.swift and swift.common.client are tested +# thoroughly +def stub_out_swift_common_client(stubs): + + fixture_headers = {'glance/2': + {'content-length': FIVE_KB, + 'etag': 'c2e5db72bd7fd153f53ede5da5a06de3'}} + fixture_objects = {'glance/2': + StringIO.StringIO("*" * FIVE_KB)} + + def fake_put_object(url, token, container, name, contents, **kwargs): + # PUT returns the ETag header for the newly-added object + fixture_key = "%s/%s" % (container, name) + if not fixture_key in fixture_headers.keys(): + if hasattr(contents, 'read'): + fixture_object = StringIO.StringIO() + chunk = contents.read(SwiftBackend.CHUNKSIZE) + while chunk: + fixture_object.write(chunk) + chunk = contents.read(SwiftBackend.CHUNKSIZE) + else: + fixture_object = StringIO.StringIO(contents) + fixture_objects[fixture_key] = fixture_object + fixture_headers[fixture_key] = { + 'content-length': fixture_object.len, + 'etag': hashlib.md5(fixture_object.read()).hexdigest()} + return fixture_headers[fixture_key]['etag'] + else: + msg = ("Object PUT failed - Object with key %s already exists" + % fixture_key) + raise swift.common.client.ClientException(msg, + http_status=httplib.CONFLICT) + + def fake_get_object(url, token, container, name, **kwargs): + # GET returns the tuple (list of headers, file object) + try: + fixture_key = "%s/%s" % (container, name) + return fixture_headers[fixture_key], fixture_objects[fixture_key] + except KeyError: + msg = "Object GET failed" + raise swift.common.client.ClientException(msg, + http_status=httplib.NOT_FOUND) + + def fake_head_object(url, token, container, name, **kwargs): + # HEAD returns the list of headers for an object + try: + fixture_key = "%s/%s" % (container, name) + return fixture_headers[fixture_key] + except KeyError: + msg = "Object HEAD failed - Object does not exist" + raise swift.common.client.ClientException(msg, + http_status=httplib.NOT_FOUND) + + def fake_delete_object(url, token, container, name, **kwargs): + # DELETE returns nothing + fixture_key = "%s/%s" % (container, name) + if fixture_key not in fixture_headers.keys(): + msg = "Object DELETE failed - Object does not exist" + raise swift.common.client.ClientException(msg, + http_status=httplib.NOT_FOUND) + else: + del fixture_headers[fixture_key] + del fixture_objects[fixture_key] + + def fake_get_connection_class(*args): + return swift.common.client.Connection + + def fake_http_connection(self): + return None + + def fake_get_auth(self): + return None, None + + stubs.Set(swift.common.client, + 'put_object', fake_put_object) + stubs.Set(swift.common.client, + 'delete_object', fake_delete_object) + stubs.Set(swift.common.client, + 'head_object', fake_head_object) + stubs.Set(swift.common.client, + 'get_object', fake_get_object) + stubs.Set(swift.common.client.Connection, + 'get_auth', fake_get_auth) + stubs.Set(swift.common.client.Connection, + 'http_connection', fake_http_connection) + stubs.Set(glance.store.swift, + 'get_connection_class', fake_get_connection_class) + + +class TestSwiftBackend(unittest.TestCase): + + def setUp(self): + """Establish a clean test environment""" + self.stubs = stubout.StubOutForTesting() + if SWIFT_INSTALLED: + stub_out_swift_common_client(self.stubs) + + def tearDown(self): + """Clear the test environment""" + self.stubs.UnsetAll() + + def test_get(self): + """Test a "normal" retrieval of an image in chunks""" + if not SWIFT_INSTALLED: + return + url_pieces = urlparse.urlparse( + "swift://user:key@auth_address/glance/2") + image_swift = SwiftBackend.get(url_pieces) + + expected_data = "*" * FIVE_KB + data = "" + + for chunk in image_swift: + data += chunk + self.assertEqual(expected_data, data) + + def test_get_mismatched_expected_size(self): + """ + Test retrieval of an image with wrong expected_size param + raises an exception + """ + if not SWIFT_INSTALLED: + return + url_pieces = urlparse.urlparse( + "swift://user:key@auth_address/glance/2") + self.assertRaises(glance.store.BackendException, + SwiftBackend.get, + url_pieces, + {'expected_size': 42}) + + def test_get_non_existing(self): + """ + Test that trying to retrieve a swift that doesn't exist + raises an error + """ + if not SWIFT_INSTALLED: + return + url_pieces = urlparse.urlparse( + "swift://user:key@auth_address/noexist") + self.assertRaises(exception.NotFound, + SwiftBackend.get, + url_pieces) + + def test_add(self): + """Test that we can add an image via the swift backend""" + if not SWIFT_INSTALLED: + return + expected_image_id = 42 + expected_swift_size = 1024 * 5 # 5K + expected_swift_contents = "*" * expected_swift_size + expected_location = "swift://%s:%s@%s/%s/%s" % ( + SWIFT_OPTIONS['swift_store_user'], + SWIFT_OPTIONS['swift_store_key'], + SWIFT_OPTIONS['swift_store_auth_address'], + SWIFT_OPTIONS['swift_store_container'], + expected_image_id) + image_swift = StringIO.StringIO(expected_swift_contents) + + location, size = SwiftBackend.add(42, image_swift, SWIFT_OPTIONS) + + self.assertEquals(expected_location, location) + self.assertEquals(expected_swift_size, size) + + url_pieces = urlparse.urlparse( + "swift://user:key@auth_address/glance/42") + new_image_swift = SwiftBackend.get(url_pieces) + new_image_contents = new_image_swift.getvalue() + new_image_swift_size = new_image_swift.len + + self.assertEquals(expected_swift_contents, new_image_contents) + self.assertEquals(expected_swift_size, new_image_swift_size) + + def test_add_already_existing(self): + """ + Tests that adding an image with an existing identifier + raises an appropriate exception + """ + if not SWIFT_INSTALLED: + return + image_swift = StringIO.StringIO("nevergonnamakeit") + self.assertRaises(exception.Duplicate, + SwiftBackend.add, + 2, image_swift, SWIFT_OPTIONS) + + def test_delete(self): + """ + Test we can delete an existing image in the swift store + """ + if not SWIFT_INSTALLED: + return + url_pieces = urlparse.urlparse( + "swift://user:key@auth_address/glance/2") + + SwiftBackend.delete(url_pieces) + + self.assertRaises(exception.NotFound, + SwiftBackend.get, + url_pieces) + + def test_delete_non_existing(self): + """ + Test that trying to delete a swift that doesn't exist + raises an error + """ + if not SWIFT_INSTALLED: + return + url_pieces = urlparse.urlparse("swift://user:key@auth_address/noexist") + self.assertRaises(exception.NotFound, + SwiftBackend.delete, + url_pieces) diff --git a/tools/pip-requires b/tools/pip-requires index 4d6d8411e7..7dc9f4862c 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -12,5 +12,6 @@ nose sphinx argparse mox==0.5.0 +swift -f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz sqlalchemy-migrate>=0.6