Adds ability for Swift to be used as a full-fledged backend.

Adds POST/PUT capabilities to the SwiftBackend
Adds lots of unit tests for both FilesystemBackend and SwiftBackend
Removes now-unused tests.unit.fakeswifthttp module

etc/glance.conf.sample

@@ -8,10 +8,6 @@ debug = False
 [app:glance-api]
 paste.app_factory = glance.server:app_factory
 
-# Directory that the Filesystem backend store
-# writes image data to
-filesystem_store_datadir=/var/lib/glance/images/
-
 # Which backend store should Glance use by default is not specified
 # in a request to add a new image to Glance? Default: 'file'
 # Available choices are 'file', 'swift', and 's3'
@@ -29,6 +25,32 @@ registry_host = 0.0.0.0
 # Port the registry server is listening on
 registry_port = 9191
 
+# ============ Filesystem Store Options ========================
+
+# Directory that the Filesystem backend store
+# writes image data to
+filesystem_store_datadir=/var/lib/glance/images/
+
+# ============ Swift Store Options =============================
+
+# Address where the Swift authentication service lives
+swift_store_auth_address = 127.0.0.1:8080
+
+# User to authenticate against the Swift authentication service
+swift_store_user = jdoe
+
+# Auth key for the user authenticating against the
+# Swift authentication service
+swift_store_key = a86850deb2742ec3cb41518e26aa2d89
+
+# Account to use for the user:key Swift auth combination
+# for storing images in Swift
+swift_store_account = glance
+
+# Container within the account that the account should use
+# for storing images in Swift
+swift_store_container = glance
+
 [app:glance-registry]
 paste.app_factory = glance.registry.server:app_factory
 

glance/server.py

@@ -154,7 +154,8 @@ class Controller(wsgi.Controller):
 
         def image_iterator():
             chunks = get_from_backend(image['location'],
-                                      expected_size=image['size'])
+                                      expected_size=image['size'],
+                                      options=self.options)
 
             for chunk in chunks:
                 yield chunk

glance/store/filesystem.py

@@ -19,12 +19,15 @@
 A simple filesystem-backed store
 """
 
+import logging
 import os
 import urlparse
 
 from glance.common import exception
 import glance.store
 
+logger = logging.getLogger('glance.store.filesystem')
+
 
 class ChunkedFile(object):
 
@@ -60,8 +63,7 @@ class ChunkedFile(object):
 
 class FilesystemBackend(glance.store.Backend):
     @classmethod
-    def get(cls, parsed_uri, opener=lambda p: open(p, "rb"),
+    def get(cls, parsed_uri, expected_size=None, options=None):
-            expected_size=None):
         """ Filesystem-based backend
 
         file:///path/to/file.tar.gz.0
@@ -71,6 +73,8 @@ class FilesystemBackend(glance.store.Backend):
         if not os.path.exists(filepath):
             raise exception.NotFound("Image file %s not found" % filepath)
         else:
+            logger.debug("Found image at %s. Returning in ChunkedFile.",
+                         filepath)
             return ChunkedFile(filepath)
 
     @classmethod
@@ -87,6 +91,7 @@ class FilesystemBackend(glance.store.Backend):
         fn = parsed_uri.path
         if os.path.exists(fn):
             try:
+                logger.debug("Deleting image at %s", fn)
                 os.unlink(fn)
             except OSError:
                 raise exception.NotAuthorized("You cannot delete file %s" % fn)
@@ -112,6 +117,8 @@ class FilesystemBackend(glance.store.Backend):
         datadir = options['filesystem_store_datadir']
 
         if not os.path.exists(datadir):
+            logger.info("Directory to write image files does not exist "
+                        "(%s). Creating.", datadir)
             os.makedirs(datadir)
 
         filepath = os.path.join(datadir, str(id))
@@ -129,6 +136,8 @@ class FilesystemBackend(glance.store.Backend):
                 bytes_written += len(buf)
                 f.write(buf)
 
+        logger.debug("Wrote %(bytes_written)d bytes to %(filepath)s"
+                     % locals())
         return ('file://%s' % filepath, bytes_written)
 
     @classmethod

glance/store/swift.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2010 OpenStack, LLC
+# Copyright 2010-2011 OpenStack, LLC
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -15,20 +15,33 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from __future__ import absolute_import
+"""Storage backend for SWIFT"""
+
+import httplib
+import logging
+import urllib
+
+from glance.common import config
+from glance.common import exception
 import glance.store
 
+DEFAULT_SWIFT_ACCOUNT = 'glance'
+DEFAULT_SWIFT_CONTAINER = 'glance'
+
+logger = logging.getLogger('glance.store.swift')
+
 
 class SwiftBackend(glance.store.Backend):
     """
     An implementation of the swift backend adapter.
     """
-    EXAMPLE_URL = "swift://user:password@auth_url/container/file.gz.0"
+    EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"
 
     CHUNKSIZE = 65536
 
     @classmethod
-    def get(cls, parsed_uri, expected_size, conn_class=None):
+    def get(cls, parsed_uri, expected_size=None, options=None,
+            conn_class=None):
         """
         Takes a parsed_uri in the format of:
         swift://user:password@auth_url/container/file.gz.0, connects to the
@@ -43,12 +56,23 @@ class SwiftBackend(glance.store.Backend):
         # snet=True
         connection_class = get_connection_class(conn_class)
 
-        swift_conn = conn_class(
+        swift_conn = connection_class(
             authurl=authurl, user=user, key=key, snet=False)
 
+        try:
             (resp_headers, resp_body) = swift_conn.get_object(
                 container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE)
 
+        # TODO(jaypipes) use real exceptions when remove all the cruft
+        # around importing Swift stuff...
+        except Exception, e:
+            if e.http_status == httplib.NOT_FOUND:
+                location = "swift://%s:%s@%s/%s/%s" % (user, key, authurl,
+                                                    container, obj)
+                raise exception.NotFound("Swift could not find image at "
+                                         "location %(location)s" % locals())
+
+        if expected_size:
             obj_size = int(resp_headers['content-length'])
             if  obj_size != expected_size:
                 raise glance.store.BackendException(
@@ -57,6 +81,105 @@ class SwiftBackend(glance.store.Backend):
 
         return resp_body
 
+    @classmethod
+    def add(cls, id, data, options, conn_class=None):
+        """
+        Stores image data to Swift and returns a location that the image was
+        written to.
+
+        Swift writes the image data using the scheme:
+            ``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
+        where:
+            <USER> = ``swift_store_user``
+            <KEY> = ``swift_store_key``
+            <AUTH_ADDRESS> = ``swift_store_auth_address``
+            <CONTAINER> = ``swift_store_container``
+            <ID> = The id of the image being added
+
+        :param id: The opaque image identifier
+        :param data: The image data to write, as a file-like object
+        :param options: Conf mapping
+
+        :retval Tuple with (location, size)
+                The location that was written,
+                and the size in bytes of the data written
+        """
+        account = options.get('swift_store_account',
+                              DEFAULT_SWIFT_ACCOUNT)
+        container = options.get('swift_store_container',
+                                DEFAULT_SWIFT_CONTAINER)
+        auth_address = options.get('swift_store_auth_address')
+        user = options.get('swift_store_user')
+        key = options.get('swift_store_key')
+
+        # TODO(jaypipes): This needs to be checked every time
+        # because of the decision to make glance.store.Backend's
+        # interface all @classmethods. This is inefficient. Backend
+        # should be a stateful object with options parsed once in
+        # a constructor.
+        if not auth_address:
+            logger.error(msg)
+            msg = ("Could not find swift_store_auth_address in configuration "
+                   "options.")
+            raise glance.store.BackendException(msg)
+        else:
+            full_auth_address = auth_address
+            if not full_auth_address.startswith('http'):
+                full_auth_address = 'https://' + full_auth_address
+
+        if not user:
+            logger.error(msg)
+            msg = ("Could not find swift_store_user in configuration "
+                   "options.")
+            raise glance.store.BackendException(msg)
+
+        if not key:
+            logger.error(msg)
+            msg = ("Could not find swift_store_key in configuration "
+                   "options.")
+            raise glance.store.BackendException(msg)
+
+        connection_class = get_connection_class(conn_class)
+        swift_conn = connection_class(authurl=full_auth_address, user=user,
+                                      key=key, snet=False)
+
+        logger.debug("Adding image object to Swift using "
+                     "(auth_address=%(auth_address)s, user=%(user)s, "
+                     "key=%(key)s)")
+
+        try:
+            obj_etag = swift_conn.put_object(container, id, data)
+
+            # NOTE: We return the user and key here! Have to because
+            # location is used by the API server to return the actual
+            # image data. We *really* should consider NOT returning
+            # the location attribute from GET /images/<ID> and
+            # GET /images/details
+            location = "swift://%(user)s:%(key)s@%(auth_address)s/"\
+                       "%(container)s/%(id)s" % locals()
+
+            # We do a HEAD on the newly-added image to determine the size
+            # of the image. A bit slow, but better than taking the word
+            # of the user adding the image with size attribute in the metadata
+            resp_headers = swift_conn.head_object(container, id)
+            size = 0
+            # header keys are lowercased by Swift
+            if 'content-length' in resp_headers:
+                size = int(resp_headers['content-length'])
+            return (location, size)
+
+        # TODO(jaypipes) use real exceptions when remove all the cruft
+        # around importing Swift stuff...
+        except Exception, e:
+            if e.http_status == httplib.CONFLICT:
+                location = "swift://%s:%s@%s/%s/%s" % (user, key, auth_address,
+                                                    container, id)
+                raise exception.Duplicate("Swift already has an image at "
+                                         "location %(location)s" % locals())
+            msg = ("Failed to add object to Swift.\n"
+                   "Got error from Swift: %(e)s" % locals())
+            raise glance.store.BackendException(msg)
+
     @classmethod
     def delete(cls, parsed_uri, conn_class=None):
         """
@@ -70,14 +193,20 @@ class SwiftBackend(glance.store.Backend):
         # snet=True
         connection_class = get_connection_class(conn_class)
 
-        swift_conn = conn_class(
+        swift_conn = connection_class(
             authurl=authurl, user=user, key=key, snet=False)
 
-        (resp_headers, resp_body) = swift_conn.delete_object(
+        try:
-            container=container, obj=obj)
+            swift_conn.delete_object(container, obj)
 
-        # TODO(jaypipes): What to return here?  After reading the docs
+        # TODO(jaypipes) use real exceptions when remove all the cruft
-        # at swift.common.client, I'm not sure what to check for...
+        # around importing Swift stuff...
+        except Exception, e:
+            if e.http_status == httplib.NOT_FOUND:
+                location = "swift://%s:%s@%s/%s/%s" % (user, key, authurl,
+                                                    container, obj)
+                raise exception.NotFound("Swift could not find image at "
+                                         "location %(location)s" % locals())
 
     @classmethod
     def _parse_swift_tokens(cls, parsed_uri):

tests/stubs.py

@@ -184,35 +184,6 @@ def stub_out_swift_backend(stubs):
               fake_swift_backend.get)
 
 
-def stub_out_registry(stubs):
-    """Stubs out the Registry registry with fake data returns.
-
-    The stubbed Registry always returns the following fixture::
-
-        {'files': [
-          {'location': 'file:///chunk0', 'size': 12345},
-          {'location': 'file:///chunk1', 'size': 1235}
-        ]}
-
-    :param stubs: Set of stubout stubs
-
-    """
-    class FakeRegistry(object):
-
-        DATA = \
-            {'files': [
-              {'location': 'file:///chunk0', 'size': 12345},
-              {'location': 'file:///chunk1', 'size': 1235}]}
-
-        @classmethod
-        def lookup(cls, _parsed_uri):
-            return cls.DATA
-
-    fake_registry_registry = FakeRegistry()
-    stubs.Set(glance.store.registries.Registry, 'lookup',
-              fake_registry_registry.lookup)
-
-
 def stub_out_registry_and_store_server(stubs):
     """
     Mocks calls to 127.0.0.1 on 9191 and 9292 for testing so

tests/unit/swiftfakehttp.py

@@ -1,294 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2010 OpenStack, LLC
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-fakehttp/socket implementation
-
-- TrackerSocket: an object which masquerades as a socket and responds to
-  requests in a manner consistent with a *very* stupid CloudFS tracker.
-
-- CustomHTTPConnection: an object which subclasses httplib.HTTPConnection
-  in order to replace it's socket with a TrackerSocket instance.
-
-The unittests each have setup methods which create freerange connection
-instances that have had their HTTPConnection instances replaced by
-intances of CustomHTTPConnection.
-"""
-
-from httplib import HTTPConnection as connbase
-import StringIO
-
-
-class FakeSocket(object):
-    def __init__(self):
-        self._rbuffer = StringIO.StringIO()
-        self._wbuffer = StringIO.StringIO()
-
-    def close(self):
-        pass
-
-    def send(self, data, flags=0):
-        self._rbuffer.write(data)
-    sendall = send
-
-    def recv(self, len=1024, flags=0):
-        return self._wbuffer(len)
-
-    def connect(self):
-        pass
-
-    def makefile(self, mode, flags):
-        return self._wbuffer
-
-
-class TrackerSocket(FakeSocket):
-    def write(self, data):
-        self._wbuffer.write(data)
-
-    def read(self, length=-1):
-        return self._rbuffer.read(length)
-
-    def _create_GET_account_content(self, path, args):
-        if 'format' in args and args['format'] == 'json':
-            containers = []
-            containers.append('[\n')
-            containers.append('{"name":"container1","count":2,"bytes":78},\n')
-            containers.append('{"name":"container2","count":1,"bytes":39},\n')
-            containers.append('{"name":"container3","count":3,"bytes":117}\n')
-            containers.append(']\n')
-        elif 'format' in args and args['format'] == 'xml':
-            containers = []
-            containers.append('<?xml version="1.0" encoding="UTF-8"?>\n')
-            containers.append('<account name="FakeAccount">\n')
-            containers.append('<container><name>container1</name>'
-                              '<count>2</count>'
-                              '<bytes>78</bytes></container>\n')
-            containers.append('<container><name>container2</name>'
-                              '<count>1</count>'
-                              '<bytes>39</bytes></container>\n')
-            containers.append('<container><name>container3</name>'
-                              '<count>3</count>'
-                              '<bytes>117</bytes></container>\n')
-            containers.append('</account>\n')
-        else:
-            containers = ['container%s\n' % i for i in range(1, 4)]
-        return ''.join(containers)
-
-    def _create_GET_container_content(self, path, args):
-        left = 0
-        right = 9
-        if 'offset' in args:
-            left = int(args['offset'])
-        if 'limit' in args:
-            right = left + int(args['limit'])
-
-        if 'format' in args and args['format'] == 'json':
-            objects = []
-            objects.append('{"name":"object1",'
-                           '"hash":"4281c348eaf83e70ddce0e07221c3d28",'
-                           '"bytes":14,'
-                           '"content_type":"application\/octet-stream",'
-                           '"last_modified":"2007-03-04 20:32:17"}')
-            objects.append('{"name":"object2",'
-                           '"hash":"b039efe731ad111bc1b0ef221c3849d0",'
-                           '"bytes":64,'
-                           '"content_type":"application\/octet-stream",'
-                           '"last_modified":"2007-03-04 20:32:17"}')
-            objects.append('{"name":"object3",'
-                           '"hash":"4281c348eaf83e70ddce0e07221c3d28",'
-                           '"bytes":14,'
-                           '"content_type":"application\/octet-stream",'
-                           '"last_modified":"2007-03-04 20:32:17"}')
-            objects.append('{"name":"object4",'
-                           '"hash":"b039efe731ad111bc1b0ef221c3849d0",'
-                           '"bytes":64,'
-                           '"content_type":"application\/octet-stream",'
-                           '"last_modified":"2007-03-04 20:32:17"}')
-            objects.append('{"name":"object5",'
-                           '"hash":"4281c348eaf83e70ddce0e07221c3d28",'
-                           '"bytes":14,'
-                           '"content_type":"application\/octet-stream",'
-                           '"last_modified":"2007-03-04 20:32:17"}')
-            objects.append('{"name":"object6",'
-                           '"hash":"b039efe731ad111bc1b0ef221c3849d0",'
-                           '"bytes":64,'
-                           '"content_type":"application\/octet-stream",'
-                           '"last_modified":"2007-03-04 20:32:17"}')
-            objects.append('{"name":"object7",'
-                           '"hash":"4281c348eaf83e70ddce0e07221c3d28",'
-                           '"bytes":14,'
-                           '"content_type":"application\/octet-stream",'
-                           '"last_modified":"2007-03-04 20:32:17"}')
-            objects.append('{"name":"object8",'
-                           '"hash":"b039efe731ad111bc1b0ef221c3849d0",'
-                           '"bytes":64,'
-                           '"content_type":"application\/octet-stream",'
-                           '"last_modified":"2007-03-04 20:32:17"}')
-            output = '[\n%s\n]\n' % (',\n'.join(objects[left:right]))
-        elif 'format' in args and args['format'] == 'xml':
-            objects = []
-            objects.append('<object><name>object1</name>'
-                       '<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
-                       '<bytes>14</bytes>'
-                       '<content_type>application/octet-stream</content_type>'
-                       '<last_modified>2007-03-04 20:32:17</last_modified>'
-                       '</object>\n')
-            objects.append('<object><name>object2</name>'
-                       '<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
-                       '<bytes>64</bytes>'
-                       '<content_type>application/octet-stream</content_type>'
-                       '<last_modified>2007-03-04 20:32:17</last_modified>'
-                       '</object>\n')
-            objects.append('<object><name>object3</name>'
-                       '<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
-                       '<bytes>14</bytes>'
-                       '<content_type>application/octet-stream</content_type>'
-                       '<last_modified>2007-03-04 20:32:17</last_modified>'
-                       '</object>\n')
-            objects.append('<object><name>object4</name>'
-                       '<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
-                       '<bytes>64</bytes>'
-                       '<content_type>application/octet-stream</content_type>'
-                       '<last_modified>2007-03-04 20:32:17</last_modified>'
-                       '</object>\n')
-            objects.append('<object><name>object5</name>'
-                       '<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
-                       '<bytes>14</bytes>'
-                       '<content_type>application/octet-stream</content_type>'
-                       '<last_modified>2007-03-04 20:32:17</last_modified>'
-                       '</object>\n')
-            objects.append('<object><name>object6</name>'
-                       '<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
-                       '<bytes>64</bytes>'
-                       '<content_type>application/octet-stream</content_type>'
-                       '<last_modified>2007-03-04 20:32:17</last_modified>'
-                       '</object>\n')
-            objects.append('<object><name>object7</name>'
-                       '<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
-                       '<bytes>14</bytes>'
-                       '<content_type>application/octet-stream</content_type>'
-                       '<last_modified>2007-03-04 20:32:17</last_modified>'
-                       '</object>\n')
-            objects.append('<object><name>object8</name>'
-                       '<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
-                       '<bytes>64</bytes>'
-                       '<content_type>application/octet-stream</content_type>'
-                       '<last_modified>2007-03-04 20:32:17</last_modified>'
-                       '</object>\n')
-            objects = objects[left:right]
-            objects.insert(0, '<?xml version="1.0" encoding="UTF-8"?>\n')
-            objects.insert(1, '<container name="test_container_1"\n')
-            objects.append('</container>\n')
-            output = ''.join(objects)
-        else:
-            objects = ['object%s\n' % i for i in range(1, 9)]
-            objects = objects[left:right]
-            output = ''.join(objects)
-
-        # prefix/path don't make much sense given our test data
-        if 'prefix' in args or 'path' in args:
-            pass
-        return output
-
-    def render_GET(self, path, args):
-        # Special path that returns 404 Not Found
-        if (len(path) == 4) and (path[3] == 'bogus'):
-            self.write('HTTP/1.1 404 Not Found\n')
-            self.write('Content-Type: text/plain\n')
-            self.write('Content-Length: 0\n')
-            self.write('Connection: close\n\n')
-            return
-
-        self.write('HTTP/1.1 200 Ok\n')
-        self.write('Content-Type: text/plain\n')
-        if len(path) == 2:
-            content = self._create_GET_account_content(path, args)
-        elif len(path) == 3:
-            content = self._create_GET_container_content(path, args)
-        # Object
-        elif len(path) == 4:
-            content = 'I am a teapot, short and stout\n'
-        self.write('Content-Length: %d\n' % len(content))
-        self.write('Connection: close\n\n')
-        self.write(content)
-
-    def render_HEAD(self, path, args):
-        # Account
-        if len(path) == 2:
-            self.write('HTTP/1.1 204 No Content\n')
-            self.write('Content-Type: text/plain\n')
-            self.write('Connection: close\n')
-            self.write('X-Account-Container-Count: 3\n')
-            self.write('X-Account-Bytes-Used: 234\n\n')
-        else:
-            self.write('HTTP/1.1 200 Ok\n')
-            self.write('Content-Type: text/plain\n')
-            self.write('ETag: d5c7f3babf6c602a8da902fb301a9f27\n')
-            self.write('Content-Length: 21\n')
-            self.write('Connection: close\n\n')
-
-    def render_POST(self, path, args):
-        self.write('HTTP/1.1 202 Ok\n')
-        self.write('Connection: close\n\n')
-
-    def render_PUT(self, path, args):
-        self.write('HTTP/1.1 200 Ok\n')
-        self.write('Content-Type: text/plain\n')
-        self.write('Connection: close\n\n')
-    render_DELETE = render_PUT
-
-    def render(self, method, uri):
-        if '?' in uri:
-            parts = uri.split('?')
-            query = parts[1].strip('&').split('&')
-            args = dict([tuple(i.split('=', 1)) for i in query])
-            path = parts[0].strip('/').split('/')
-        else:
-            args = {}
-            path = uri.strip('/').split('/')
-
-        if hasattr(self, 'render_%s' % method):
-            getattr(self, 'render_%s' % method)(path, args)
-        else:
-            self.write('HTTP/1.1 406 Not Acceptable\n')
-            self.write('Content-Type: text/plain\n')
-            self.write('Connection: close\n')
-
-    def makefile(self, mode, flags):
-        self._rbuffer.seek(0)
-        lines = self.read().splitlines()
-        (method, uri, version) = lines[0].split()
-
-        self.render(method, uri)
-
-        self._wbuffer.seek(0)
-        return self._wbuffer
-
-
-class CustomHTTPConnection(connbase):
-    def connect(self):
-        self.sock = TrackerSocket()
-
-
-if __name__ == '__main__':
-    conn = CustomHTTPConnection('localhost', 8000)
-    conn.request('HEAD', '/v1/account/container/object')
-    response = conn.getresponse()
-    print "Status:", response.status, response.reason
-    for (key, value) in response.getheaders():
-        print "%s: %s" % (key, value)
-    print response.read()

tests/unit/test_filesystem_store.py

@@ -0,0 +1,135 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack, LLC
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Tests the filesystem backend store"""
+
+import StringIO
+import unittest
+import urlparse
+
+import stubout
+
+from glance.common import exception
+from glance.store.filesystem import FilesystemBackend, ChunkedFile
+from tests import stubs
+
+
+class TestFilesystemBackend(unittest.TestCase):
+
+    def setUp(self):
+        """Establish a clean test environment"""
+        self.stubs = stubout.StubOutForTesting()
+        stubs.stub_out_filesystem_backend()
+        self.orig_chunksize = ChunkedFile.CHUNKSIZE
+        ChunkedFile.CHUNKSIZE = 10
+
+    def tearDown(self):
+        """Clear the test environment"""
+        stubs.clean_out_fake_filesystem_backend()
+        self.stubs.UnsetAll()
+        ChunkedFile.CHUNKSIZE = self.orig_chunksize
+
+    def test_get(self):
+        """Test a "normal" retrieval of an image in chunks"""
+        url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2")
+        image_file = FilesystemBackend.get(url_pieces)
+
+        expected_data = "chunk00000remainder"
+        expected_num_chunks = 2
+        data = ""
+        num_chunks = 0
+
+        for chunk in image_file:
+            num_chunks += 1
+            data += chunk
+        self.assertEqual(expected_data, data)
+        self.assertEqual(expected_num_chunks, num_chunks)
+
+    def test_get_non_existing(self):
+        """
+        Test that trying to retrieve a file that doesn't exist
+        raises an error
+        """
+        url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing")
+        self.assertRaises(exception.NotFound,
+                          FilesystemBackend.get,
+                          url_pieces)
+
+    def test_add(self):
+        """Test that we can add an image via the filesystem backend"""
+        ChunkedFile.CHUNKSIZE = 1024
+        expected_image_id = 42
+        expected_file_size = 1024 * 5  # 5K
+        expected_file_contents = "*" * expected_file_size
+        expected_location = "file://%s/%s" % (stubs.FAKE_FILESYSTEM_ROOTDIR,
+                                              expected_image_id)
+        image_file = StringIO.StringIO(expected_file_contents)
+        options = {'verbose': True,
+                   'debug': True,
+                   'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR}
+
+        location, size = FilesystemBackend.add(42, image_file, options)
+
+        self.assertEquals(expected_location, location)
+        self.assertEquals(expected_file_size, size)
+
+        url_pieces = urlparse.urlparse("file:///tmp/glance-tests/42")
+        new_image_file = FilesystemBackend.get(url_pieces)
+        new_image_contents = ""
+        new_image_file_size = 0
+
+        for chunk in new_image_file:
+            new_image_file_size += len(chunk)
+            new_image_contents += chunk
+
+        self.assertEquals(expected_file_contents, new_image_contents)
+        self.assertEquals(expected_file_size, new_image_file_size)
+
+    def test_add_already_existing(self):
+        """
+        Tests that adding an image with an existing identifier
+        raises an appropriate exception
+        """
+        image_file = StringIO.StringIO("nevergonnamakeit")
+        options = {'verbose': True,
+                   'debug': True,
+                   'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR}
+        self.assertRaises(exception.Duplicate,
+                          FilesystemBackend.add,
+                          2, image_file, options)
+
+    def test_delete(self):
+        """
+        Test we can delete an existing image in the filesystem store
+        """
+        url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2")
+
+        FilesystemBackend.delete(url_pieces)
+
+        self.assertRaises(exception.NotFound,
+                          FilesystemBackend.get,
+                          url_pieces)
+
+    def test_delete_non_existing(self):
+        """
+        Test that trying to delete a file that doesn't exist
+        raises an error
+        """
+        url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing")
+        self.assertRaises(exception.NotFound,
+                          FilesystemBackend.delete,
+                          url_pieces)

tests/unit/test_stores.py

@@ -40,27 +40,6 @@ class TestBackend(unittest.TestCase):
         self.stubs.UnsetAll()
 
 
-class TestFilesystemBackend(TestBackend):
-
-    def setUp(self):
-        """Establish a clean test environment"""
-        stubs.stub_out_filesystem_backend()
-
-    def tearDown(self):
-        """Clear the test environment"""
-        stubs.clean_out_fake_filesystem_backend()
-
-    def test_get(self):
-
-        fetcher = get_from_backend("file:///tmp/glance-tests/2",
-                                   expected_size=19)
-
-        data = ""
-        for chunk in fetcher:
-            data += chunk
-        self.assertEqual(data, "chunk00000remainder")
-
-
 class TestHTTPBackend(TestBackend):
 
     def setUp(self):
@@ -104,45 +83,3 @@ class TestS3Backend(TestBackend):
 
         chunks = [c for c in fetcher]
         self.assertEqual(chunks, expected_returns)
-
-
-class TestSwiftBackend(TestBackend):
-    def setUp(self):
-        super(TestSwiftBackend, self).setUp()
-        stubs.stub_out_swift_backend(self.stubs)
-
-    def test_get(self):
-
-        swift_uri = "swift://user:pass@localhost/container1/file.tar.gz"
-        expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
-                            'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
-
-        fetcher = get_from_backend(swift_uri,
-                                   expected_size=21,
-                                   conn_class=SwiftBackend)
-
-        chunks = [c for c in fetcher]
-
-        self.assertEqual(chunks, expected_returns)
-
-    def test_get_bad_uri(self):
-
-        swift_url = "swift://localhost/container1/file.tar.gz"
-
-        self.assertRaises(BackendException, get_from_backend,
-                          swift_url, expected_size=21)
-
-    def test_url_parsing(self):
-
-        swift_uri = "swift://user:pass@localhost/v1.0/container1/file.tar.gz"
-
-        parsed_uri = urlparse.urlparse(swift_uri)
-
-        (user, key, authurl, container, obj) = \
-            SwiftBackend._parse_swift_tokens(parsed_uri)
-
-        self.assertEqual(user, 'user')
-        self.assertEqual(key, 'pass')
-        self.assertEqual(authurl, 'https://localhost/v1.0')
-        self.assertEqual(container, 'container1')
-        self.assertEqual(obj, 'file.tar.gz')

tests/unit/test_swift_store.py

@@ -0,0 +1,261 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack, LLC
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this swift except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Tests the Swift backend store"""
+
+import StringIO
+import hashlib
+import httplib
+import sys
+import unittest
+import urlparse
+
+import stubout
+
+from glance.common import exception
+import glance.store.swift
+
+SwiftBackend = glance.store.swift.SwiftBackend
+
+SWIFT_INSTALLED = False
+
+try:
+    import swift.common.client
+    SWIFT_INSTALLED = True
+except ImportError:
+    print "Skipping Swift store tests since Swift is not installed."
+
+FIVE_KB = (5 * 1024)
+SWIFT_OPTIONS = {'verbose': True,
+                 'debug': True,
+                 'swift_store_user': 'glance',
+                 'swift_store_key': 'key',
+                 'swift_store_auth_address': 'localhost:8080',
+                 'swift_store_container': 'glance'}
+
+
+# We stub out as little as possible to ensure that the code paths
+# between glance.store.swift and swift.common.client are tested
+# thoroughly
+def stub_out_swift_common_client(stubs):
+
+    fixture_headers = {'glance/2':
+                {'content-length': FIVE_KB,
+                 'etag': 'c2e5db72bd7fd153f53ede5da5a06de3'}}
+    fixture_objects = {'glance/2':
+                       StringIO.StringIO("*" * FIVE_KB)}
+
+    def fake_put_object(url, token, container, name, contents, **kwargs):
+        # PUT returns the ETag header for the newly-added object
+        fixture_key = "%s/%s" % (container, name)
+        if not fixture_key in fixture_headers.keys():
+            if hasattr(contents, 'read'):
+                fixture_object = StringIO.StringIO()
+                chunk = contents.read(SwiftBackend.CHUNKSIZE)
+                while chunk:
+                    fixture_object.write(chunk)
+                    chunk = contents.read(SwiftBackend.CHUNKSIZE)
+            else:
+                fixture_object = StringIO.StringIO(contents)
+            fixture_objects[fixture_key] = fixture_object
+            fixture_headers[fixture_key] = {
+                'content-length': fixture_object.len,
+                'etag': hashlib.md5(fixture_object.read()).hexdigest()}
+            return fixture_headers[fixture_key]['etag']
+        else:
+            msg = ("Object PUT failed - Object with key %s already exists"
+                   % fixture_key)
+            raise swift.common.client.ClientException(msg,
+                        http_status=httplib.CONFLICT)
+
+    def fake_get_object(url, token, container, name, **kwargs):
+        # GET returns the tuple (list of headers, file object)
+        try:
+            fixture_key = "%s/%s" % (container, name)
+            return fixture_headers[fixture_key], fixture_objects[fixture_key]
+        except KeyError:
+            msg = "Object GET failed"
+            raise swift.common.client.ClientException(msg,
+                        http_status=httplib.NOT_FOUND)
+
+    def fake_head_object(url, token, container, name, **kwargs):
+        # HEAD returns the list of headers for an object
+        try:
+            fixture_key = "%s/%s" % (container, name)
+            return fixture_headers[fixture_key]
+        except KeyError:
+            msg = "Object HEAD failed - Object does not exist"
+            raise swift.common.client.ClientException(msg,
+                        http_status=httplib.NOT_FOUND)
+
+    def fake_delete_object(url, token, container, name, **kwargs):
+        # DELETE returns nothing
+        fixture_key = "%s/%s" % (container, name)
+        if fixture_key not in fixture_headers.keys():
+            msg = "Object DELETE failed - Object does not exist"
+            raise swift.common.client.ClientException(msg,
+                        http_status=httplib.NOT_FOUND)
+        else:
+            del fixture_headers[fixture_key]
+            del fixture_objects[fixture_key]
+
+    def fake_get_connection_class(*args):
+        return swift.common.client.Connection
+
+    def fake_http_connection(self):
+        return None
+
+    def fake_get_auth(self):
+        return None, None
+
+    stubs.Set(swift.common.client,
+              'put_object', fake_put_object)
+    stubs.Set(swift.common.client,
+              'delete_object', fake_delete_object)
+    stubs.Set(swift.common.client,
+              'head_object', fake_head_object)
+    stubs.Set(swift.common.client,
+              'get_object', fake_get_object)
+    stubs.Set(swift.common.client.Connection,
+              'get_auth', fake_get_auth)
+    stubs.Set(swift.common.client.Connection,
+              'http_connection', fake_http_connection)
+    stubs.Set(glance.store.swift,
+              'get_connection_class', fake_get_connection_class)
+
+
+class TestSwiftBackend(unittest.TestCase):
+
+    def setUp(self):
+        """Establish a clean test environment"""
+        self.stubs = stubout.StubOutForTesting()
+        if SWIFT_INSTALLED:
+            stub_out_swift_common_client(self.stubs)
+
+    def tearDown(self):
+        """Clear the test environment"""
+        self.stubs.UnsetAll()
+
+    def test_get(self):
+        """Test a "normal" retrieval of an image in chunks"""
+        if not SWIFT_INSTALLED:
+            return
+        url_pieces = urlparse.urlparse(
+            "swift://user:key@auth_address/glance/2")
+        image_swift = SwiftBackend.get(url_pieces)
+
+        expected_data = "*" * FIVE_KB
+        data = ""
+
+        for chunk in image_swift:
+            data += chunk
+        self.assertEqual(expected_data, data)
+
+    def test_get_mismatched_expected_size(self):
+        """
+        Test retrieval of an image with wrong expected_size param
+        raises an exception
+        """
+        if not SWIFT_INSTALLED:
+            return
+        url_pieces = urlparse.urlparse(
+            "swift://user:key@auth_address/glance/2")
+        self.assertRaises(glance.store.BackendException,
+                          SwiftBackend.get,
+                          url_pieces,
+                          {'expected_size': 42})
+
+    def test_get_non_existing(self):
+        """
+        Test that trying to retrieve a swift that doesn't exist
+        raises an error
+        """
+        if not SWIFT_INSTALLED:
+            return
+        url_pieces = urlparse.urlparse(
+            "swift://user:key@auth_address/noexist")
+        self.assertRaises(exception.NotFound,
+                          SwiftBackend.get,
+                          url_pieces)
+
+    def test_add(self):
+        """Test that we can add an image via the swift backend"""
+        if not SWIFT_INSTALLED:
+            return
+        expected_image_id = 42
+        expected_swift_size = 1024 * 5  # 5K
+        expected_swift_contents = "*" * expected_swift_size
+        expected_location = "swift://%s:%s@%s/%s/%s" % (
+            SWIFT_OPTIONS['swift_store_user'],
+            SWIFT_OPTIONS['swift_store_key'],
+            SWIFT_OPTIONS['swift_store_auth_address'],
+            SWIFT_OPTIONS['swift_store_container'],
+            expected_image_id)
+        image_swift = StringIO.StringIO(expected_swift_contents)
+
+        location, size = SwiftBackend.add(42, image_swift, SWIFT_OPTIONS)
+
+        self.assertEquals(expected_location, location)
+        self.assertEquals(expected_swift_size, size)
+
+        url_pieces = urlparse.urlparse(
+            "swift://user:key@auth_address/glance/42")
+        new_image_swift = SwiftBackend.get(url_pieces)
+        new_image_contents = new_image_swift.getvalue()
+        new_image_swift_size = new_image_swift.len
+
+        self.assertEquals(expected_swift_contents, new_image_contents)
+        self.assertEquals(expected_swift_size, new_image_swift_size)
+
+    def test_add_already_existing(self):
+        """
+        Tests that adding an image with an existing identifier
+        raises an appropriate exception
+        """
+        if not SWIFT_INSTALLED:
+            return
+        image_swift = StringIO.StringIO("nevergonnamakeit")
+        self.assertRaises(exception.Duplicate,
+                          SwiftBackend.add,
+                          2, image_swift, SWIFT_OPTIONS)
+
+    def test_delete(self):
+        """
+        Test we can delete an existing image in the swift store
+        """
+        if not SWIFT_INSTALLED:
+            return
+        url_pieces = urlparse.urlparse(
+            "swift://user:key@auth_address/glance/2")
+
+        SwiftBackend.delete(url_pieces)
+
+        self.assertRaises(exception.NotFound,
+                          SwiftBackend.get,
+                          url_pieces)
+
+    def test_delete_non_existing(self):
+        """
+        Test that trying to delete a swift that doesn't exist
+        raises an error
+        """
+        if not SWIFT_INSTALLED:
+            return
+        url_pieces = urlparse.urlparse("swift://user:key@auth_address/noexist")
+        self.assertRaises(exception.NotFound,
+                          SwiftBackend.delete,
+                          url_pieces)

tools/pip-requires

@@ -12,5 +12,6 @@ nose
 sphinx
 argparse
 mox==0.5.0
+swift
 -f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz
 sqlalchemy-migrate>=0.6