Implements the S3 store to the level of the swift store.

This branch is Chris' work with a merge of trunk, fix of merge conflicts from trunk, and moving the import of boto into a conditional block so tests can run with the fakes when boto is not installed on the local machine.

glance/store/__init__.py

@@ -53,13 +53,17 @@ def get_backend_class(backend):
     """
     # NOTE(sirp): avoiding circular import
     from glance.store.http import HTTPBackend
+    from glance.store.s3 import S3Backend
     from glance.store.swift import SwiftBackend
     from glance.store.filesystem import FilesystemBackend
 
-    BACKENDS = {"file": FilesystemBackend,
+    BACKENDS = {
-                "http": HTTPBackend,
+        "file": FilesystemBackend,
-                "https": HTTPBackend,
+        "http": HTTPBackend,
-                "swift": SwiftBackend}
+        "https": HTTPBackend,
+        "swift": SwiftBackend,
+        "s3": S3Backend
+    }
 
     try:
         return BACKENDS[backend]
@@ -99,3 +103,41 @@ def get_store_from_location(location):
     """
     loc_pieces = urlparse.urlparse(location)
     return loc_pieces.scheme
+
+
+def parse_uri_tokens(parsed_uri, example_url):
+    """
+    Given a URI and an example_url, attempt to parse the uri to assemble an
+    authurl. This method returns the user, key, authurl, referenced container,
+    and the object we're looking for in that container.
+
+    Parsing the uri is three phases:
+        1) urlparse to split the tokens
+        2) use RE to split on @ and /
+        3) reassemble authurl
+    
+    """
+    path = parsed_uri.path.lstrip('//')
+    netloc = parsed_uri.netloc
+
+    try:
+        try:
+            creds, netloc = netloc.split('@')
+        except ValueError:
+            # Python 2.6.1 compat
+            # see lp659445 and Python issue7904
+            creds, path = path.split('@')
+        user, key = creds.split(':')
+        path_parts = path.split('/')
+        obj = path_parts.pop()
+        container = path_parts.pop()
+    except (ValueError, IndexError):
+        raise BackendException(
+             "Expected four values to unpack in: %s:%s. "
+             "Should have received something like: %s."
+             % (parsed_uri.scheme, parsed_uri.path, example_url))
+
+    authurl = "https://%s" % '/'.join(path_parts)
+
+    return user, key, authurl, container, obj
+    

glance/store/backends/__init__.py

@@ -14,107 +14,3 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
-import os
-import urlparse
-
-from glance.common import exception
-
-
-# TODO(sirp): should this be moved out to common/utils.py ?
-def _file_iter(f, size):
-    """
-    Return an iterator for a file-like object
-    """
-    chunk = f.read(size)
-    while chunk:
-        yield chunk
-        chunk = f.read(size)
-
-
-class BackendException(Exception):
-    pass
-
-
-class UnsupportedBackend(BackendException):
-    pass
-
-
-class Backend(object):
-    CHUNKSIZE = 4096
-
-
-class FilesystemBackend(Backend):
-    @classmethod
-    def get(cls, parsed_uri, expected_size, opener=lambda p: open(p, "rb")):
-        """ Filesystem-based backend
-
-        file:///path/to/file.tar.gz.0
-        """
-        #FIXME: must prevent attacks using ".." and "." paths
-        with opener(parsed_uri.path) as f:
-            return _file_iter(f, cls.CHUNKSIZE)
-
-    @classmethod
-    def delete(cls, parsed_uri):
-        """
-        Removes a file from the filesystem backend.
-
-        :param parsed_uri: Parsed pieces of URI in form of::
-            file:///path/to/filename.ext
-
-        :raises NotFound if file does not exist
-        :raises NotAuthorized if cannot delete because of permissions
-        """
-        fn = parsed_uri.path
-        if os.path.exists(fn):
-            try:
-                os.unlink(fn)
-            except OSError:
-                raise exception.NotAuthorized("You cannot delete file %s" % fn)
-        else:
-            raise exception.NotFound("File %s does not exist" % fn)
-
-
-def get_backend_class(backend):
-    """
-    Returns the backend class as designated in the
-    backend name
-
-    :param backend: Name of backend to create
-    """
-    # NOTE(sirp): avoiding circular import
-    from glance.store.backends.http import HTTPBackend
-    from glance.store.backends.swift import SwiftBackend
-
-    BACKENDS = {"file": FilesystemBackend,
-                "http": HTTPBackend,
-                "https": HTTPBackend,
-                "swift": SwiftBackend}
-
-    try:
-        return BACKENDS[backend]
-    except KeyError:
-        raise UnsupportedBackend("No backend found for '%s'" % scheme)
-
-
-def get_from_backend(uri, **kwargs):
-    """Yields chunks of data from backend specified by uri"""
-
-    parsed_uri = urlparse.urlparse(uri)
-    scheme = parsed_uri.scheme
-
-    backend_class = get_backend_class(scheme)
-
-    return backend_class.get(parsed_uri, **kwargs)
-
-
-def delete_from_backend(uri, **kwargs):
-    """Removes chunks of data from backend specified by uri"""
-
-    parsed_uri = urlparse.urlparse(uri)
-    scheme = parsed_uri.scheme
-
-    backend_class = get_backend_class(scheme)
-
-    return backend_class.delete(parsed_uri, **kwargs)

glance/store/s3.py

@@ -0,0 +1,109 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 OpenStack, LLC
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""The s3 backend adapter"""
+
+import glance.store
+
+
+class S3Backend(glance.store.Backend):
+    """An implementation of the s3 adapter."""
+
+    EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0"
+
+    @classmethod
+    def get(cls, parsed_uri, expected_size, conn_class=None):
+        """
+        Takes a parsed_uri in the format of:
+        s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects
+        to s3 and downloads the file. Returns the generator resp_body provided
+        by get_object.
+        """
+
+        if conn_class:
+            pass
+        else:
+            import boto.s3.connection
+            conn_class = boto.s3.connection.S3Connection
+
+        (access_key, secret_key, host, bucket, obj) = \
+            cls._parse_s3_tokens(parsed_uri)
+
+        # Close the connection when we're through.
+        with conn_class(access_key, secret_key, host=host) as s3_conn:
+            bucket = cls._get_bucket(s3_conn, bucket)
+
+            # Close the key when we're through.
+            with cls._get_key(bucket, obj) as key:
+                if not key.size == expected_size:
+                    raise glance.store.BackendException(
+                        "Expected %s bytes, got %s" %
+                        (expected_size, key.size))
+
+                key.BufferSize = cls.CHUNKSIZE
+                for chunk in key:
+                    yield chunk
+
+    @classmethod
+    def delete(cls, parsed_uri, conn_class=None):
+        """
+        Takes a parsed_uri in the format of:
+        s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects
+        to s3 and deletes the file. Returns whatever boto.s3.key.Key.delete()
+        returns.
+        """
+
+        if conn_class:
+            pass
+        else:
+            conn_class = boto.s3.connection.S3Connection
+
+        (access_key, secret_key, host, bucket, obj) = \
+            cls._parse_s3_tokens(parsed_uri)
+
+        # Close the connection when we're through.
+        with conn_class(access_key, secret_key, host=host) as s3_conn:
+            bucket = cls._get_bucket(s3_conn, bucket)
+
+            # Close the key when we're through.
+            with cls._get_key(bucket, obj) as key:
+                return key.delete()
+
+    @classmethod
+    def _get_bucket(cls, conn, bucket_id):
+        """Get a bucket from an s3 connection"""
+
+        bucket = conn.get_bucket(bucket_id)
+        if not bucket:
+            raise glance.store.BackendException("Could not find bucket: %s" %
+                bucket_id)
+
+        return bucket
+
+    @classmethod
+    def _get_key(cls, bucket, obj):
+        """Get a key from a bucket"""
+
+        key = bucket.get_key(obj)
+        if not key:
+            raise glance.store.BackendException("Could not get key: %s" % key)
+        return key
+
+    @classmethod
+    def _parse_s3_tokens(cls, parsed_uri):
+        """Parse tokens from the parsed_uri"""
+        return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL)

glance/store/swift.py

@@ -15,6 +15,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from __future__ import absolute_import
 import glance.store
 
 
@@ -114,21 +115,7 @@ class SwiftBackend(glance.store.Backend):
 
 
 def get_connection_class(conn_class):
-    if conn_class:
+    if not conn_class:
-        pass  # Use the provided conn_class
+        import swift.common.client
-    else:
-        # NOTE(sirp): A standard import statement won't work here because
-        # this file ('swift.py') is shadowing the swift module, and since
-        # the import statement searches locally before globally, we'd end
-        # up importing ourselves.
-        #
-        # NOTE(jaypipes): This can be resolved by putting this code in
-        #                 /glance/store/swift/__init__.py
-        #
-        # see http://docs.python.org/library/functions.html#__import__
-        PERFORM_ABSOLUTE_IMPORTS = 0
-        swift = __import__('swift.common.client', globals(), locals(), [],
-                            PERFORM_ABSOLUTE_IMPORTS)
-
         conn_class = swift.common.client.Connection
     return conn_class

tests/stubs.py

@@ -105,6 +105,44 @@ def stub_out_filesystem_backend():
     f.close()
 
 
+def stub_out_s3_backend(stubs):
+    """ Stubs out the S3 Backend with fake data and calls.
+
+    The stubbed swift backend provides back an iterator over
+    the data ""
+
+    :param stubs: Set of stubout stubs
+
+    """
+
+    class FakeSwiftAuth(object):
+        pass
+    class FakeS3Connection(object):
+        pass
+
+    class FakeS3Backend(object):
+        CHUNK_SIZE = 2
+        DATA = 'I am a teapot, short and stout\n'
+
+        @classmethod
+        def get(cls, parsed_uri, expected_size, conn_class=None):
+            S3Backend = glance.store.s3.S3Backend
+
+            # raise BackendException if URI is bad.
+            (user, key, authurl, container, obj) = \
+                S3Backend._parse_s3_tokens(parsed_uri)
+
+            def chunk_it():
+                for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE):
+                    yield cls.DATA[i:i+cls.CHUNK_SIZE]
+            
+            return chunk_it()
+
+    fake_swift_backend = FakeS3Backend()
+    stubs.Set(glance.store.s3.S3Backend, 'get',
+              fake_swift_backend.get)
+
+
 def stub_out_swift_backend(stubs):
     """Stubs out the Swift Glance backend with fake data
     and calls.

tests/unit/test_stores.py

@@ -21,6 +21,7 @@ import stubout
 import unittest
 import urlparse
 
+from glance.store.s3 import S3Backend
 from glance.store.swift import SwiftBackend
 from glance.store import Backend, BackendException, get_from_backend
 from tests import stubs
@@ -87,6 +88,25 @@ class TestHTTPBackend(TestBackend):
         self.assertEqual(chunks, expected_returns)
 
 
+class TestS3Backend(TestBackend):
+    def setUp(self):
+        super(TestS3Backend, self).setUp()
+        stubs.stub_out_s3_backend(self.stubs)
+
+    def test_get(self):
+        s3_uri = "s3://user:password@localhost/bucket1/file.tar.gz"
+
+        expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', 
+                            'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
+        fetcher = get_from_backend(s3_uri, 
+                                   expected_size=8, 
+                                   conn_class=S3Backend)
+
+        chunks = [c for c in fetcher]
+        self.assertEqual(chunks, expected_returns)
+
+
+
 class TestSwiftBackend(TestBackend):
 
     def setUp(self):