Implement S3 to the level of swift

glance/store/__init__.py

@@ -53,6 +53,7 @@ def get_backend_class(backend):
     """
     # NOTE(sirp): avoiding circular import
     from glance.store.http import HTTPBackend
+    from glance.store.s3 import S3Backend
     from glance.store.swift import SwiftBackend
     from glance.store.filesystem import FilesystemBackend
 
@@ -60,13 +61,14 @@ def get_backend_class(backend):
         "file": FilesystemBackend,
         "http": HTTPBackend,
         "https": HTTPBackend,
-        "swift": SwiftBackend
+        "swift": SwiftBackend,
+        "s3": S3Backend
     }
 
     try:
         return BACKENDS[backend]
     except KeyError:
-        raise UnsupportedBackend("No backend found for '%s'" % scheme)
+        raise UnsupportedBackend("No backend found for '%s'" % backend)
 
 
 def get_from_backend(uri, **kwargs):
@@ -101,3 +103,41 @@ def get_store_from_location(location):
     """
     loc_pieces = urlparse.urlparse(location)
     return loc_pieces.scheme
+
+
+def parse_uri_tokens(parsed_uri, example_url):
+    """
+    Given a URI and an example_url, attempt to parse the uri to assemble an
+    authurl. This method returns the user, key, authurl, referenced container,
+    and the object we're looking for in that container.
+
+    Parsing the uri is three phases:
+        1) urlparse to split the tokens
+        2) use RE to split on @ and /
+        3) reassemble authurl
+    
+    """
+    path = parsed_uri.path.lstrip('//')
+    netloc = parsed_uri.netloc
+
+    try:
+        try:
+            creds, netloc = netloc.split('@')
+        except ValueError:
+            # Python 2.6.1 compat
+            # see lp659445 and Python issue7904
+            creds, path = path.split('@')
+        user, key = creds.split(':')
+        path_parts = path.split('/')
+        obj = path_parts.pop()
+        container = path_parts.pop()
+    except (ValueError, IndexError):
+        raise BackendException(
+             "Expected four values to unpack in: %s:%s. "
+             "Should have received something like: %s."
+             % (parsed_uri.scheme, parsed_uri.path, example_url))
+
+    authurl = "https://%s" % '/'.join(path_parts)
+
+    return user, key, authurl, container, obj
+    

glance/store/backends/__init__.py

@@ -86,18 +86,20 @@ def get_backend_class(backend):
     # NOTE(sirp): avoiding circular import
     from glance.store.backends.http import HTTPBackend
     from glance.store.backends.swift import SwiftBackend
+    from glance.store.backends.s3 import S3Backend
 
     BACKENDS = {
         "file": FilesystemBackend,
         "http": HTTPBackend,
         "https": HTTPBackend,
-        "swift": SwiftBackend
+        "swift": SwiftBackend,
+        "s3": S3Backend
     }
 
     try:
         return BACKENDS[backend]
     except KeyError:
-        raise UnsupportedBackend("No backend found for '%s'" % scheme)
+        raise UnsupportedBackend("No backend found for '%s'" % backend)
 
 
 def get_from_backend(uri, **kwargs):

glance/store/s3.py

@@ -0,0 +1,110 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 OpenStack, LLC
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+""" the s3 backend adapter """
+
+from __future__ import absolute_import
+import glance.store
+import boto.s3.connection
+
+
+class S3Backend(glance.store.Backend):
+    """ An implementation of the s3 adapter.  """
+
+    EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0"
+
+    @classmethod
+    def get(cls, parsed_uri, expected_size, conn_class=None):
+        """
+        Takes a parsed_uri in the format of: 
+        s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects to s3
+        and downloads the file. Returns the generator resp_body provided by 
+        get_object.
+        """
+
+        if conn_class: 
+            pass
+        else:
+            conn_class = boto.s3.connection.S3Connection
+
+        (access_key, secret_key, host, bucket, obj) = \
+            cls._parse_s3_tokens(parsed_uri)
+
+        # Close the connection when we're through.
+        with conn_class(access_key, secret_key, host=host) as s3_conn:
+            bucket = cls._get_bucket(s3_conn, bucket)
+
+            # Close the key when we're through.
+            with cls._get_key(bucket, obj) as key:
+                if not key.size == expected_size:
+                    raise glance.store.BackendException("Expected %s bytes, got %s"
+                        % (expected_size, key.size))
+
+                key.BufferSize = cls.CHUNKSIZE
+                for chunk in key:
+                    yield chunk
+
+    @classmethod
+    def delete(cls, parsed_uri, conn_class=None):
+        """
+        Takes a parsed_uri in the format of: 
+        s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects to s3
+        and deletes the file. Returns whatever boto.s3.key.Key.delete() returns.
+        """
+
+        if conn_class: 
+            pass
+        else:
+            conn_class = boto.s3.connection.S3Connection
+
+        (access_key, secret_key, host, bucket, obj) = \
+            cls._parse_s3_tokens(parsed_uri)
+
+        # Close the connection when we're through.
+        with conn_class(access_key, secret_key, host=host) as s3_conn:
+            bucket = cls._get_bucket(s3_conn, bucket)
+
+            # Close the key when we're through.
+            with cls._get_key(bucket, obj) as key:
+                return key.delete()
+
+
+
+    @classmethod
+    def _get_bucket(cls, conn, bucket_id):
+        """ Get a bucket from an s3 connection """
+
+        bucket = conn.get_bucket(bucket_id)
+        if not bucket:
+            raise glance.store.BackendException("Could not find bucket: %s" % 
+                bucket_id)
+
+        return bucket
+
+    @classmethod
+    def _get_key(cls, bucket, obj):
+        """ Get a key from a bucket """
+
+        key = bucket.get_key(obj)
+        if not key:
+            raise glance.store.BackendException("Could not get key: %s" % key)
+        return key
+
+    @classmethod
+    def _parse_s3_tokens(cls, parsed_uri):
+        """ Parse tokens from the parsed_uri """
+        return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL)

glance/store/swift.py

@@ -15,7 +15,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from __future__ import absolute_import
 import glance.store
+import swift.common.client
 
 
 class SwiftBackend(glance.store.Backend):
@@ -113,21 +115,6 @@ class SwiftBackend(glance.store.Backend):
 
 
 def get_connection_class(conn_class):
-    if conn_class:
-        pass # Use the provided conn_class
-    else:
-        # NOTE(sirp): A standard import statement won't work here because
-        # this file ('swift.py') is shadowing the swift module, and since
-        # the import statement searches locally before globally, we'd end
-        # up importing ourselves.
-        #
-        # NOTE(jaypipes): This can be resolved by putting this code in
-        #                 /glance/store/swift/__init__.py
-        #
-        # see http://docs.python.org/library/functions.html#__import__
-        PERFORM_ABSOLUTE_IMPORTS = 0
-        swift = __import__('swift.common.client', globals(), locals(), [],
-                            PERFORM_ABSOLUTE_IMPORTS)
-
+    if not conn_class:
         conn_class = swift.common.client.Connection
     return conn_class