Merge "Support new image copied from external storage."

bin/glance

@@ -235,13 +235,21 @@ EXAMPLES
             print 'Found non-settable field %s. Removing.' % field
             fields.pop(field)
 
-    if 'location' in fields.keys():
+    def _external_source(fields, image_data):
-        image_meta['location'] = fields.pop('location')
+        source = None
+        features = {}
+        if 'location' in fields.keys():
+            source = fields.pop('location')
+            image_meta['location'] = source
+        elif 'copy_from' in fields.keys():
+            source = fields.pop('copy_from')
+            features['x-glance-api-copy-from'] = source
+        return source, features
 
     # We need either a location or image data/stream to add...
-    image_location = image_meta.get('location')
+    location, features = _external_source(fields, image_meta)
     image_data = None
-    if not image_location:
+    if not location:
         # Grab the image data stream from stdin or redirect,
         # otherwise error out
         image_data = sys.stdin
@@ -251,7 +259,8 @@ EXAMPLES
 
     if not options.dry_run:
         try:
-            image_meta = c.add_image(image_meta, image_data)
+            image_meta = c.add_image(image_meta, image_data,
+                                     features=features)
             image_id = image_meta['id']
             print "Added new image with ID: %s" % image_id
             if options.verbose:
@@ -278,9 +287,17 @@ EXAMPLES
             return FAILURE
     else:
         print "Dry run. We would have done the following:"
+
+        def _dump(dict):
+            for k, v in sorted(dict.items()):
+                print " %(k)30s => %(v)s" % locals()
+
         print "Add new image with metadata:"
-        for k, v in sorted(image_meta.items()):
+        _dump(image_meta)
-            print " %(k)30s => %(v)s" % locals()
+
+        if features:
+            print "with features enabled:"
+            _dump(features)
 
     return SUCCESS
 
@@ -299,7 +316,8 @@ to Glance that represents the metadata for an image.
 Field names that can be specified:
 
 name                A name for the image.
-location            The location of the image.
+location            An external location to serve out from.
+copy_from           An external location (HTTP, S3 or Swift URI) to copy from.
 is_public           If specified, interpreted as a boolean value
                     and sets or unsets the image's availability to the public.
 protected           If specified, interpreted as a boolean value

doc/source/glance.rst

@@ -260,13 +260,20 @@ To upload an EC2 tarball VM image with an associated property (e.g., distro)::
      container_format=ovf disk_format=raw \
      distro="ubuntu 10.10" < /root/maverick-server-uec-amd64.tar.gz
 
-To upload an EC2 tarball VM image from a URL::
+To reference an EC2 tarball VM image available at an external URL::
 
-  $> glance add name="uubntu-10.04-amd64" is_public=true \
+  $> glance add name="ubuntu-10.04-amd64" is_public=true \
      container_format=ovf disk_format=raw \
      location="http://uec-images.ubuntu.com/lucid/current/\
      lucid-server-uec-amd64.tar.gz"
 
+To upload a copy of that same EC2 tarball VM image::
+
+  $> glance add name="ubuntu-10.04-amd64" is_public=true \
+     container_format=ovf disk_format=raw \
+     copy_from="http://uec-images.ubuntu.com/lucid/current/\
+     lucid-server-uec-amd64.tar.gz"
+
 To upload a qcow2 image::
 
   $> glance add name="ubuntu-11.04-amd64" is_public=true \

glance/api/v1/images.py

@@ -226,6 +226,23 @@ class Controller(controller.BaseController):
             'image_meta': image_meta
         }
 
+    @staticmethod
+    def _copy_from(req):
+        return req.headers.get('x-glance-api-copy-from')
+
+    @staticmethod
+    def _external_source(image_meta, req):
+        return image_meta.get('location', Controller._copy_from(req))
+
+    @staticmethod
+    def _get_from_store(where):
+        try:
+            image_data, image_size = get_from_backend(where)
+        except exception.NotFound, e:
+            raise HTTPNotFound(explanation="%s" % e)
+        image_size = int(image_size) if image_size else None
+        return image_data, image_size
+
     def show(self, req, id):
         """
         Returns an iterator that can be used to retrieve an image's
@@ -239,16 +256,9 @@ class Controller(controller.BaseController):
         self._enforce(req, 'get_image')
         image_meta = self.get_active_image_meta_or_404(req, id)
 
-        def get_from_store(image_meta):
+        image_iterator, size = self._get_from_store(image_meta['location'])
-            try:
+        image_meta['size'] = size or image_meta['size']
-                location = image_meta['location']
-                image_data, image_size = get_from_backend(location)
-                image_meta["size"] = image_size or image_meta["size"]
-            except exception.NotFound, e:
-                raise HTTPNotFound(explanation="%s" % e)
-            return image_data
 
-        image_iterator = get_from_store(image_meta)
         del image_meta['location']
         return {
             'image_iterator': image_iterator,
@@ -263,11 +273,12 @@ class Controller(controller.BaseController):
 
         :param req: The WSGI/Webob Request object
         :param id: The opaque image identifier
+        :param image_meta: The image metadata
 
         :raises HTTPConflict if image already exists
         :raises HTTPBadRequest if image metadata is not valid
         """
-        location = image_meta.get('location')
+        location = self._external_source(image_meta, req)
         if location:
             store = get_store_from_location(location)
             # check the store exists before we hit the registry, but we
@@ -278,9 +289,9 @@ class Controller(controller.BaseController):
             image_meta['size'] = image_meta.get('size', 0) \
                                  or get_size_from_backend(location)
         else:
-            # Ensure that the size attribute is set to zero for uploadable
+            # Ensure that the size attribute is set to zero for directly
-            # images (if not provided). The size will be set to a non-zero
+            # uploadable images (if not provided). The size will be set
-            # value during upload
+            # to a non-zero value during upload
             image_meta['size'] = image_meta.get('size', 0)
 
         image_meta['status'] = 'queued'
@@ -317,13 +328,30 @@ class Controller(controller.BaseController):
         :raises HTTPConflict if image already exists
         :retval The location where the image was stored
         """
-        try:
+
-            req.get_content_type('application/octet-stream')
+        copy_from = self._copy_from(req)
-        except exception.InvalidContentType:
+        if copy_from:
-            self._safe_kill(req, image_meta['id'])
+            image_data, image_size = self._get_from_store(copy_from)
-            msg = _("Content-Type must be application/octet-stream")
+            image_meta['size'] = image_size or image_meta['size']
-            logger.error(msg)
+        else:
-            raise HTTPBadRequest(explanation=msg)
+            try:
+                req.get_content_type('application/octet-stream')
+            except exception.InvalidContentType:
+                self._safe_kill(req, image_meta['id'])
+                msg = _("Content-Type must be application/octet-stream")
+                logger.error(msg)
+                raise HTTPBadRequest(explanation=msg)
+
+            image_data = req.body_file
+
+            if req.content_length:
+                image_size = int(req.content_length)
+            elif 'x-image-meta-size' in req.headers:
+                image_size = int(req.headers['x-image-meta-size'])
+            else:
+                logger.debug(_("Got request with no content-length and no "
+                               "x-image-meta-size header"))
+                image_size = 0
 
         store_name = req.headers.get('x-image-meta-store',
                                      self.conf.default_store)
@@ -337,14 +365,6 @@ class Controller(controller.BaseController):
         try:
             logger.debug(_("Uploading image data for image %(image_id)s "
                          "to %(store_name)s store"), locals())
-            if req.content_length:
-                image_size = int(req.content_length)
-            elif 'x-image-meta-size' in req.headers:
-                image_size = int(req.headers['x-image-meta-size'])
-            else:
-                logger.debug(_("Got request with no content-length and no "
-                               "x-image-meta-size header"))
-                image_size = 0
 
             if image_size > IMAGE_SIZE_CAP:
                 max_image_size = IMAGE_SIZE_CAP
@@ -355,7 +375,7 @@ class Controller(controller.BaseController):
                 raise HTTPBadRequest(msg, request=request)
 
             location, size, checksum = store.add(image_meta['id'],
-                                                 req.body_file,
+                                                 image_data,
                                                  image_size)
 
             # Verify any supplied checksum value matches checksum
@@ -505,19 +525,27 @@ class Controller(controller.BaseController):
 
     def create(self, req, image_meta, image_data):
         """
-        Adds a new image to Glance. Three scenarios exist when creating an
+        Adds a new image to Glance. Four scenarios exist when creating an
         image:
 
-        1. If the image data is available for upload, create can be passed the
+        1. If the image data is available directly for upload, create can be
-           image data as the request body and the metadata as the request
+           passed the image data as the request body and the metadata as the
-           headers. The image will initially be 'queued', during upload it
+           request headers. The image will initially be 'queued', during
-           will be in the 'saving' status, and then 'killed' or 'active'
+           upload it will be in the 'saving' status, and then 'killed' or
-           depending on whether the upload completed successfully.
+           'active' depending on whether the upload completed successfully.
 
-        2. If the image data exists somewhere else, you can pass in the source
+        2. If the image data exists somewhere else, you can upload indirectly
-           using the x-image-meta-location header
+           from the external source using the x-glance-api-copy-from header.
+           Once the image is uploaded, the external store is not subsequently
+           consulted, i.e. the image content is served out from the configured
+           glance image store.  State transitions are as for option #1.
 
-        3. If the image data is not available yet, but you'd like reserve a
+        3. If the image data exists somewhere else, you can reference the
+           source using the x-image-meta-location header. The image content
+           will be served out from the external store, i.e. is never uploaded
+           to the configured glance image store.
+
+        4. If the image data is not available yet, but you'd like reserve a
            spot for it, you can omit the data and a record will be created in
            the 'queued' state. This exists primarily to maintain backwards
            compatibility with OpenStack/Rackspace API semantics.
@@ -547,7 +575,7 @@ class Controller(controller.BaseController):
         image_meta = self._reserve(req, image_meta)
         image_id = image_meta['id']
 
-        if image_data is not None:
+        if image_data or self._copy_from(req):
             image_meta = self._upload_and_activate(req, image_meta)
         else:
             location = image_meta.get('location')
@@ -594,10 +622,12 @@ class Controller(controller.BaseController):
         if image_data is not None and orig_status != 'queued':
             raise HTTPConflict(_("Cannot upload to an unqueued image"))
 
-        # Only allow the Location fields to be modified if the image is
+        # Only allow the Location|Copy-From fields to be modified if the
-        # in queued status, which indicates that the user called POST /images
+        # image is in queued status, which indicates that the user called
-        # but did not supply either a Location field OR image data
+        # POST /images but originally supply neither a Location|Copy-From
-        if not orig_status == 'queued' and 'location' in image_meta:
+        # field NOR image data
+        location = self._external_source(image_meta, req)
+        if not orig_status == 'queued' and location:
             msg = _("Attempted to update Location field for an image "
                     "not in queued status.")
             raise HTTPBadRequest(msg, request=req, content_type="text/plain")

glance/client.py

@@ -130,7 +130,7 @@ class V1Client(base_client.BaseClient):
                 else:
                     raise
 
-    def add_image(self, image_meta=None, image_data=None):
+    def add_image(self, image_meta=None, image_data=None, features=None):
         """
         Tells Glance about an image's metadata as well
         as optionally the image_data itself
@@ -140,6 +140,7 @@ class V1Client(base_client.BaseClient):
         :param image_data: Optional string of raw image data
                            or file-like object that can be
                            used to read the image data
+        :param features: Optional map of features
 
         :retval The newly-stored image's metadata.
         """
@@ -155,6 +156,8 @@ class V1Client(base_client.BaseClient):
         else:
             body = None
 
+        utils.add_features_to_http_headers(features, headers)
+
         res = self.do_request("POST", "/images", body, headers)
         data = json.loads(res.read())
         return data['image']

glance/common/utils.py

@@ -89,6 +89,19 @@ def image_meta_to_http_headers(image_meta):
     return headers
 
 
+def add_features_to_http_headers(features, headers):
+    """
+    Adds additional headers representing glance features to be enabled.
+
+    :param headers: Base set of headers
+    :param features: Map of enabled features
+    """
+    if features:
+        for k, v in features.items():
+            if v is not None:
+                headers[k.lower()] = unicode(v)
+
+
 def get_image_meta_from_headers(response):
     """
     Processes HTTP headers from a supplied response that

glance/store/__init__.py

@@ -66,6 +66,70 @@ class UnsupportedBackend(BackendException):
     pass
 
 
+class Indexable(object):
+
+    """
+    Wrapper that allows an iterator or filelike be treated as an indexable
+    data structure. This is required in the case where the return value from
+    Store.get() is passed to Store.add() when adding a Copy-From image to a
+    Store where the client library relies on eventlet GreenSockets, in which
+    case the data to be written is indexed over.
+    """
+
+    def __init__(self, wrapped, size):
+        """
+        Initialize the object
+
+        :param wrappped: the wrapped iterator or filelike.
+        :param size: the size of data available
+        """
+        self.wrapped = wrapped
+        self.size = int(size) if size else (wrapped.len
+                                            if hasattr(wrapped, 'len') else 0)
+        self.cursor = 0
+        self.chunk = None
+
+    def __iter__(self):
+        """
+        Delegate iteration to the wrapped instance.
+        """
+        for self.chunk in self.wrapped:
+            yield self.chunk
+
+    def __getitem__(self, i):
+        """
+        Index into the next chunk (or previous chunk in the case where
+        the last data returned was not fully consumed).
+
+        :param i: a slice-to-the-end
+        """
+        start = i.start if isinstance(i, slice) else i
+        if start < self.cursor:
+            return self.chunk[(start - self.cursor):]
+
+        self.chunk = self.another()
+        if self.chunk:
+            self.cursor += len(self.chunk)
+
+        return self.chunk
+
+    def another(self):
+        """Implemented by subclasses to return the next element"""
+        raise NotImplementedError
+
+    def getvalue(self):
+        """
+        Return entire string value... used in testing
+        """
+        return self.wrapped.getvalue()
+
+    def __len__(self):
+        """
+        Length accessor.
+        """
+        return self.size
+
+
 def register_store(store_module, schemes):
     """
     Registers a store module and a set of schemes

glance/store/filesystem.py

@@ -27,6 +27,7 @@ import urlparse
 
 from glance.common import cfg
 from glance.common import exception
+from glance.common import utils
 import glance.store
 import glance.store.base
 import glance.store.location
@@ -198,10 +199,8 @@ class Store(glance.store.base.Store):
         bytes_written = 0
         try:
             with open(filepath, 'wb') as f:
-                while True:
+                for buf in utils.chunkreadable(image_file,
-                    buf = image_file.read(ChunkedFile.CHUNKSIZE)
+                                              ChunkedFile.CHUNKSIZE):
-                    if not buf:
-                        break
                     bytes_written += len(buf)
                     checksum.update(buf)
                     f.write(buf)

glance/store/http.py

@@ -117,7 +117,14 @@ class Store(glance.store.base.Store):
 
         iterator = http_response_iterator(conn, resp, self.CHUNKSIZE)
 
-        return (iterator, content_length)
+        class ResponseIndexable(glance.store.Indexable):
+            def another(self):
+                try:
+                    return self.wrapped.next()
+                except StopIteration:
+                    return ''
+
+        return (ResponseIndexable(iterator, content_length), content_length)
 
     def get_size(self, location):
         """

glance/store/s3.py

@@ -25,6 +25,7 @@ import urlparse
 
 from glance.common import cfg
 from glance.common import exception
+from glance.common import utils
 import glance.store
 import glance.store.base
 import glance.store.location
@@ -250,7 +251,13 @@ class Store(glance.store.base.Store):
         key = self._retrieve_key(location)
 
         key.BufferSize = self.CHUNKSIZE
-        return (ChunkedFile(key), key.size)
+
+        class ChunkedIndexable(glance.store.Indexable):
+            def another(self):
+                return (self.wrapped.fp.read(ChunkedFile.CHUNKSIZE)
+                        if self.wrapped.fp else None)
+
+        return (ChunkedIndexable(ChunkedFile(key), key.size), key.size)
 
     def get_size(self, location):
         """
@@ -361,11 +368,9 @@ class Store(glance.store.base.Store):
         tmpdir = self.s3_store_object_buffer_dir
         temp_file = tempfile.NamedTemporaryFile(dir=tmpdir)
         checksum = hashlib.md5()
-        chunk = image_file.read(self.CHUNKSIZE)
+        for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE):
-        while chunk:
             checksum.update(chunk)
             temp_file.write(chunk)
-            chunk = image_file.read(self.CHUNKSIZE)
         temp_file.flush()
 
         msg = _("Uploading temporary file to S3 for %s") % loc.get_uri()

glance/store/swift.py

@@ -273,7 +273,15 @@ class Store(glance.store.base.Store):
         #            "Expected %s byte file, Swift has %s bytes" %
         #            (expected_size, obj_size))
 
-        return (resp_body, resp_headers.get('content-length'))
+        class ResponseIndexable(glance.store.Indexable):
+            def another(self):
+                try:
+                    return self.wrapped.next()
+                except StopIteration:
+                    return ''
+
+        length = resp_headers.get('content-length')
+        return (ResponseIndexable(resp_body, length), length)
 
     def get_size(self, location):
         """

glance/tests/functional/store_utils.py

@@ -0,0 +1,273 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack, LLC
+# Copyright 2012 Red Hat, Inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Utility methods to set testcases up for Swift and/or S3 tests.
+"""
+
+import BaseHTTPServer
+import ConfigParser
+import httplib
+import os
+import thread
+
+
+FIVE_KB = 5 * 1024
+
+
+class RemoteImageHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_HEAD(self):
+        """
+        Respond to an image HEAD request fake metadata
+        """
+        if 'images' in self.path:
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/octet-stream')
+            self.send_header('Content-Length', FIVE_KB)
+            self.end_headers()
+            return
+        else:
+            self.send_error(404, 'File Not Found: %s' % self.path)
+            return
+
+    def do_GET(self):
+        """
+        Respond to an image GET request with fake image content.
+        """
+        if 'images' in self.path:
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/octet-stream')
+            self.send_header('Content-Length', FIVE_KB)
+            self.end_headers()
+            image_data = '*' * FIVE_KB
+            self.wfile.write(image_data)
+            self.wfile.close()
+            return
+        else:
+            self.send_error(404, 'File Not Found: %s' % self.path)
+            return
+
+
+def setup_http(test):
+    server_class = BaseHTTPServer.HTTPServer
+    remote_server = server_class(('127.0.0.1', 0), RemoteImageHandler)
+    remote_ip, remote_port = remote_server.server_address
+
+    def serve_requests(httpd):
+        httpd.serve_forever()
+
+    thread.start_new_thread(serve_requests, (remote_server,))
+    test.http_server = remote_server
+    test.http_ip = remote_ip
+    test.http_port = remote_port
+
+
+def teardown_http(test):
+    test.http_server.shutdown()
+
+
+def get_http_uri(test, image_id):
+    uri = 'http://%(http_ip)s:%(http_port)d/images/' % test.__dict__
+    uri += image_id
+    return uri
+
+
+def setup_swift(test):
+    # Test machines can set the GLANCE_TEST_SWIFT_CONF variable
+    # to override the location of the config file for migration testing
+    CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_SWIFT_CONF')
+
+    if not CONFIG_FILE_PATH:
+        test.disabled_message = "GLANCE_TEST_SWIFT_CONF environ not set."
+        print "GLANCE_TEST_SWIFT_CONF environ not set."
+        test.disabled = True
+        return
+
+    if os.path.exists(CONFIG_FILE_PATH):
+        cp = ConfigParser.RawConfigParser()
+        try:
+            cp.read(CONFIG_FILE_PATH)
+            defaults = cp.defaults()
+            for key, value in defaults.items():
+                test.__dict__[key] = value
+        except ConfigParser.ParsingError, e:
+            test.disabled_message = ("Failed to read test_swift.conf "
+                                     "file. Got error: %s" % e)
+            test.disabled = True
+            return
+
+    from swift.common import client as swift_client
+
+    try:
+        swift_host = test.swift_store_auth_address
+        if not swift_host.startswith('http'):
+            swift_host = 'https://' + swift_host
+        user = test.swift_store_user
+        key = test.swift_store_key
+        container_name = test.swift_store_container
+    except AttributeError, e:
+        test.disabled_message = ("Failed to find required configuration "
+                                 "options for Swift store. "
+                                 "Got error: %s" % e)
+        test.disabled = True
+        return
+
+    swift_conn = swift_client.Connection(
+        authurl=swift_host, user=user, key=key, snet=False, retries=1)
+
+    try:
+        _resp_headers, containers = swift_conn.get_account()
+    except Exception, e:
+        test.disabled_message = ("Failed to get_account from Swift "
+                                 "Got error: %s" % e)
+        test.disabled = True
+        return
+
+    try:
+        for container in containers:
+            if container == container_name:
+                swift_conn.delete_container(container)
+    except swift_client.ClientException, e:
+        test.disabled_message = ("Failed to delete container from Swift "
+                                 "Got error: %s" % e)
+        test.disabled = True
+        return
+
+    test.swift_conn = swift_conn
+
+    try:
+        swift_conn.put_container(container_name)
+    except swift_client.ClientException, e:
+        test.disabled_message = ("Failed to create container. "
+                                 "Got error: %s" % e)
+        test.disabled = True
+        return
+
+
+def teardown_swift(test):
+    if not test.disabled:
+        from swift.common import client as swift_client
+        try:
+            test.swift_conn.delete_container(test.swift_store_container)
+        except swift_client.ClientException, e:
+            if e.http_status == httplib.CONFLICT:
+                pass
+            else:
+                raise
+        test.swift_conn.put_container(test.swift_store_container)
+
+
+def get_swift_uri(test, image_id):
+    uri = ('swift+http://%(swift_store_user)s:%(swift_store_key)s' %
+           test.__dict__)
+    uri += ('@%(swift_store_auth_address)s/%(swift_store_container)s/' %
+           test.__dict__)
+    uri += image_id
+    return uri.replace('@http://', '@')
+
+
+def setup_s3(test):
+    # Test machines can set the GLANCE_TEST_S3_CONF variable
+    # to override the location of the config file for S3 testing
+    CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_S3_CONF')
+
+    if not CONFIG_FILE_PATH:
+        test.disabled_message = "GLANCE_TEST_S3_CONF environ not set."
+        test.disabled = True
+        return
+
+    if os.path.exists(CONFIG_FILE_PATH):
+        cp = ConfigParser.RawConfigParser()
+        try:
+            cp.read(CONFIG_FILE_PATH)
+            defaults = cp.defaults()
+            for key, value in defaults.items():
+                test.__dict__[key] = value
+        except ConfigParser.ParsingError, e:
+            test.disabled_message = ("Failed to read test_s3.conf config "
+                                     "file. Got error: %s" % e)
+            test.disabled = True
+            return
+
+    from boto.s3.connection import S3Connection
+    from boto.exception import S3ResponseError
+
+    try:
+        s3_host = test.s3_store_host
+        access_key = test.s3_store_access_key
+        secret_key = test.s3_store_secret_key
+        bucket_name = test.s3_store_bucket
+    except AttributeError, e:
+        test.disabled_message = ("Failed to find required configuration "
+                                 "options for S3 store. Got error: %s" % e)
+        test.disabled = True
+        return
+
+    s3_conn = S3Connection(access_key, secret_key, host=s3_host)
+
+    test.bucket = None
+    try:
+        buckets = s3_conn.get_all_buckets()
+        for bucket in buckets:
+            if bucket.name == bucket_name:
+                test.bucket = bucket
+    except S3ResponseError, e:
+        test.disabled_message = ("Failed to connect to S3 with "
+                                 "credentials, to find bucket. "
+                                 "Got error: %s" % e)
+        test.disabled = True
+        return
+    except TypeError, e:
+        # This hack is necessary because of a bug in boto 1.9b:
+        # http://code.google.com/p/boto/issues/detail?id=540
+        test.disabled_message = ("Failed to connect to S3 with "
+                                 "credentials. Got error: %s" % e)
+        test.disabled = True
+        return
+
+    test.s3_conn = s3_conn
+
+    if not test.bucket:
+        try:
+            test.bucket = s3_conn.create_bucket(bucket_name)
+        except boto.exception.S3ResponseError, e:
+            test.disabled_message = ("Failed to create bucket. "
+                                     "Got error: %s" % e)
+            test.disabled = True
+            return
+    else:
+        for key in test.bucket.list():
+            key.delete()
+
+
+def teardown_s3(test):
+    if not test.disabled:
+        # It's not possible to simply clear a bucket. You
+        # need to loop over all the keys and delete them
+        # all first...
+        for key in test.bucket.list():
+            key.delete()
+
+
+def get_s3_uri(test, image_id):
+    uri = ('s3://%(s3_store_access_key)s:%(s3_store_secret_key)s' %
+           test.__dict__)
+    uri += '@%(s3_conn)s/' % test.__dict__
+    uri += '%(s3_store_bucket)s/' % test.__dict__
+    uri += image_id
+    return uri.replace('S3Connection:', '')

glance/tests/functional/test_api.py

@@ -1307,7 +1307,7 @@ class TestApi(functional.FunctionalTest):
         to fail to start.
         """
         self.cleanup()
-        self.api_server.default_store = 'shouldnotexist'
+        self.default_store = 'shouldnotexist'
 
         # ensure failure exit code is available to assert on
         self.api_server.server_control_options += ' --await-child=1'

glance/tests/functional/test_bin_glance.py

@@ -23,7 +23,10 @@ import tempfile
 
 from glance.common import utils
 from glance.tests import functional
-from glance.tests.utils import execute, minimal_add_command
+from glance.tests.utils import execute, requires, minimal_add_command
+from glance.tests.functional.store_utils import (setup_http,
+                                                 teardown_http,
+                                                 get_http_uri)
 
 
 class TestBinGlance(functional.FunctionalTest):
@@ -80,6 +83,48 @@ class TestBinGlance(functional.FunctionalTest):
         self.assertEqual('0', size, "Expected image to be 0 bytes in size, "
                                     "but got %s. " % size)
 
+    @requires(setup_http, teardown_http)
+    def test_add_copying_from(self):
+        self.cleanup()
+        self.start_servers(**self.__dict__.copy())
+
+        api_port = self.api_port
+        registry_port = self.registry_port
+
+        # 0. Verify no public images
+        cmd = "bin/glance --port=%d index" % api_port
+
+        exitcode, out, err = execute(cmd)
+
+        self.assertEqual(0, exitcode)
+        self.assertEqual('', out.strip())
+
+        # 1. Add public image
+        suffix = 'copy_from=%s' % get_http_uri(self, 'foobar')
+        cmd = minimal_add_command(api_port, 'MyImage', suffix)
+        exitcode, out, err = execute(cmd)
+
+        self.assertEqual(0, exitcode)
+        self.assertTrue(out.strip().startswith('Added new image with ID:'))
+
+        # 2. Verify image added as public image
+        cmd = "bin/glance --port=%d index" % api_port
+
+        exitcode, out, err = execute(cmd)
+
+        self.assertEqual(0, exitcode)
+        lines = out.split("\n")[2:-1]
+        self.assertEqual(1, len(lines))
+
+        line = lines[0]
+
+        image_id, name, disk_format, container_format, size = \
+            [c.strip() for c in line.split()]
+        self.assertEqual('MyImage', name)
+
+        self.assertEqual('5120', size, "Expected image to be 0 bytes in size,"
+                                       " but got %s. " % size)
+
     def test_add_with_location_and_stdin(self):
         self.cleanup()
         self.start_servers(**self.__dict__.copy())

glance/tests/functional/test_cache_middleware.py

@@ -29,53 +29,22 @@ import shutil
 import thread
 import time
 
-import BaseHTTPServer
 import httplib2
 
 from glance.tests import functional
 from glance.tests.utils import (skip_if_disabled,
+                                requires,
                                 execute,
                                 xattr_writes_supported,
-                                minimal_headers,
+                                minimal_headers)
-                               )
 
+from glance.tests.functional.store_utils import (setup_http,
+                                                 teardown_http,
+                                                 get_http_uri)
 
 FIVE_KB = 5 * 1024
 
 
-class RemoteImageHandler(BaseHTTPServer.BaseHTTPRequestHandler):
-    def do_HEAD(self):
-        """
-        Respond to an image HEAD request fake metadata
-        """
-        if 'images' in self.path:
-            self.send_response(200)
-            self.send_header('Content-Type', 'application/octet-stream')
-            self.send_header('Content-Length', FIVE_KB)
-            self.end_headers()
-            return
-        else:
-            self.send_error(404, 'File Not Found: %s' % self.path)
-            return
-
-    def do_GET(self):
-        """
-        Respond to an image GET request with fake image content.
-        """
-        if 'images' in self.path:
-            self.send_response(200)
-            self.send_header('Content-Type', 'application/octet-stream')
-            self.send_header('Content-Length', FIVE_KB)
-            self.end_headers()
-            image_data = '*' * FIVE_KB
-            self.wfile.write(image_data)
-            self.wfile.close()
-            return
-        else:
-            self.send_error(404, 'File Not Found: %s' % self.path)
-            return
-
-
 class BaseCacheMiddlewareTest(object):
 
     @skip_if_disabled
@@ -153,6 +122,7 @@ class BaseCacheMiddlewareTest(object):
 
         self.stop_servers()
 
+    @requires(setup_http, teardown_http)
     @skip_if_disabled
     def test_cache_remote_image(self):
         """
@@ -164,18 +134,8 @@ class BaseCacheMiddlewareTest(object):
         api_port = self.api_port
         registry_port = self.registry_port
 
-        # set up "remote" image server
-        server_class = BaseHTTPServer.HTTPServer
-        remote_server = server_class(('127.0.0.1', 0), RemoteImageHandler)
-        remote_ip, remote_port = remote_server.server_address
-
-        def serve_requests(httpd):
-            httpd.serve_forever()
-
-        thread.start_new_thread(serve_requests, (remote_server,))
-
         # Add a remote image and verify a 201 Created is returned
-        remote_uri = 'http://%s:%d/images/2' % (remote_ip, remote_port)
+        remote_uri = get_http_uri(self, '2')
         headers = {'X-Image-Meta-Name': 'Image2',
                    'X-Image-Meta-disk_format': 'raw',
                    'X-Image-Meta-container_format': 'ovf',
@@ -204,8 +164,6 @@ class BaseCacheMiddlewareTest(object):
         self.assertEqual(response.status, 200)
         self.assertEqual(int(response['content-length']), FIVE_KB)
 
-        remote_server.shutdown()
-
         self.stop_servers()
 
 

glance/tests/functional/test_copy_to_file.py

@@ -0,0 +1,225 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack, LLC
+# Copyright 2012 Red Hat, Inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Tests copying images to a Glance API server which uses a filesystem-
+based storage backend.
+
+The from_swift testcase requires that a real Swift account is available.
+It looks in a file GLANCE_TEST_SWIFT_CONF environ variable for the
+credentials to use.
+
+Note that this test clears the entire container from the Swift account
+for use by the test case, so make sure you supply credentials for
+test accounts only.
+
+The from_s3 testcase requires that a real S3 account is available.
+It looks in a file specified in the GLANCE_TEST_S3_CONF environ variable
+for the credentials to use.
+
+Note that this test clears the entire bucket from the S3 account
+for use by the test case, so make sure you supply credentials for
+test accounts only.
+
+In either case, if a connection to the external store cannot be
+established, then the relevant test case is skipped.
+"""
+
+import hashlib
+import httplib2
+import json
+
+from glance.tests import functional
+from glance.tests.utils import skip_if_disabled, requires
+from glance.tests.functional.store_utils import (setup_swift,
+                                                 teardown_swift,
+                                                 get_swift_uri,
+                                                 setup_s3,
+                                                 teardown_s3,
+                                                 get_s3_uri,
+                                                 setup_http,
+                                                 teardown_http,
+                                                 get_http_uri)
+
+FIVE_KB = 5 * 1024
+
+
+class TestCopyToFile(functional.FunctionalTest):
+
+    """
+    Functional tests for copying images from the Swift, S3 & HTTP storage
+    backends to file
+    """
+
+    def _do_test_copy_from(self, from_store, get_uri):
+        """
+        Ensure we can copy from an external image in from_store.
+        """
+        self.cleanup()
+
+        self.start_servers(**self.__dict__.copy())
+
+        api_port = self.api_port
+        registry_port = self.registry_port
+
+        # POST /images with public image to be stored in from_store,
+        # to stand in for the 'external' image
+        image_data = "*" * FIVE_KB
+        headers = {'Content-Type': 'application/octet-stream',
+                   'X-Image-Meta-Name': 'external',
+                   'X-Image-Meta-Store': from_store,
+                   'X-Image-Meta-disk_format': 'raw',
+                   'X-Image-Meta-container_format': 'ovf',
+                   'X-Image-Meta-Is-Public': 'True'}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers,
+                                         body=image_data)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        original_image_id = data['image']['id']
+
+        copy_from = get_uri(self, original_image_id)
+
+        # POST /images with public image copied from_store (to Swift)
+        headers = {'X-Image-Meta-Name': 'copied',
+                   'X-Image-Meta-disk_format': 'raw',
+                   'X-Image-Meta-container_format': 'ovf',
+                   'X-Image-Meta-Is-Public': 'True',
+                   'X-Glance-API-Copy-From': copy_from}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        copy_image_id = data['image']['id']
+
+        # GET image and make sure image content is as expected
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE original image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              original_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        # GET image again to make sure the existence of the original
+        # image in from_store is not depended on
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE copied image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        self.stop_servers()
+
+    @requires(setup_swift, teardown_swift)
+    @skip_if_disabled
+    def test_copy_from_swift(self):
+        """
+        Ensure we can copy from an external image in Swift.
+        """
+        self._do_test_copy_from('swift', get_swift_uri)
+
+    @requires(setup_s3, teardown_s3)
+    @skip_if_disabled
+    def test_copy_from_s3(self):
+        """
+        Ensure we can copy from an external image in S3.
+        """
+        self._do_test_copy_from('s3', get_s3_uri)
+
+    @requires(setup_http, teardown_http)
+    @skip_if_disabled
+    def test_copy_from_http(self):
+        """
+        Ensure we can copy from an external image in HTTP.
+        """
+        self.cleanup()
+
+        self.start_servers(**self.__dict__.copy())
+
+        api_port = self.api_port
+        registry_port = self.registry_port
+
+        copy_from = get_http_uri(self, 'foobar')
+
+        # POST /images with public image copied HTTP (to Swift)
+        headers = {'X-Image-Meta-Name': 'copied',
+                   'X-Image-Meta-disk_format': 'raw',
+                   'X-Image-Meta-container_format': 'ovf',
+                   'X-Image-Meta-Is-Public': 'True',
+                   'X-Glance-API-Copy-From': copy_from}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        copy_image_id = data['image']['id']
+
+        # GET image and make sure image content is as expected
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE copied image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        self.stop_servers()

glance/tests/functional/test_s3.py

@@ -30,10 +30,8 @@ If a connection cannot be established, all the test cases are
 skipped.
 """
 
-import ConfigParser
 import hashlib
 import json
-import os
 import tempfile
 import unittest
 
@@ -42,7 +40,19 @@ import httplib2
 from glance.common import crypt
 from glance.common import utils
 from glance.tests.functional import test_api
-from glance.tests.utils import execute, skip_if_disabled
+from glance.tests.utils import (execute,
+                                skip_if_disabled,
+                                requires,
+                                minimal_headers)
+from glance.tests.functional.store_utils import (setup_s3,
+                                                 teardown_s3,
+                                                 get_s3_uri,
+                                                 setup_swift,
+                                                 teardown_swift,
+                                                 get_swift_uri,
+                                                 setup_http,
+                                                 teardown_http,
+                                                 get_http_uri)
 
 
 FIVE_KB = 5 * 1024
@@ -52,108 +62,25 @@ class TestS3(test_api.TestApi):
 
     """Functional tests for the S3 backend"""
 
-    # Test machines can set the GLANCE_TEST_S3_CONF variable
-    # to override the location of the config file for S3 testing
-    CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_S3_CONF')
-
     def setUp(self):
         """
         Test a connection to an S3 store using the credentials
         found in the environs or /tests/functional/test_s3.conf, if found.
         If the connection fails, mark all tests to skip.
         """
-        self.inited = False
+        if self.disabled:
-        self.disabled = True
-
-        if self.inited:
             return
 
-        if not self.CONFIG_FILE_PATH:
+        setup_s3(self)
-            self.disabled_message = "GLANCE_TEST_S3_CONF environ not set."
-            self.inited = True
-            return
 
-        if os.path.exists(TestS3.CONFIG_FILE_PATH):
-            cp = ConfigParser.RawConfigParser()
-            try:
-                cp.read(TestS3.CONFIG_FILE_PATH)
-                defaults = cp.defaults()
-                for key, value in defaults.items():
-                    self.__dict__[key] = value
-            except ConfigParser.ParsingError, e:
-                self.disabled_message = ("Failed to read test_s3.conf config "
-                                         "file. Got error: %s" % e)
-                self.inited = True
-                return
-
-        from boto.s3.connection import S3Connection
-        from boto.exception import S3ResponseError
-
-        try:
-            s3_host = self.s3_store_host
-            access_key = self.s3_store_access_key
-            secret_key = self.s3_store_secret_key
-            bucket_name = self.s3_store_bucket
-        except AttributeError, e:
-            self.disabled_message = ("Failed to find required configuration "
-                                     "options for S3 store. Got error: %s" % e)
-            self.inited = True
-            return
-
-        s3_conn = S3Connection(access_key, secret_key, host=s3_host)
-
-        self.bucket = None
-        try:
-            buckets = s3_conn.get_all_buckets()
-            for bucket in buckets:
-                if bucket.name == bucket_name:
-                    self.bucket = bucket
-        except S3ResponseError, e:
-            self.disabled_message = ("Failed to connect to S3 with "
-                                     "credentials, to find bucket. "
-                                     "Got error: %s" % e)
-            self.inited = True
-            return
-        except TypeError, e:
-            # This hack is necessary because of a bug in boto 1.9b:
-            # http://code.google.com/p/boto/issues/detail?id=540
-            self.disabled_message = ("Failed to connect to S3 with "
-                                     "credentials. Got error: %s" % e)
-            self.inited = True
-            return
-
-        self.s3_conn = s3_conn
-
-        if not self.bucket:
-            try:
-                self.bucket = s3_conn.create_bucket(bucket_name)
-            except boto.exception.S3ResponseError, e:
-                self.disabled_message = ("Failed to create bucket. "
-                                         "Got error: %s" % e)
-                self.inited = True
-                return
-        else:
-            self.clear_bucket()
-
-        self.disabled = False
-        self.inited = True
         self.default_store = 's3'
 
         super(TestS3, self).setUp()
 
     def tearDown(self):
-        if not self.disabled:
+        teardown_s3(self)
-            self.clear_bucket()
         super(TestS3, self).tearDown()
 
-    def clear_bucket(self):
-        # It's not possible to simply clear a bucket. You
-        # need to loop over all the keys and delete them
-        # all first...
-        keys = self.bucket.list()
-        for key in keys:
-            key.delete()
-
     @skip_if_disabled
     def test_remote_image(self):
         """Verify an image added using a 'Location' header can be retrieved"""
@@ -162,9 +89,7 @@ class TestS3(test_api.TestApi):
 
         # 1. POST /images with public image named Image1
         image_data = "*" * FIVE_KB
-        headers = {'Content-Type': 'application/octet-stream',
+        headers = minimal_headers('Image1')
-                   'X-Image-Meta-Name': 'Image1',
-                   'X-Image-Meta-Is-Public': 'True'}
         path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
         http = httplib2.Http()
         response, content = http.request(path, 'POST', headers=headers,
@@ -204,11 +129,9 @@ class TestS3(test_api.TestApi):
         # 4. POST /images using location generated by Image1
         image_id2 = utils.generate_uuid()
         image_data = "*" * FIVE_KB
-        headers = {'Content-Type': 'application/octet-stream',
+        headers = minimal_headers('Image2')
-                   'X-Image-Meta-Id': image_id2,
+        headers['X-Image-Meta-Id'] = image_id2
-                   'X-Image-Meta-Name': 'Image2',
+        headers['X-Image-Meta-Location'] = s3_store_location
-                   'X-Image-Meta-Is-Public': 'True',
-                   'X-Image-Meta-Location': s3_store_location}
         path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
         http = httplib2.Http()
         response, content = http.request(path, 'POST', headers=headers)
@@ -245,3 +168,156 @@ class TestS3(test_api.TestApi):
         http.request(path % args, 'DELETE')
 
         self.stop_servers()
+
+    def _do_test_copy_from(self, from_store, get_uri):
+        """
+        Ensure we can copy from an external image in from_store.
+        """
+        self.cleanup()
+
+        self.start_servers(**self.__dict__.copy())
+
+        api_port = self.api_port
+        registry_port = self.registry_port
+
+        # POST /images with public image to be stored in from_store,
+        # to stand in for the 'external' image
+        image_data = "*" * FIVE_KB
+        headers = minimal_headers('external')
+        headers['X-Image-Meta-Store'] = from_store
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers,
+                                         body=image_data)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        original_image_id = data['image']['id']
+
+        copy_from = get_uri(self, original_image_id)
+
+        # POST /images with public image copied from_store (to Swift)
+        headers = {'X-Image-Meta-Name': 'copied',
+                   'X-Image-Meta-Is-Public': 'True',
+                   'X-Image-Meta-disk_format': 'raw',
+                   'X-Image-Meta-container_format': 'ovf',
+                   'X-Glance-API-Copy-From': copy_from}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        copy_image_id = data['image']['id']
+
+        # GET image and make sure image content is as expected
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE original image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              original_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        # GET image again to make sure the existence of the original
+        # image in from_store is not depended on
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE copied image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        self.stop_servers()
+
+    @skip_if_disabled
+    def test_copy_from_s3(self):
+        """
+        Ensure we can copy from an external image in S3.
+        """
+        self._do_test_copy_from('s3', get_s3_uri)
+
+    @requires(setup_swift, teardown_swift)
+    @skip_if_disabled
+    def test_copy_from_swift(self):
+        """
+        Ensure we can copy from an external image in Swift.
+        """
+        self._do_test_copy_from('swift', get_swift_uri)
+
+    @requires(setup_http, teardown_http)
+    @skip_if_disabled
+    def test_copy_from_http(self):
+        """
+        Ensure we can copy from an external image in HTTP.
+        """
+        self.cleanup()
+
+        self.start_servers(**self.__dict__.copy())
+
+        api_port = self.api_port
+        registry_port = self.registry_port
+
+        copy_from = get_http_uri(self, 'foobar')
+
+        # POST /images with public image copied HTTP (to S3)
+        headers = {'X-Image-Meta-Name': 'copied',
+                   'X-Image-Meta-disk_format': 'raw',
+                   'X-Image-Meta-container_format': 'ovf',
+                   'X-Image-Meta-Is-Public': 'True',
+                   'X-Glance-API-Copy-From': copy_from}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        copy_image_id = data['image']['id']
+
+        # GET image and make sure image content is as expected
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE copied image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        self.stop_servers()

glance/tests/functional/test_swift.py

@@ -30,18 +30,25 @@ If a connection cannot be established, all the test cases are
 skipped.
 """
 
-import ConfigParser
 import hashlib
 import httplib
 import httplib2
 import json
-import os
 
 from glance.common import crypt
 import glance.store.swift  # Needed to register driver for location
 from glance.store.location import get_location_from_uri
 from glance.tests.functional import test_api
-from glance.tests.utils import skip_if_disabled
+from glance.tests.utils import skip_if_disabled, requires, minimal_headers
+from glance.tests.functional.store_utils import (setup_swift,
+                                                 teardown_swift,
+                                                 get_swift_uri,
+                                                 setup_s3,
+                                                 teardown_s3,
+                                                 get_s3_uri,
+                                                 setup_http,
+                                                 teardown_http,
+                                                 get_http_uri)
 
 FIVE_KB = 5 * 1024
 FIVE_MB = 5 * 1024 * 1024
@@ -51,89 +58,17 @@ class TestSwift(test_api.TestApi):
 
     """Functional tests for the Swift backend"""
 
-    # Test machines can set the GLANCE_TEST_SWIFT_CONF variable
-    # to override the location of the config file for migration testing
-    CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_SWIFT_CONF')
-
     def setUp(self):
         """
         Test a connection to an Swift store using the credentials
         found in the environs or /tests/functional/test_swift.conf, if found.
         If the connection fails, mark all tests to skip.
         """
-        self.inited = False
+        if self.disabled:
-        self.disabled = True
-
-        if self.inited:
             return
 
-        if not self.CONFIG_FILE_PATH:
+        setup_swift(self)
-            self.disabled_message = "GLANCE_TEST_SWIFT_CONF environ not set."
-            self.inited = True
-            return
 
-        if os.path.exists(TestSwift.CONFIG_FILE_PATH):
-            cp = ConfigParser.RawConfigParser()
-            try:
-                cp.read(TestSwift.CONFIG_FILE_PATH)
-                defaults = cp.defaults()
-                for key, value in defaults.items():
-                    self.__dict__[key] = value
-            except ConfigParser.ParsingError, e:
-                self.disabled_message = ("Failed to read test_swift.conf "
-                                         "file. Got error: %s" % e)
-                self.inited = True
-                return
-
-        from swift.common import client as swift_client
-
-        try:
-            swift_host = self.swift_store_auth_address
-            if not swift_host.startswith('http'):
-                swift_host = 'https://' + swift_host
-            user = self.swift_store_user
-            key = self.swift_store_key
-            container_name = self.swift_store_container
-        except AttributeError, e:
-            self.disabled_message = ("Failed to find required configuration "
-                                     "options for Swift store. "
-                                     "Got error: %s" % e)
-            self.inited = True
-            return
-
-        self.swift_conn = swift_conn = swift_client.Connection(
-            authurl=swift_host, user=user, key=key, snet=False, retries=1)
-
-        try:
-            _resp_headers, containers = swift_conn.get_account()
-        except Exception, e:
-            self.disabled_message = ("Failed to get_account from Swift "
-                                     "Got error: %s" % e)
-            self.inited = True
-            return
-
-        try:
-            for container in containers:
-                if container == container_name:
-                    swift_conn.delete_container(container)
-        except swift_client.ClientException, e:
-            self.disabled_message = ("Failed to delete container from Swift "
-                                     "Got error: %s" % e)
-            self.inited = True
-            return
-
-        self.swift_conn = swift_conn
-
-        try:
-            swift_conn.put_container(container_name)
-        except swift_client.ClientException, e:
-            self.disabled_message = ("Failed to create container. "
-                                     "Got error: %s" % e)
-            self.inited = True
-            return
-
-        self.disabled = False
-        self.inited = True
         self.default_store = 'swift'
 
         super(TestSwift, self).setUp()
@@ -185,9 +120,7 @@ class TestSwift(test_api.TestApi):
         # POST /images with public image named Image1
         # attribute and no custom properties. Verify a 200 OK is returned
         image_data = "*" * FIVE_MB
-        headers = {'Content-Type': 'application/octet-stream',
+        headers = minimal_headers('Image1')
-                   'X-Image-Meta-Name': 'Image1',
-                   'X-Image-Meta-Is-Public': 'True'}
         path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
         http = httplib2.Http()
         response, content = http.request(path, 'POST', headers=headers,
@@ -224,8 +157,8 @@ class TestSwift(test_api.TestApi):
             'x-image-meta-name': 'Image1',
             'x-image-meta-is_public': 'True',
             'x-image-meta-status': 'active',
-            'x-image-meta-disk_format': '',
+            'x-image-meta-disk_format': 'raw',
-            'x-image-meta-container_format': '',
+            'x-image-meta-container_format': 'ovf',
             'x-image-meta-size': str(FIVE_MB)
         }
 
@@ -335,9 +268,7 @@ class TestSwift(test_api.TestApi):
         # 1. POST /images with public image named Image1
         # attribute and no custom properties. Verify a 200 OK is returned
         image_data = "*" * FIVE_MB
-        headers = {'Content-Type': 'application/octet-stream',
+        headers = minimal_headers('Image1')
-                   'X-Image-Meta-Name': 'Image1',
-                   'X-Image-Meta-Is-Public': 'True'}
         path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
         http = httplib2.Http()
         response, content = http.request(path, 'POST', headers=headers,
@@ -374,8 +305,8 @@ class TestSwift(test_api.TestApi):
             'x-image-meta-name': 'Image1',
             'x-image-meta-is_public': 'True',
             'x-image-meta-status': 'active',
-            'x-image-meta-disk_format': '',
+            'x-image-meta-disk_format': 'raw',
-            'x-image-meta-container_format': '',
+            'x-image-meta-container_format': 'ovf',
             'x-image-meta-size': str(FIVE_MB)
         }
 
@@ -423,9 +354,7 @@ class TestSwift(test_api.TestApi):
 
         # POST /images with public image named Image1
         image_data = "*" * FIVE_KB
-        headers = {'Content-Type': 'application/octet-stream',
+        headers = minimal_headers('Image1')
-                   'X-Image-Meta-Name': 'Image1',
-                   'X-Image-Meta-Is-Public': 'True'}
         path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
         http = httplib2.Http()
         response, content = http.request(path, 'POST', headers=headers,
@@ -470,10 +399,8 @@ class TestSwift(test_api.TestApi):
 
         # POST /images with public image named Image1 without uploading data
         image_data = "*" * FIVE_KB
-        headers = {'Content-Type': 'application/octet-stream',
+        headers = minimal_headers('Image1')
-                   'X-Image-Meta-Name': 'Image1',
+        headers['X-Image-Meta-Location'] = swift_location
-                   'X-Image-Meta-Is-Public': 'True',
-                   'X-Image-Meta-Location': swift_location}
         path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
         http = httplib2.Http()
         response, content = http.request(path, 'POST', headers=headers)
@@ -512,3 +439,156 @@ class TestSwift(test_api.TestApi):
         self.assertEqual(response.status, 200)
 
         self.stop_servers()
+
+    def _do_test_copy_from(self, from_store, get_uri):
+        """
+        Ensure we can copy from an external image in from_store.
+        """
+        self.cleanup()
+
+        self.start_servers(**self.__dict__.copy())
+
+        api_port = self.api_port
+        registry_port = self.registry_port
+
+        # POST /images with public image to be stored in from_store,
+        # to stand in for the 'external' image
+        image_data = "*" * FIVE_KB
+        headers = minimal_headers('external')
+        headers['X-Image-Meta-Store'] = from_store
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers,
+                                         body=image_data)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        original_image_id = data['image']['id']
+
+        copy_from = get_uri(self, original_image_id)
+
+        # POST /images with public image copied from_store (to Swift)
+        headers = {'X-Image-Meta-Name': 'copied',
+                   'X-Image-Meta-disk_format': 'raw',
+                   'X-Image-Meta-container_format': 'ovf',
+                   'X-Image-Meta-Is-Public': 'True',
+                   'X-Glance-API-Copy-From': copy_from}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        copy_image_id = data['image']['id']
+
+        # GET image and make sure image content is as expected
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE original image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              original_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        # GET image again to make sure the existence of the original
+        # image in from_store is not depended on
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE copied image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        self.stop_servers()
+
+    @skip_if_disabled
+    def test_copy_from_swift(self):
+        """
+        Ensure we can copy from an external image in Swift.
+        """
+        self._do_test_copy_from('swift', get_swift_uri)
+
+    @requires(setup_s3, teardown_s3)
+    @skip_if_disabled
+    def test_copy_from_s3(self):
+        """
+        Ensure we can copy from an external image in S3.
+        """
+        self._do_test_copy_from('s3', get_s3_uri)
+
+    @requires(setup_http, teardown_http)
+    @skip_if_disabled
+    def test_copy_from_http(self):
+        """
+        Ensure we can copy from an external image in HTTP.
+        """
+        self.cleanup()
+
+        self.start_servers(**self.__dict__.copy())
+
+        api_port = self.api_port
+        registry_port = self.registry_port
+
+        copy_from = get_http_uri(self, 'foobar')
+
+        # POST /images with public image copied HTTP (to Swift)
+        headers = {'X-Image-Meta-Name': 'copied',
+                   'X-Image-Meta-disk_format': 'raw',
+                   'X-Image-Meta-container_format': 'ovf',
+                   'X-Image-Meta-Is-Public': 'True',
+                   'X-Glance-API-Copy-From': copy_from}
+        path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
+        http = httplib2.Http()
+        response, content = http.request(path, 'POST', headers=headers)
+        self.assertEqual(response.status, 201, content)
+        data = json.loads(content)
+
+        copy_image_id = data['image']['id']
+
+        # GET image and make sure image content is as expected
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response['content-length'], str(FIVE_KB))
+
+        self.assertEqual(content, "*" * FIVE_KB)
+        self.assertEqual(hashlib.md5(content).hexdigest(),
+                         hashlib.md5("*" * FIVE_KB).hexdigest())
+        self.assertEqual(data['image']['size'], FIVE_KB)
+        self.assertEqual(data['image']['name'], "copied")
+
+        # DELETE copied image
+        path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
+                                              copy_image_id)
+        http = httplib2.Http()
+        response, content = http.request(path, 'DELETE')
+        self.assertEqual(response.status, 200)
+
+        self.stop_servers()

glance/tests/unit/test_s3_store.py

@@ -273,7 +273,7 @@ class TestStore(unittest.TestCase):
             loc = get_location_from_uri(expected_location)
             (new_image_s3, new_image_size) = self.store.get(loc)
             new_image_contents = new_image_s3.getvalue()
-            new_image_s3_size = new_image_s3.len
+            new_image_s3_size = len(new_image_s3)
 
             self.assertEquals(expected_s3_contents, new_image_contents)
             self.assertEquals(expected_s3_size, new_image_s3_size)

glance/tests/unit/test_swift_store.py

@@ -261,7 +261,7 @@ class TestStore(unittest.TestCase):
         loc = get_location_from_uri(expected_location)
         (new_image_swift, new_image_size) = self.store.get(loc)
         new_image_contents = new_image_swift.getvalue()
-        new_image_swift_size = new_image_swift.len
+        new_image_swift_size = len(new_image_swift)
 
         self.assertEquals(expected_swift_contents, new_image_contents)
         self.assertEquals(expected_swift_size, new_image_swift_size)
@@ -318,7 +318,7 @@ class TestStore(unittest.TestCase):
             loc = get_location_from_uri(expected_location)
             (new_image_swift, new_image_size) = self.store.get(loc)
             new_image_contents = new_image_swift.getvalue()
-            new_image_swift_size = new_image_swift.len
+            new_image_swift_size = len(new_image_swift)
 
             self.assertEquals(expected_swift_contents, new_image_contents)
             self.assertEquals(expected_swift_size, new_image_swift_size)
@@ -382,7 +382,7 @@ class TestStore(unittest.TestCase):
         loc = get_location_from_uri(expected_location)
         (new_image_swift, new_image_size) = self.store.get(loc)
         new_image_contents = new_image_swift.getvalue()
-        new_image_swift_size = new_image_swift.len
+        new_image_swift_size = len(new_image_swift)
 
         self.assertEquals(expected_swift_contents, new_image_contents)
         self.assertEquals(expected_swift_size, new_image_swift_size)
@@ -430,7 +430,7 @@ class TestStore(unittest.TestCase):
         loc = get_location_from_uri(expected_location)
         (new_image_swift, new_image_size) = self.store.get(loc)
         new_image_contents = new_image_swift.getvalue()
-        new_image_swift_size = new_image_swift.len
+        new_image_swift_size = len(new_image_swift)
 
         self.assertEquals(expected_swift_contents, new_image_contents)
         self.assertEquals(expected_swift_size, new_image_swift_size)
@@ -491,7 +491,7 @@ class TestStore(unittest.TestCase):
         loc = get_location_from_uri(expected_location)
         (new_image_swift, new_image_size) = self.store.get(loc)
         new_image_contents = new_image_swift.getvalue()
-        new_image_swift_size = new_image_swift.len
+        new_image_swift_size = len(new_image_swift)
 
         self.assertEquals(expected_swift_contents, new_image_contents)
         self.assertEquals(expected_swift_size, new_image_swift_size)

glance/tests/utils.py

@@ -154,6 +154,23 @@ class skip_unless(object):
         return _skipper
 
 
+class requires(object):
+    """Decorator that initiates additional test setup/teardown."""
+    def __init__(self, setup, teardown=None):
+        self.setup = setup
+        self.teardown = teardown
+
+    def __call__(self, func):
+        def _runner(*args, **kw):
+            self.setup(args[0])
+            func(*args, **kw)
+            if self.teardown:
+                self.teardown(args[0])
+        _runner.__name__ = func.__name__
+        _runner.__doc__ = func.__doc__
+        return _runner
+
+
 def skip_if_disabled(func):
     """Decorator that skips a test if test case is disabled."""
     @functools.wraps(func)