diff --git a/bin/glance b/bin/glance index b8afbcd2e0..f70dc3c699 100755 --- a/bin/glance +++ b/bin/glance @@ -235,13 +235,21 @@ EXAMPLES print 'Found non-settable field %s. Removing.' % field fields.pop(field) - if 'location' in fields.keys(): - image_meta['location'] = fields.pop('location') + def _external_source(fields, image_data): + source = None + features = {} + if 'location' in fields.keys(): + source = fields.pop('location') + image_meta['location'] = source + elif 'copy_from' in fields.keys(): + source = fields.pop('copy_from') + features['x-glance-api-copy-from'] = source + return source, features # We need either a location or image data/stream to add... - image_location = image_meta.get('location') + location, features = _external_source(fields, image_meta) image_data = None - if not image_location: + if not location: # Grab the image data stream from stdin or redirect, # otherwise error out image_data = sys.stdin @@ -251,7 +259,8 @@ EXAMPLES if not options.dry_run: try: - image_meta = c.add_image(image_meta, image_data) + image_meta = c.add_image(image_meta, image_data, + features=features) image_id = image_meta['id'] print "Added new image with ID: %s" % image_id if options.verbose: @@ -278,9 +287,17 @@ EXAMPLES return FAILURE else: print "Dry run. We would have done the following:" + + def _dump(dict): + for k, v in sorted(dict.items()): + print " %(k)30s => %(v)s" % locals() + print "Add new image with metadata:" - for k, v in sorted(image_meta.items()): - print " %(k)30s => %(v)s" % locals() + _dump(image_meta) + + if features: + print "with features enabled:" + _dump(features) return SUCCESS @@ -299,7 +316,8 @@ to Glance that represents the metadata for an image. Field names that can be specified: name A name for the image. -location The location of the image. +location An external location to serve out from. +copy_from An external location (HTTP, S3 or Swift URI) to copy from. is_public If specified, interpreted as a boolean value and sets or unsets the image's availability to the public. protected If specified, interpreted as a boolean value diff --git a/doc/source/glance.rst b/doc/source/glance.rst index 2b22b012a3..cba64e5762 100644 --- a/doc/source/glance.rst +++ b/doc/source/glance.rst @@ -260,13 +260,20 @@ To upload an EC2 tarball VM image with an associated property (e.g., distro):: container_format=ovf disk_format=raw \ distro="ubuntu 10.10" < /root/maverick-server-uec-amd64.tar.gz -To upload an EC2 tarball VM image from a URL:: +To reference an EC2 tarball VM image available at an external URL:: - $> glance add name="uubntu-10.04-amd64" is_public=true \ + $> glance add name="ubuntu-10.04-amd64" is_public=true \ container_format=ovf disk_format=raw \ location="http://uec-images.ubuntu.com/lucid/current/\ lucid-server-uec-amd64.tar.gz" +To upload a copy of that same EC2 tarball VM image:: + + $> glance add name="ubuntu-10.04-amd64" is_public=true \ + container_format=ovf disk_format=raw \ + copy_from="http://uec-images.ubuntu.com/lucid/current/\ + lucid-server-uec-amd64.tar.gz" + To upload a qcow2 image:: $> glance add name="ubuntu-11.04-amd64" is_public=true \ diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index c291b417ca..6147febd3e 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -226,6 +226,23 @@ class Controller(controller.BaseController): 'image_meta': image_meta } + @staticmethod + def _copy_from(req): + return req.headers.get('x-glance-api-copy-from') + + @staticmethod + def _external_source(image_meta, req): + return image_meta.get('location', Controller._copy_from(req)) + + @staticmethod + def _get_from_store(where): + try: + image_data, image_size = get_from_backend(where) + except exception.NotFound, e: + raise HTTPNotFound(explanation="%s" % e) + image_size = int(image_size) if image_size else None + return image_data, image_size + def show(self, req, id): """ Returns an iterator that can be used to retrieve an image's @@ -239,16 +256,9 @@ class Controller(controller.BaseController): self._enforce(req, 'get_image') image_meta = self.get_active_image_meta_or_404(req, id) - def get_from_store(image_meta): - try: - location = image_meta['location'] - image_data, image_size = get_from_backend(location) - image_meta["size"] = image_size or image_meta["size"] - except exception.NotFound, e: - raise HTTPNotFound(explanation="%s" % e) - return image_data + image_iterator, size = self._get_from_store(image_meta['location']) + image_meta['size'] = size or image_meta['size'] - image_iterator = get_from_store(image_meta) del image_meta['location'] return { 'image_iterator': image_iterator, @@ -263,11 +273,12 @@ class Controller(controller.BaseController): :param req: The WSGI/Webob Request object :param id: The opaque image identifier + :param image_meta: The image metadata :raises HTTPConflict if image already exists :raises HTTPBadRequest if image metadata is not valid """ - location = image_meta.get('location') + location = self._external_source(image_meta, req) if location: store = get_store_from_location(location) # check the store exists before we hit the registry, but we @@ -278,9 +289,9 @@ class Controller(controller.BaseController): image_meta['size'] = image_meta.get('size', 0) \ or get_size_from_backend(location) else: - # Ensure that the size attribute is set to zero for uploadable - # images (if not provided). The size will be set to a non-zero - # value during upload + # Ensure that the size attribute is set to zero for directly + # uploadable images (if not provided). The size will be set + # to a non-zero value during upload image_meta['size'] = image_meta.get('size', 0) image_meta['status'] = 'queued' @@ -317,13 +328,30 @@ class Controller(controller.BaseController): :raises HTTPConflict if image already exists :retval The location where the image was stored """ - try: - req.get_content_type('application/octet-stream') - except exception.InvalidContentType: - self._safe_kill(req, image_meta['id']) - msg = _("Content-Type must be application/octet-stream") - logger.error(msg) - raise HTTPBadRequest(explanation=msg) + + copy_from = self._copy_from(req) + if copy_from: + image_data, image_size = self._get_from_store(copy_from) + image_meta['size'] = image_size or image_meta['size'] + else: + try: + req.get_content_type('application/octet-stream') + except exception.InvalidContentType: + self._safe_kill(req, image_meta['id']) + msg = _("Content-Type must be application/octet-stream") + logger.error(msg) + raise HTTPBadRequest(explanation=msg) + + image_data = req.body_file + + if req.content_length: + image_size = int(req.content_length) + elif 'x-image-meta-size' in req.headers: + image_size = int(req.headers['x-image-meta-size']) + else: + logger.debug(_("Got request with no content-length and no " + "x-image-meta-size header")) + image_size = 0 store_name = req.headers.get('x-image-meta-store', self.conf.default_store) @@ -337,14 +365,6 @@ class Controller(controller.BaseController): try: logger.debug(_("Uploading image data for image %(image_id)s " "to %(store_name)s store"), locals()) - if req.content_length: - image_size = int(req.content_length) - elif 'x-image-meta-size' in req.headers: - image_size = int(req.headers['x-image-meta-size']) - else: - logger.debug(_("Got request with no content-length and no " - "x-image-meta-size header")) - image_size = 0 if image_size > IMAGE_SIZE_CAP: max_image_size = IMAGE_SIZE_CAP @@ -355,7 +375,7 @@ class Controller(controller.BaseController): raise HTTPBadRequest(msg, request=request) location, size, checksum = store.add(image_meta['id'], - req.body_file, + image_data, image_size) # Verify any supplied checksum value matches checksum @@ -505,19 +525,27 @@ class Controller(controller.BaseController): def create(self, req, image_meta, image_data): """ - Adds a new image to Glance. Three scenarios exist when creating an + Adds a new image to Glance. Four scenarios exist when creating an image: - 1. If the image data is available for upload, create can be passed the - image data as the request body and the metadata as the request - headers. The image will initially be 'queued', during upload it - will be in the 'saving' status, and then 'killed' or 'active' - depending on whether the upload completed successfully. + 1. If the image data is available directly for upload, create can be + passed the image data as the request body and the metadata as the + request headers. The image will initially be 'queued', during + upload it will be in the 'saving' status, and then 'killed' or + 'active' depending on whether the upload completed successfully. - 2. If the image data exists somewhere else, you can pass in the source - using the x-image-meta-location header + 2. If the image data exists somewhere else, you can upload indirectly + from the external source using the x-glance-api-copy-from header. + Once the image is uploaded, the external store is not subsequently + consulted, i.e. the image content is served out from the configured + glance image store. State transitions are as for option #1. - 3. If the image data is not available yet, but you'd like reserve a + 3. If the image data exists somewhere else, you can reference the + source using the x-image-meta-location header. The image content + will be served out from the external store, i.e. is never uploaded + to the configured glance image store. + + 4. If the image data is not available yet, but you'd like reserve a spot for it, you can omit the data and a record will be created in the 'queued' state. This exists primarily to maintain backwards compatibility with OpenStack/Rackspace API semantics. @@ -547,7 +575,7 @@ class Controller(controller.BaseController): image_meta = self._reserve(req, image_meta) image_id = image_meta['id'] - if image_data is not None: + if image_data or self._copy_from(req): image_meta = self._upload_and_activate(req, image_meta) else: location = image_meta.get('location') @@ -594,10 +622,12 @@ class Controller(controller.BaseController): if image_data is not None and orig_status != 'queued': raise HTTPConflict(_("Cannot upload to an unqueued image")) - # Only allow the Location fields to be modified if the image is - # in queued status, which indicates that the user called POST /images - # but did not supply either a Location field OR image data - if not orig_status == 'queued' and 'location' in image_meta: + # Only allow the Location|Copy-From fields to be modified if the + # image is in queued status, which indicates that the user called + # POST /images but originally supply neither a Location|Copy-From + # field NOR image data + location = self._external_source(image_meta, req) + if not orig_status == 'queued' and location: msg = _("Attempted to update Location field for an image " "not in queued status.") raise HTTPBadRequest(msg, request=req, content_type="text/plain") diff --git a/glance/client.py b/glance/client.py index e42eb90449..c02d442da4 100644 --- a/glance/client.py +++ b/glance/client.py @@ -130,7 +130,7 @@ class V1Client(base_client.BaseClient): else: raise - def add_image(self, image_meta=None, image_data=None): + def add_image(self, image_meta=None, image_data=None, features=None): """ Tells Glance about an image's metadata as well as optionally the image_data itself @@ -140,6 +140,7 @@ class V1Client(base_client.BaseClient): :param image_data: Optional string of raw image data or file-like object that can be used to read the image data + :param features: Optional map of features :retval The newly-stored image's metadata. """ @@ -155,6 +156,8 @@ class V1Client(base_client.BaseClient): else: body = None + utils.add_features_to_http_headers(features, headers) + res = self.do_request("POST", "/images", body, headers) data = json.loads(res.read()) return data['image'] diff --git a/glance/common/utils.py b/glance/common/utils.py index 154d63aa68..0e4a01329f 100644 --- a/glance/common/utils.py +++ b/glance/common/utils.py @@ -89,6 +89,19 @@ def image_meta_to_http_headers(image_meta): return headers +def add_features_to_http_headers(features, headers): + """ + Adds additional headers representing glance features to be enabled. + + :param headers: Base set of headers + :param features: Map of enabled features + """ + if features: + for k, v in features.items(): + if v is not None: + headers[k.lower()] = unicode(v) + + def get_image_meta_from_headers(response): """ Processes HTTP headers from a supplied response that diff --git a/glance/store/__init__.py b/glance/store/__init__.py index 9ef577482d..4f2f334162 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -66,6 +66,70 @@ class UnsupportedBackend(BackendException): pass +class Indexable(object): + + """ + Wrapper that allows an iterator or filelike be treated as an indexable + data structure. This is required in the case where the return value from + Store.get() is passed to Store.add() when adding a Copy-From image to a + Store where the client library relies on eventlet GreenSockets, in which + case the data to be written is indexed over. + """ + + def __init__(self, wrapped, size): + """ + Initialize the object + + :param wrappped: the wrapped iterator or filelike. + :param size: the size of data available + """ + self.wrapped = wrapped + self.size = int(size) if size else (wrapped.len + if hasattr(wrapped, 'len') else 0) + self.cursor = 0 + self.chunk = None + + def __iter__(self): + """ + Delegate iteration to the wrapped instance. + """ + for self.chunk in self.wrapped: + yield self.chunk + + def __getitem__(self, i): + """ + Index into the next chunk (or previous chunk in the case where + the last data returned was not fully consumed). + + :param i: a slice-to-the-end + """ + start = i.start if isinstance(i, slice) else i + if start < self.cursor: + return self.chunk[(start - self.cursor):] + + self.chunk = self.another() + if self.chunk: + self.cursor += len(self.chunk) + + return self.chunk + + def another(self): + """Implemented by subclasses to return the next element""" + raise NotImplementedError + + def getvalue(self): + """ + Return entire string value... used in testing + """ + return self.wrapped.getvalue() + + def __len__(self): + """ + Length accessor. + """ + return self.size + + def register_store(store_module, schemes): """ Registers a store module and a set of schemes diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py index df80b51a0b..ecbd2041c6 100644 --- a/glance/store/filesystem.py +++ b/glance/store/filesystem.py @@ -27,6 +27,7 @@ import urlparse from glance.common import cfg from glance.common import exception +from glance.common import utils import glance.store import glance.store.base import glance.store.location @@ -198,10 +199,8 @@ class Store(glance.store.base.Store): bytes_written = 0 try: with open(filepath, 'wb') as f: - while True: - buf = image_file.read(ChunkedFile.CHUNKSIZE) - if not buf: - break + for buf in utils.chunkreadable(image_file, + ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) diff --git a/glance/store/http.py b/glance/store/http.py index 04697c9f1a..8c56d95bf8 100644 --- a/glance/store/http.py +++ b/glance/store/http.py @@ -117,7 +117,14 @@ class Store(glance.store.base.Store): iterator = http_response_iterator(conn, resp, self.CHUNKSIZE) - return (iterator, content_length) + class ResponseIndexable(glance.store.Indexable): + def another(self): + try: + return self.wrapped.next() + except StopIteration: + return '' + + return (ResponseIndexable(iterator, content_length), content_length) def get_size(self, location): """ diff --git a/glance/store/s3.py b/glance/store/s3.py index 4e88afbb41..7f10718112 100644 --- a/glance/store/s3.py +++ b/glance/store/s3.py @@ -25,6 +25,7 @@ import urlparse from glance.common import cfg from glance.common import exception +from glance.common import utils import glance.store import glance.store.base import glance.store.location @@ -250,7 +251,13 @@ class Store(glance.store.base.Store): key = self._retrieve_key(location) key.BufferSize = self.CHUNKSIZE - return (ChunkedFile(key), key.size) + + class ChunkedIndexable(glance.store.Indexable): + def another(self): + return (self.wrapped.fp.read(ChunkedFile.CHUNKSIZE) + if self.wrapped.fp else None) + + return (ChunkedIndexable(ChunkedFile(key), key.size), key.size) def get_size(self, location): """ @@ -361,11 +368,9 @@ class Store(glance.store.base.Store): tmpdir = self.s3_store_object_buffer_dir temp_file = tempfile.NamedTemporaryFile(dir=tmpdir) checksum = hashlib.md5() - chunk = image_file.read(self.CHUNKSIZE) - while chunk: + for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE): checksum.update(chunk) temp_file.write(chunk) - chunk = image_file.read(self.CHUNKSIZE) temp_file.flush() msg = _("Uploading temporary file to S3 for %s") % loc.get_uri() diff --git a/glance/store/swift.py b/glance/store/swift.py index d4ac71d267..616408cd2a 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -273,7 +273,15 @@ class Store(glance.store.base.Store): # "Expected %s byte file, Swift has %s bytes" % # (expected_size, obj_size)) - return (resp_body, resp_headers.get('content-length')) + class ResponseIndexable(glance.store.Indexable): + def another(self): + try: + return self.wrapped.next() + except StopIteration: + return '' + + length = resp_headers.get('content-length') + return (ResponseIndexable(resp_body, length), length) def get_size(self, location): """ diff --git a/glance/tests/functional/store_utils.py b/glance/tests/functional/store_utils.py new file mode 100644 index 0000000000..de2e4d0ab6 --- /dev/null +++ b/glance/tests/functional/store_utils.py @@ -0,0 +1,273 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# Copyright 2012 Red Hat, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utility methods to set testcases up for Swift and/or S3 tests. +""" + +import BaseHTTPServer +import ConfigParser +import httplib +import os +import thread + + +FIVE_KB = 5 * 1024 + + +class RemoteImageHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def do_HEAD(self): + """ + Respond to an image HEAD request fake metadata + """ + if 'images' in self.path: + self.send_response(200) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', FIVE_KB) + self.end_headers() + return + else: + self.send_error(404, 'File Not Found: %s' % self.path) + return + + def do_GET(self): + """ + Respond to an image GET request with fake image content. + """ + if 'images' in self.path: + self.send_response(200) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', FIVE_KB) + self.end_headers() + image_data = '*' * FIVE_KB + self.wfile.write(image_data) + self.wfile.close() + return + else: + self.send_error(404, 'File Not Found: %s' % self.path) + return + + +def setup_http(test): + server_class = BaseHTTPServer.HTTPServer + remote_server = server_class(('127.0.0.1', 0), RemoteImageHandler) + remote_ip, remote_port = remote_server.server_address + + def serve_requests(httpd): + httpd.serve_forever() + + thread.start_new_thread(serve_requests, (remote_server,)) + test.http_server = remote_server + test.http_ip = remote_ip + test.http_port = remote_port + + +def teardown_http(test): + test.http_server.shutdown() + + +def get_http_uri(test, image_id): + uri = 'http://%(http_ip)s:%(http_port)d/images/' % test.__dict__ + uri += image_id + return uri + + +def setup_swift(test): + # Test machines can set the GLANCE_TEST_SWIFT_CONF variable + # to override the location of the config file for migration testing + CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_SWIFT_CONF') + + if not CONFIG_FILE_PATH: + test.disabled_message = "GLANCE_TEST_SWIFT_CONF environ not set." + print "GLANCE_TEST_SWIFT_CONF environ not set." + test.disabled = True + return + + if os.path.exists(CONFIG_FILE_PATH): + cp = ConfigParser.RawConfigParser() + try: + cp.read(CONFIG_FILE_PATH) + defaults = cp.defaults() + for key, value in defaults.items(): + test.__dict__[key] = value + except ConfigParser.ParsingError, e: + test.disabled_message = ("Failed to read test_swift.conf " + "file. Got error: %s" % e) + test.disabled = True + return + + from swift.common import client as swift_client + + try: + swift_host = test.swift_store_auth_address + if not swift_host.startswith('http'): + swift_host = 'https://' + swift_host + user = test.swift_store_user + key = test.swift_store_key + container_name = test.swift_store_container + except AttributeError, e: + test.disabled_message = ("Failed to find required configuration " + "options for Swift store. " + "Got error: %s" % e) + test.disabled = True + return + + swift_conn = swift_client.Connection( + authurl=swift_host, user=user, key=key, snet=False, retries=1) + + try: + _resp_headers, containers = swift_conn.get_account() + except Exception, e: + test.disabled_message = ("Failed to get_account from Swift " + "Got error: %s" % e) + test.disabled = True + return + + try: + for container in containers: + if container == container_name: + swift_conn.delete_container(container) + except swift_client.ClientException, e: + test.disabled_message = ("Failed to delete container from Swift " + "Got error: %s" % e) + test.disabled = True + return + + test.swift_conn = swift_conn + + try: + swift_conn.put_container(container_name) + except swift_client.ClientException, e: + test.disabled_message = ("Failed to create container. " + "Got error: %s" % e) + test.disabled = True + return + + +def teardown_swift(test): + if not test.disabled: + from swift.common import client as swift_client + try: + test.swift_conn.delete_container(test.swift_store_container) + except swift_client.ClientException, e: + if e.http_status == httplib.CONFLICT: + pass + else: + raise + test.swift_conn.put_container(test.swift_store_container) + + +def get_swift_uri(test, image_id): + uri = ('swift+http://%(swift_store_user)s:%(swift_store_key)s' % + test.__dict__) + uri += ('@%(swift_store_auth_address)s/%(swift_store_container)s/' % + test.__dict__) + uri += image_id + return uri.replace('@http://', '@') + + +def setup_s3(test): + # Test machines can set the GLANCE_TEST_S3_CONF variable + # to override the location of the config file for S3 testing + CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_S3_CONF') + + if not CONFIG_FILE_PATH: + test.disabled_message = "GLANCE_TEST_S3_CONF environ not set." + test.disabled = True + return + + if os.path.exists(CONFIG_FILE_PATH): + cp = ConfigParser.RawConfigParser() + try: + cp.read(CONFIG_FILE_PATH) + defaults = cp.defaults() + for key, value in defaults.items(): + test.__dict__[key] = value + except ConfigParser.ParsingError, e: + test.disabled_message = ("Failed to read test_s3.conf config " + "file. Got error: %s" % e) + test.disabled = True + return + + from boto.s3.connection import S3Connection + from boto.exception import S3ResponseError + + try: + s3_host = test.s3_store_host + access_key = test.s3_store_access_key + secret_key = test.s3_store_secret_key + bucket_name = test.s3_store_bucket + except AttributeError, e: + test.disabled_message = ("Failed to find required configuration " + "options for S3 store. Got error: %s" % e) + test.disabled = True + return + + s3_conn = S3Connection(access_key, secret_key, host=s3_host) + + test.bucket = None + try: + buckets = s3_conn.get_all_buckets() + for bucket in buckets: + if bucket.name == bucket_name: + test.bucket = bucket + except S3ResponseError, e: + test.disabled_message = ("Failed to connect to S3 with " + "credentials, to find bucket. " + "Got error: %s" % e) + test.disabled = True + return + except TypeError, e: + # This hack is necessary because of a bug in boto 1.9b: + # http://code.google.com/p/boto/issues/detail?id=540 + test.disabled_message = ("Failed to connect to S3 with " + "credentials. Got error: %s" % e) + test.disabled = True + return + + test.s3_conn = s3_conn + + if not test.bucket: + try: + test.bucket = s3_conn.create_bucket(bucket_name) + except boto.exception.S3ResponseError, e: + test.disabled_message = ("Failed to create bucket. " + "Got error: %s" % e) + test.disabled = True + return + else: + for key in test.bucket.list(): + key.delete() + + +def teardown_s3(test): + if not test.disabled: + # It's not possible to simply clear a bucket. You + # need to loop over all the keys and delete them + # all first... + for key in test.bucket.list(): + key.delete() + + +def get_s3_uri(test, image_id): + uri = ('s3://%(s3_store_access_key)s:%(s3_store_secret_key)s' % + test.__dict__) + uri += '@%(s3_conn)s/' % test.__dict__ + uri += '%(s3_store_bucket)s/' % test.__dict__ + uri += image_id + return uri.replace('S3Connection:', '') diff --git a/glance/tests/functional/test_api.py b/glance/tests/functional/test_api.py index 93259fe1bb..8359324424 100644 --- a/glance/tests/functional/test_api.py +++ b/glance/tests/functional/test_api.py @@ -1307,7 +1307,7 @@ class TestApi(functional.FunctionalTest): to fail to start. """ self.cleanup() - self.api_server.default_store = 'shouldnotexist' + self.default_store = 'shouldnotexist' # ensure failure exit code is available to assert on self.api_server.server_control_options += ' --await-child=1' diff --git a/glance/tests/functional/test_bin_glance.py b/glance/tests/functional/test_bin_glance.py index ac36c3f786..b4c81e9efd 100644 --- a/glance/tests/functional/test_bin_glance.py +++ b/glance/tests/functional/test_bin_glance.py @@ -23,7 +23,10 @@ import tempfile from glance.common import utils from glance.tests import functional -from glance.tests.utils import execute, minimal_add_command +from glance.tests.utils import execute, requires, minimal_add_command +from glance.tests.functional.store_utils import (setup_http, + teardown_http, + get_http_uri) class TestBinGlance(functional.FunctionalTest): @@ -80,6 +83,48 @@ class TestBinGlance(functional.FunctionalTest): self.assertEqual('0', size, "Expected image to be 0 bytes in size, " "but got %s. " % size) + @requires(setup_http, teardown_http) + def test_add_copying_from(self): + self.cleanup() + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # 0. Verify no public images + cmd = "bin/glance --port=%d index" % api_port + + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + self.assertEqual('', out.strip()) + + # 1. Add public image + suffix = 'copy_from=%s' % get_http_uri(self, 'foobar') + cmd = minimal_add_command(api_port, 'MyImage', suffix) + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + self.assertTrue(out.strip().startswith('Added new image with ID:')) + + # 2. Verify image added as public image + cmd = "bin/glance --port=%d index" % api_port + + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + lines = out.split("\n")[2:-1] + self.assertEqual(1, len(lines)) + + line = lines[0] + + image_id, name, disk_format, container_format, size = \ + [c.strip() for c in line.split()] + self.assertEqual('MyImage', name) + + self.assertEqual('5120', size, "Expected image to be 0 bytes in size," + " but got %s. " % size) + def test_add_with_location_and_stdin(self): self.cleanup() self.start_servers(**self.__dict__.copy()) diff --git a/glance/tests/functional/test_cache_middleware.py b/glance/tests/functional/test_cache_middleware.py index c3c9ab8211..b9f54b69f8 100644 --- a/glance/tests/functional/test_cache_middleware.py +++ b/glance/tests/functional/test_cache_middleware.py @@ -29,53 +29,22 @@ import shutil import thread import time -import BaseHTTPServer import httplib2 from glance.tests import functional from glance.tests.utils import (skip_if_disabled, + requires, execute, xattr_writes_supported, - minimal_headers, - ) + minimal_headers) +from glance.tests.functional.store_utils import (setup_http, + teardown_http, + get_http_uri) FIVE_KB = 5 * 1024 -class RemoteImageHandler(BaseHTTPServer.BaseHTTPRequestHandler): - def do_HEAD(self): - """ - Respond to an image HEAD request fake metadata - """ - if 'images' in self.path: - self.send_response(200) - self.send_header('Content-Type', 'application/octet-stream') - self.send_header('Content-Length', FIVE_KB) - self.end_headers() - return - else: - self.send_error(404, 'File Not Found: %s' % self.path) - return - - def do_GET(self): - """ - Respond to an image GET request with fake image content. - """ - if 'images' in self.path: - self.send_response(200) - self.send_header('Content-Type', 'application/octet-stream') - self.send_header('Content-Length', FIVE_KB) - self.end_headers() - image_data = '*' * FIVE_KB - self.wfile.write(image_data) - self.wfile.close() - return - else: - self.send_error(404, 'File Not Found: %s' % self.path) - return - - class BaseCacheMiddlewareTest(object): @skip_if_disabled @@ -153,6 +122,7 @@ class BaseCacheMiddlewareTest(object): self.stop_servers() + @requires(setup_http, teardown_http) @skip_if_disabled def test_cache_remote_image(self): """ @@ -164,18 +134,8 @@ class BaseCacheMiddlewareTest(object): api_port = self.api_port registry_port = self.registry_port - # set up "remote" image server - server_class = BaseHTTPServer.HTTPServer - remote_server = server_class(('127.0.0.1', 0), RemoteImageHandler) - remote_ip, remote_port = remote_server.server_address - - def serve_requests(httpd): - httpd.serve_forever() - - thread.start_new_thread(serve_requests, (remote_server,)) - # Add a remote image and verify a 201 Created is returned - remote_uri = 'http://%s:%d/images/2' % (remote_ip, remote_port) + remote_uri = get_http_uri(self, '2') headers = {'X-Image-Meta-Name': 'Image2', 'X-Image-Meta-disk_format': 'raw', 'X-Image-Meta-container_format': 'ovf', @@ -204,8 +164,6 @@ class BaseCacheMiddlewareTest(object): self.assertEqual(response.status, 200) self.assertEqual(int(response['content-length']), FIVE_KB) - remote_server.shutdown() - self.stop_servers() diff --git a/glance/tests/functional/test_copy_to_file.py b/glance/tests/functional/test_copy_to_file.py new file mode 100644 index 0000000000..e376154bd8 --- /dev/null +++ b/glance/tests/functional/test_copy_to_file.py @@ -0,0 +1,225 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# Copyright 2012 Red Hat, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests copying images to a Glance API server which uses a filesystem- +based storage backend. + +The from_swift testcase requires that a real Swift account is available. +It looks in a file GLANCE_TEST_SWIFT_CONF environ variable for the +credentials to use. + +Note that this test clears the entire container from the Swift account +for use by the test case, so make sure you supply credentials for +test accounts only. + +The from_s3 testcase requires that a real S3 account is available. +It looks in a file specified in the GLANCE_TEST_S3_CONF environ variable +for the credentials to use. + +Note that this test clears the entire bucket from the S3 account +for use by the test case, so make sure you supply credentials for +test accounts only. + +In either case, if a connection to the external store cannot be +established, then the relevant test case is skipped. +""" + +import hashlib +import httplib2 +import json + +from glance.tests import functional +from glance.tests.utils import skip_if_disabled, requires +from glance.tests.functional.store_utils import (setup_swift, + teardown_swift, + get_swift_uri, + setup_s3, + teardown_s3, + get_s3_uri, + setup_http, + teardown_http, + get_http_uri) + +FIVE_KB = 5 * 1024 + + +class TestCopyToFile(functional.FunctionalTest): + + """ + Functional tests for copying images from the Swift, S3 & HTTP storage + backends to file + """ + + def _do_test_copy_from(self, from_store, get_uri): + """ + Ensure we can copy from an external image in from_store. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # POST /images with public image to be stored in from_store, + # to stand in for the 'external' image + image_data = "*" * FIVE_KB + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'external', + 'X-Image-Meta-Store': from_store, + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + original_image_id = data['image']['id'] + + copy_from = get_uri(self, original_image_id) + + # POST /images with public image copied from_store (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE original image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + original_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + # GET image again to make sure the existence of the original + # image in from_store is not depended on + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() + + @requires(setup_swift, teardown_swift) + @skip_if_disabled + def test_copy_from_swift(self): + """ + Ensure we can copy from an external image in Swift. + """ + self._do_test_copy_from('swift', get_swift_uri) + + @requires(setup_s3, teardown_s3) + @skip_if_disabled + def test_copy_from_s3(self): + """ + Ensure we can copy from an external image in S3. + """ + self._do_test_copy_from('s3', get_s3_uri) + + @requires(setup_http, teardown_http) + @skip_if_disabled + def test_copy_from_http(self): + """ + Ensure we can copy from an external image in HTTP. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + copy_from = get_http_uri(self, 'foobar') + + # POST /images with public image copied HTTP (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() diff --git a/glance/tests/functional/test_s3.py b/glance/tests/functional/test_s3.py index abc2526407..79fdf066b8 100644 --- a/glance/tests/functional/test_s3.py +++ b/glance/tests/functional/test_s3.py @@ -30,10 +30,8 @@ If a connection cannot be established, all the test cases are skipped. """ -import ConfigParser import hashlib import json -import os import tempfile import unittest @@ -42,7 +40,19 @@ import httplib2 from glance.common import crypt from glance.common import utils from glance.tests.functional import test_api -from glance.tests.utils import execute, skip_if_disabled +from glance.tests.utils import (execute, + skip_if_disabled, + requires, + minimal_headers) +from glance.tests.functional.store_utils import (setup_s3, + teardown_s3, + get_s3_uri, + setup_swift, + teardown_swift, + get_swift_uri, + setup_http, + teardown_http, + get_http_uri) FIVE_KB = 5 * 1024 @@ -52,108 +62,25 @@ class TestS3(test_api.TestApi): """Functional tests for the S3 backend""" - # Test machines can set the GLANCE_TEST_S3_CONF variable - # to override the location of the config file for S3 testing - CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_S3_CONF') - def setUp(self): """ Test a connection to an S3 store using the credentials found in the environs or /tests/functional/test_s3.conf, if found. If the connection fails, mark all tests to skip. """ - self.inited = False - self.disabled = True - - if self.inited: + if self.disabled: return - if not self.CONFIG_FILE_PATH: - self.disabled_message = "GLANCE_TEST_S3_CONF environ not set." - self.inited = True - return + setup_s3(self) - if os.path.exists(TestS3.CONFIG_FILE_PATH): - cp = ConfigParser.RawConfigParser() - try: - cp.read(TestS3.CONFIG_FILE_PATH) - defaults = cp.defaults() - for key, value in defaults.items(): - self.__dict__[key] = value - except ConfigParser.ParsingError, e: - self.disabled_message = ("Failed to read test_s3.conf config " - "file. Got error: %s" % e) - self.inited = True - return - - from boto.s3.connection import S3Connection - from boto.exception import S3ResponseError - - try: - s3_host = self.s3_store_host - access_key = self.s3_store_access_key - secret_key = self.s3_store_secret_key - bucket_name = self.s3_store_bucket - except AttributeError, e: - self.disabled_message = ("Failed to find required configuration " - "options for S3 store. Got error: %s" % e) - self.inited = True - return - - s3_conn = S3Connection(access_key, secret_key, host=s3_host) - - self.bucket = None - try: - buckets = s3_conn.get_all_buckets() - for bucket in buckets: - if bucket.name == bucket_name: - self.bucket = bucket - except S3ResponseError, e: - self.disabled_message = ("Failed to connect to S3 with " - "credentials, to find bucket. " - "Got error: %s" % e) - self.inited = True - return - except TypeError, e: - # This hack is necessary because of a bug in boto 1.9b: - # http://code.google.com/p/boto/issues/detail?id=540 - self.disabled_message = ("Failed to connect to S3 with " - "credentials. Got error: %s" % e) - self.inited = True - return - - self.s3_conn = s3_conn - - if not self.bucket: - try: - self.bucket = s3_conn.create_bucket(bucket_name) - except boto.exception.S3ResponseError, e: - self.disabled_message = ("Failed to create bucket. " - "Got error: %s" % e) - self.inited = True - return - else: - self.clear_bucket() - - self.disabled = False - self.inited = True self.default_store = 's3' super(TestS3, self).setUp() def tearDown(self): - if not self.disabled: - self.clear_bucket() + teardown_s3(self) super(TestS3, self).tearDown() - def clear_bucket(self): - # It's not possible to simply clear a bucket. You - # need to loop over all the keys and delete them - # all first... - keys = self.bucket.list() - for key in keys: - key.delete() - @skip_if_disabled def test_remote_image(self): """Verify an image added using a 'Location' header can be retrieved""" @@ -162,9 +89,7 @@ class TestS3(test_api.TestApi): # 1. POST /images with public image named Image1 image_data = "*" * FIVE_KB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True'} + headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, @@ -204,11 +129,9 @@ class TestS3(test_api.TestApi): # 4. POST /images using location generated by Image1 image_id2 = utils.generate_uuid() image_data = "*" * FIVE_KB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Id': image_id2, - 'X-Image-Meta-Name': 'Image2', - 'X-Image-Meta-Is-Public': 'True', - 'X-Image-Meta-Location': s3_store_location} + headers = minimal_headers('Image2') + headers['X-Image-Meta-Id'] = image_id2 + headers['X-Image-Meta-Location'] = s3_store_location path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) @@ -245,3 +168,156 @@ class TestS3(test_api.TestApi): http.request(path % args, 'DELETE') self.stop_servers() + + def _do_test_copy_from(self, from_store, get_uri): + """ + Ensure we can copy from an external image in from_store. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # POST /images with public image to be stored in from_store, + # to stand in for the 'external' image + image_data = "*" * FIVE_KB + headers = minimal_headers('external') + headers['X-Image-Meta-Store'] = from_store + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + original_image_id = data['image']['id'] + + copy_from = get_uri(self, original_image_id) + + # POST /images with public image copied from_store (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-Is-Public': 'True', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE original image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + original_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + # GET image again to make sure the existence of the original + # image in from_store is not depended on + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() + + @skip_if_disabled + def test_copy_from_s3(self): + """ + Ensure we can copy from an external image in S3. + """ + self._do_test_copy_from('s3', get_s3_uri) + + @requires(setup_swift, teardown_swift) + @skip_if_disabled + def test_copy_from_swift(self): + """ + Ensure we can copy from an external image in Swift. + """ + self._do_test_copy_from('swift', get_swift_uri) + + @requires(setup_http, teardown_http) + @skip_if_disabled + def test_copy_from_http(self): + """ + Ensure we can copy from an external image in HTTP. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + copy_from = get_http_uri(self, 'foobar') + + # POST /images with public image copied HTTP (to S3) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() diff --git a/glance/tests/functional/test_swift.py b/glance/tests/functional/test_swift.py index 4d6de1ea95..4e36cc4034 100644 --- a/glance/tests/functional/test_swift.py +++ b/glance/tests/functional/test_swift.py @@ -30,18 +30,25 @@ If a connection cannot be established, all the test cases are skipped. """ -import ConfigParser import hashlib import httplib import httplib2 import json -import os from glance.common import crypt import glance.store.swift # Needed to register driver for location from glance.store.location import get_location_from_uri from glance.tests.functional import test_api -from glance.tests.utils import skip_if_disabled +from glance.tests.utils import skip_if_disabled, requires, minimal_headers +from glance.tests.functional.store_utils import (setup_swift, + teardown_swift, + get_swift_uri, + setup_s3, + teardown_s3, + get_s3_uri, + setup_http, + teardown_http, + get_http_uri) FIVE_KB = 5 * 1024 FIVE_MB = 5 * 1024 * 1024 @@ -51,89 +58,17 @@ class TestSwift(test_api.TestApi): """Functional tests for the Swift backend""" - # Test machines can set the GLANCE_TEST_SWIFT_CONF variable - # to override the location of the config file for migration testing - CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_SWIFT_CONF') - def setUp(self): """ Test a connection to an Swift store using the credentials found in the environs or /tests/functional/test_swift.conf, if found. If the connection fails, mark all tests to skip. """ - self.inited = False - self.disabled = True - - if self.inited: + if self.disabled: return - if not self.CONFIG_FILE_PATH: - self.disabled_message = "GLANCE_TEST_SWIFT_CONF environ not set." - self.inited = True - return + setup_swift(self) - if os.path.exists(TestSwift.CONFIG_FILE_PATH): - cp = ConfigParser.RawConfigParser() - try: - cp.read(TestSwift.CONFIG_FILE_PATH) - defaults = cp.defaults() - for key, value in defaults.items(): - self.__dict__[key] = value - except ConfigParser.ParsingError, e: - self.disabled_message = ("Failed to read test_swift.conf " - "file. Got error: %s" % e) - self.inited = True - return - - from swift.common import client as swift_client - - try: - swift_host = self.swift_store_auth_address - if not swift_host.startswith('http'): - swift_host = 'https://' + swift_host - user = self.swift_store_user - key = self.swift_store_key - container_name = self.swift_store_container - except AttributeError, e: - self.disabled_message = ("Failed to find required configuration " - "options for Swift store. " - "Got error: %s" % e) - self.inited = True - return - - self.swift_conn = swift_conn = swift_client.Connection( - authurl=swift_host, user=user, key=key, snet=False, retries=1) - - try: - _resp_headers, containers = swift_conn.get_account() - except Exception, e: - self.disabled_message = ("Failed to get_account from Swift " - "Got error: %s" % e) - self.inited = True - return - - try: - for container in containers: - if container == container_name: - swift_conn.delete_container(container) - except swift_client.ClientException, e: - self.disabled_message = ("Failed to delete container from Swift " - "Got error: %s" % e) - self.inited = True - return - - self.swift_conn = swift_conn - - try: - swift_conn.put_container(container_name) - except swift_client.ClientException, e: - self.disabled_message = ("Failed to create container. " - "Got error: %s" % e) - self.inited = True - return - - self.disabled = False - self.inited = True self.default_store = 'swift' super(TestSwift, self).setUp() @@ -185,9 +120,7 @@ class TestSwift(test_api.TestApi): # POST /images with public image named Image1 # attribute and no custom properties. Verify a 200 OK is returned image_data = "*" * FIVE_MB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True'} + headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, @@ -224,8 +157,8 @@ class TestSwift(test_api.TestApi): 'x-image-meta-name': 'Image1', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'active', - 'x-image-meta-disk_format': '', - 'x-image-meta-container_format': '', + 'x-image-meta-disk_format': 'raw', + 'x-image-meta-container_format': 'ovf', 'x-image-meta-size': str(FIVE_MB) } @@ -335,9 +268,7 @@ class TestSwift(test_api.TestApi): # 1. POST /images with public image named Image1 # attribute and no custom properties. Verify a 200 OK is returned image_data = "*" * FIVE_MB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True'} + headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, @@ -374,8 +305,8 @@ class TestSwift(test_api.TestApi): 'x-image-meta-name': 'Image1', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'active', - 'x-image-meta-disk_format': '', - 'x-image-meta-container_format': '', + 'x-image-meta-disk_format': 'raw', + 'x-image-meta-container_format': 'ovf', 'x-image-meta-size': str(FIVE_MB) } @@ -423,9 +354,7 @@ class TestSwift(test_api.TestApi): # POST /images with public image named Image1 image_data = "*" * FIVE_KB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True'} + headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, @@ -470,10 +399,8 @@ class TestSwift(test_api.TestApi): # POST /images with public image named Image1 without uploading data image_data = "*" * FIVE_KB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True', - 'X-Image-Meta-Location': swift_location} + headers = minimal_headers('Image1') + headers['X-Image-Meta-Location'] = swift_location path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) @@ -512,3 +439,156 @@ class TestSwift(test_api.TestApi): self.assertEqual(response.status, 200) self.stop_servers() + + def _do_test_copy_from(self, from_store, get_uri): + """ + Ensure we can copy from an external image in from_store. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # POST /images with public image to be stored in from_store, + # to stand in for the 'external' image + image_data = "*" * FIVE_KB + headers = minimal_headers('external') + headers['X-Image-Meta-Store'] = from_store + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + original_image_id = data['image']['id'] + + copy_from = get_uri(self, original_image_id) + + # POST /images with public image copied from_store (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE original image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + original_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + # GET image again to make sure the existence of the original + # image in from_store is not depended on + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() + + @skip_if_disabled + def test_copy_from_swift(self): + """ + Ensure we can copy from an external image in Swift. + """ + self._do_test_copy_from('swift', get_swift_uri) + + @requires(setup_s3, teardown_s3) + @skip_if_disabled + def test_copy_from_s3(self): + """ + Ensure we can copy from an external image in S3. + """ + self._do_test_copy_from('s3', get_s3_uri) + + @requires(setup_http, teardown_http) + @skip_if_disabled + def test_copy_from_http(self): + """ + Ensure we can copy from an external image in HTTP. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + copy_from = get_http_uri(self, 'foobar') + + # POST /images with public image copied HTTP (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() diff --git a/glance/tests/unit/test_s3_store.py b/glance/tests/unit/test_s3_store.py index e68dfee402..b7da8af915 100644 --- a/glance/tests/unit/test_s3_store.py +++ b/glance/tests/unit/test_s3_store.py @@ -273,7 +273,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_s3, new_image_size) = self.store.get(loc) new_image_contents = new_image_s3.getvalue() - new_image_s3_size = new_image_s3.len + new_image_s3_size = len(new_image_s3) self.assertEquals(expected_s3_contents, new_image_contents) self.assertEquals(expected_s3_size, new_image_s3_size) diff --git a/glance/tests/unit/test_swift_store.py b/glance/tests/unit/test_swift_store.py index 905241f5df..303f1c3379 100644 --- a/glance/tests/unit/test_swift_store.py +++ b/glance/tests/unit/test_swift_store.py @@ -261,7 +261,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) @@ -318,7 +318,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) @@ -382,7 +382,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) @@ -430,7 +430,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) @@ -491,7 +491,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) diff --git a/glance/tests/utils.py b/glance/tests/utils.py index 2fb05a3153..d69bcbcc45 100644 --- a/glance/tests/utils.py +++ b/glance/tests/utils.py @@ -154,6 +154,23 @@ class skip_unless(object): return _skipper +class requires(object): + """Decorator that initiates additional test setup/teardown.""" + def __init__(self, setup, teardown=None): + self.setup = setup + self.teardown = teardown + + def __call__(self, func): + def _runner(*args, **kw): + self.setup(args[0]) + func(*args, **kw) + if self.teardown: + self.teardown(args[0]) + _runner.__name__ = func.__name__ + _runner.__doc__ = func.__doc__ + return _runner + + def skip_if_disabled(func): """Decorator that skips a test if test case is disabled.""" @functools.wraps(func)