From c83bce1ce69bf704fbbe6ce944094c14e79996d2 Mon Sep 17 00:00:00 2001 From: Eoghan Glynn Date: Thu, 9 Feb 2012 14:23:32 +0000 Subject: [PATCH] Support new image copied from external storage. Implements bp retrieve-image-from Added support for a new header: x-glance-api-copy-from: URI where URI is a HTTP, S3, or Swift location that's accessible to the glance API service. Where present, the image content is copied from the external store to the backend store currently configured for the glance API service. Impedence between the representation returned by Store.get() and the implicit requirements of Store.add() required careful iterator/ filelike wrapping. Also rationalized the S3 and Swift functional test setup logic and added a new decorator-based mechanism to allow a functional testcase require multiple backend stores to be available and configured. Change-Id: If783a09c273832b8926aabc60b7eba8b3ab956d6 --- bin/glance | 34 ++- doc/source/glance.rst | 11 +- glance/api/v1/images.py | 116 +++++--- glance/client.py | 5 +- glance/common/utils.py | 13 + glance/store/__init__.py | 64 ++++ glance/store/filesystem.py | 7 +- glance/store/http.py | 9 +- glance/store/s3.py | 13 +- glance/store/swift.py | 10 +- glance/tests/functional/store_utils.py | 273 ++++++++++++++++++ glance/tests/functional/test_api.py | 2 +- glance/tests/functional/test_bin_glance.py | 47 ++- .../tests/functional/test_cache_middleware.py | 56 +--- glance/tests/functional/test_copy_to_file.py | 225 +++++++++++++++ glance/tests/functional/test_s3.py | 270 ++++++++++------- glance/tests/functional/test_swift.py | 268 +++++++++++------ glance/tests/unit/test_s3_store.py | 2 +- glance/tests/unit/test_swift_store.py | 10 +- glance/tests/utils.py | 17 ++ 20 files changed, 1140 insertions(+), 312 deletions(-) create mode 100644 glance/tests/functional/store_utils.py create mode 100644 glance/tests/functional/test_copy_to_file.py diff --git a/bin/glance b/bin/glance index de5483db35..a494ca7bc3 100755 --- a/bin/glance +++ b/bin/glance @@ -235,13 +235,21 @@ EXAMPLES print 'Found non-settable field %s. Removing.' % field fields.pop(field) - if 'location' in fields.keys(): - image_meta['location'] = fields.pop('location') + def _external_source(fields, image_data): + source = None + features = {} + if 'location' in fields.keys(): + source = fields.pop('location') + image_meta['location'] = source + elif 'copy_from' in fields.keys(): + source = fields.pop('copy_from') + features['x-glance-api-copy-from'] = source + return source, features # We need either a location or image data/stream to add... - image_location = image_meta.get('location') + location, features = _external_source(fields, image_meta) image_data = None - if not image_location: + if not location: # Grab the image data stream from stdin or redirect, # otherwise error out image_data = sys.stdin @@ -251,7 +259,8 @@ EXAMPLES if not options.dry_run: try: - image_meta = c.add_image(image_meta, image_data) + image_meta = c.add_image(image_meta, image_data, + features=features) image_id = image_meta['id'] print "Added new image with ID: %s" % image_id if options.verbose: @@ -278,9 +287,17 @@ EXAMPLES return FAILURE else: print "Dry run. We would have done the following:" + + def _dump(dict): + for k, v in sorted(dict.items()): + print " %(k)30s => %(v)s" % locals() + print "Add new image with metadata:" - for k, v in sorted(image_meta.items()): - print " %(k)30s => %(v)s" % locals() + _dump(image_meta) + + if features: + print "with features enabled:" + _dump(features) return SUCCESS @@ -299,7 +316,8 @@ to Glance that represents the metadata for an image. Field names that can be specified: name A name for the image. -location The location of the image. +location An external location to serve out from. +copy_from An external location (HTTP, S3 or Swift URI) to copy from. is_public If specified, interpreted as a boolean value and sets or unsets the image's availability to the public. protected If specified, interpreted as a boolean value diff --git a/doc/source/glance.rst b/doc/source/glance.rst index e585f8d416..f2c946f606 100644 --- a/doc/source/glance.rst +++ b/doc/source/glance.rst @@ -260,13 +260,20 @@ To upload an EC2 tarball VM image with an associated property (e.g., distro):: container_format=ovf disk_format=raw \ distro="ubuntu 10.10" < /root/maverick-server-uec-amd64.tar.gz -To upload an EC2 tarball VM image from a URL:: +To reference an EC2 tarball VM image available at an external URL:: - $> glance add name="uubntu-10.04-amd64" is_public=true \ + $> glance add name="ubuntu-10.04-amd64" is_public=true \ container_format=ovf disk_format=raw \ location="http://uec-images.ubuntu.com/lucid/current/\ lucid-server-uec-amd64.tar.gz" +To upload a copy of that same EC2 tarball VM image:: + + $> glance add name="ubuntu-10.04-amd64" is_public=true \ + container_format=ovf disk_format=raw \ + copy_from="http://uec-images.ubuntu.com/lucid/current/\ + lucid-server-uec-amd64.tar.gz" + To upload a qcow2 image:: $> glance add name="ubuntu-11.04-amd64" is_public=true \ diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index c291b417ca..6147febd3e 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -226,6 +226,23 @@ class Controller(controller.BaseController): 'image_meta': image_meta } + @staticmethod + def _copy_from(req): + return req.headers.get('x-glance-api-copy-from') + + @staticmethod + def _external_source(image_meta, req): + return image_meta.get('location', Controller._copy_from(req)) + + @staticmethod + def _get_from_store(where): + try: + image_data, image_size = get_from_backend(where) + except exception.NotFound, e: + raise HTTPNotFound(explanation="%s" % e) + image_size = int(image_size) if image_size else None + return image_data, image_size + def show(self, req, id): """ Returns an iterator that can be used to retrieve an image's @@ -239,16 +256,9 @@ class Controller(controller.BaseController): self._enforce(req, 'get_image') image_meta = self.get_active_image_meta_or_404(req, id) - def get_from_store(image_meta): - try: - location = image_meta['location'] - image_data, image_size = get_from_backend(location) - image_meta["size"] = image_size or image_meta["size"] - except exception.NotFound, e: - raise HTTPNotFound(explanation="%s" % e) - return image_data + image_iterator, size = self._get_from_store(image_meta['location']) + image_meta['size'] = size or image_meta['size'] - image_iterator = get_from_store(image_meta) del image_meta['location'] return { 'image_iterator': image_iterator, @@ -263,11 +273,12 @@ class Controller(controller.BaseController): :param req: The WSGI/Webob Request object :param id: The opaque image identifier + :param image_meta: The image metadata :raises HTTPConflict if image already exists :raises HTTPBadRequest if image metadata is not valid """ - location = image_meta.get('location') + location = self._external_source(image_meta, req) if location: store = get_store_from_location(location) # check the store exists before we hit the registry, but we @@ -278,9 +289,9 @@ class Controller(controller.BaseController): image_meta['size'] = image_meta.get('size', 0) \ or get_size_from_backend(location) else: - # Ensure that the size attribute is set to zero for uploadable - # images (if not provided). The size will be set to a non-zero - # value during upload + # Ensure that the size attribute is set to zero for directly + # uploadable images (if not provided). The size will be set + # to a non-zero value during upload image_meta['size'] = image_meta.get('size', 0) image_meta['status'] = 'queued' @@ -317,13 +328,30 @@ class Controller(controller.BaseController): :raises HTTPConflict if image already exists :retval The location where the image was stored """ - try: - req.get_content_type('application/octet-stream') - except exception.InvalidContentType: - self._safe_kill(req, image_meta['id']) - msg = _("Content-Type must be application/octet-stream") - logger.error(msg) - raise HTTPBadRequest(explanation=msg) + + copy_from = self._copy_from(req) + if copy_from: + image_data, image_size = self._get_from_store(copy_from) + image_meta['size'] = image_size or image_meta['size'] + else: + try: + req.get_content_type('application/octet-stream') + except exception.InvalidContentType: + self._safe_kill(req, image_meta['id']) + msg = _("Content-Type must be application/octet-stream") + logger.error(msg) + raise HTTPBadRequest(explanation=msg) + + image_data = req.body_file + + if req.content_length: + image_size = int(req.content_length) + elif 'x-image-meta-size' in req.headers: + image_size = int(req.headers['x-image-meta-size']) + else: + logger.debug(_("Got request with no content-length and no " + "x-image-meta-size header")) + image_size = 0 store_name = req.headers.get('x-image-meta-store', self.conf.default_store) @@ -337,14 +365,6 @@ class Controller(controller.BaseController): try: logger.debug(_("Uploading image data for image %(image_id)s " "to %(store_name)s store"), locals()) - if req.content_length: - image_size = int(req.content_length) - elif 'x-image-meta-size' in req.headers: - image_size = int(req.headers['x-image-meta-size']) - else: - logger.debug(_("Got request with no content-length and no " - "x-image-meta-size header")) - image_size = 0 if image_size > IMAGE_SIZE_CAP: max_image_size = IMAGE_SIZE_CAP @@ -355,7 +375,7 @@ class Controller(controller.BaseController): raise HTTPBadRequest(msg, request=request) location, size, checksum = store.add(image_meta['id'], - req.body_file, + image_data, image_size) # Verify any supplied checksum value matches checksum @@ -505,19 +525,27 @@ class Controller(controller.BaseController): def create(self, req, image_meta, image_data): """ - Adds a new image to Glance. Three scenarios exist when creating an + Adds a new image to Glance. Four scenarios exist when creating an image: - 1. If the image data is available for upload, create can be passed the - image data as the request body and the metadata as the request - headers. The image will initially be 'queued', during upload it - will be in the 'saving' status, and then 'killed' or 'active' - depending on whether the upload completed successfully. + 1. If the image data is available directly for upload, create can be + passed the image data as the request body and the metadata as the + request headers. The image will initially be 'queued', during + upload it will be in the 'saving' status, and then 'killed' or + 'active' depending on whether the upload completed successfully. - 2. If the image data exists somewhere else, you can pass in the source - using the x-image-meta-location header + 2. If the image data exists somewhere else, you can upload indirectly + from the external source using the x-glance-api-copy-from header. + Once the image is uploaded, the external store is not subsequently + consulted, i.e. the image content is served out from the configured + glance image store. State transitions are as for option #1. - 3. If the image data is not available yet, but you'd like reserve a + 3. If the image data exists somewhere else, you can reference the + source using the x-image-meta-location header. The image content + will be served out from the external store, i.e. is never uploaded + to the configured glance image store. + + 4. If the image data is not available yet, but you'd like reserve a spot for it, you can omit the data and a record will be created in the 'queued' state. This exists primarily to maintain backwards compatibility with OpenStack/Rackspace API semantics. @@ -547,7 +575,7 @@ class Controller(controller.BaseController): image_meta = self._reserve(req, image_meta) image_id = image_meta['id'] - if image_data is not None: + if image_data or self._copy_from(req): image_meta = self._upload_and_activate(req, image_meta) else: location = image_meta.get('location') @@ -594,10 +622,12 @@ class Controller(controller.BaseController): if image_data is not None and orig_status != 'queued': raise HTTPConflict(_("Cannot upload to an unqueued image")) - # Only allow the Location fields to be modified if the image is - # in queued status, which indicates that the user called POST /images - # but did not supply either a Location field OR image data - if not orig_status == 'queued' and 'location' in image_meta: + # Only allow the Location|Copy-From fields to be modified if the + # image is in queued status, which indicates that the user called + # POST /images but originally supply neither a Location|Copy-From + # field NOR image data + location = self._external_source(image_meta, req) + if not orig_status == 'queued' and location: msg = _("Attempted to update Location field for an image " "not in queued status.") raise HTTPBadRequest(msg, request=req, content_type="text/plain") diff --git a/glance/client.py b/glance/client.py index 61a93e76a2..83c78352ba 100644 --- a/glance/client.py +++ b/glance/client.py @@ -126,7 +126,7 @@ class V1Client(base_client.BaseClient): else: raise - def add_image(self, image_meta=None, image_data=None): + def add_image(self, image_meta=None, image_data=None, features=None): """ Tells Glance about an image's metadata as well as optionally the image_data itself @@ -136,6 +136,7 @@ class V1Client(base_client.BaseClient): :param image_data: Optional string of raw image data or file-like object that can be used to read the image data + :param features: Optional map of features :retval The newly-stored image's metadata. """ @@ -151,6 +152,8 @@ class V1Client(base_client.BaseClient): else: body = None + utils.add_features_to_http_headers(features, headers) + res = self.do_request("POST", "/images", body, headers) data = json.loads(res.read()) return data['image'] diff --git a/glance/common/utils.py b/glance/common/utils.py index 01ebd65914..a7997853d2 100644 --- a/glance/common/utils.py +++ b/glance/common/utils.py @@ -88,6 +88,19 @@ def image_meta_to_http_headers(image_meta): return headers +def add_features_to_http_headers(features, headers): + """ + Adds additional headers representing glance features to be enabled. + + :param headers: Base set of headers + :param features: Map of enabled features + """ + if features: + for k, v in features.items(): + if v is not None: + headers[k.lower()] = unicode(v) + + def get_image_meta_from_headers(response): """ Processes HTTP headers from a supplied response that diff --git a/glance/store/__init__.py b/glance/store/__init__.py index 9ef577482d..4f2f334162 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -66,6 +66,70 @@ class UnsupportedBackend(BackendException): pass +class Indexable(object): + + """ + Wrapper that allows an iterator or filelike be treated as an indexable + data structure. This is required in the case where the return value from + Store.get() is passed to Store.add() when adding a Copy-From image to a + Store where the client library relies on eventlet GreenSockets, in which + case the data to be written is indexed over. + """ + + def __init__(self, wrapped, size): + """ + Initialize the object + + :param wrappped: the wrapped iterator or filelike. + :param size: the size of data available + """ + self.wrapped = wrapped + self.size = int(size) if size else (wrapped.len + if hasattr(wrapped, 'len') else 0) + self.cursor = 0 + self.chunk = None + + def __iter__(self): + """ + Delegate iteration to the wrapped instance. + """ + for self.chunk in self.wrapped: + yield self.chunk + + def __getitem__(self, i): + """ + Index into the next chunk (or previous chunk in the case where + the last data returned was not fully consumed). + + :param i: a slice-to-the-end + """ + start = i.start if isinstance(i, slice) else i + if start < self.cursor: + return self.chunk[(start - self.cursor):] + + self.chunk = self.another() + if self.chunk: + self.cursor += len(self.chunk) + + return self.chunk + + def another(self): + """Implemented by subclasses to return the next element""" + raise NotImplementedError + + def getvalue(self): + """ + Return entire string value... used in testing + """ + return self.wrapped.getvalue() + + def __len__(self): + """ + Length accessor. + """ + return self.size + + def register_store(store_module, schemes): """ Registers a store module and a set of schemes diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py index df80b51a0b..ecbd2041c6 100644 --- a/glance/store/filesystem.py +++ b/glance/store/filesystem.py @@ -27,6 +27,7 @@ import urlparse from glance.common import cfg from glance.common import exception +from glance.common import utils import glance.store import glance.store.base import glance.store.location @@ -198,10 +199,8 @@ class Store(glance.store.base.Store): bytes_written = 0 try: with open(filepath, 'wb') as f: - while True: - buf = image_file.read(ChunkedFile.CHUNKSIZE) - if not buf: - break + for buf in utils.chunkreadable(image_file, + ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) diff --git a/glance/store/http.py b/glance/store/http.py index 04697c9f1a..8c56d95bf8 100644 --- a/glance/store/http.py +++ b/glance/store/http.py @@ -117,7 +117,14 @@ class Store(glance.store.base.Store): iterator = http_response_iterator(conn, resp, self.CHUNKSIZE) - return (iterator, content_length) + class ResponseIndexable(glance.store.Indexable): + def another(self): + try: + return self.wrapped.next() + except StopIteration: + return '' + + return (ResponseIndexable(iterator, content_length), content_length) def get_size(self, location): """ diff --git a/glance/store/s3.py b/glance/store/s3.py index 4e88afbb41..7f10718112 100644 --- a/glance/store/s3.py +++ b/glance/store/s3.py @@ -25,6 +25,7 @@ import urlparse from glance.common import cfg from glance.common import exception +from glance.common import utils import glance.store import glance.store.base import glance.store.location @@ -250,7 +251,13 @@ class Store(glance.store.base.Store): key = self._retrieve_key(location) key.BufferSize = self.CHUNKSIZE - return (ChunkedFile(key), key.size) + + class ChunkedIndexable(glance.store.Indexable): + def another(self): + return (self.wrapped.fp.read(ChunkedFile.CHUNKSIZE) + if self.wrapped.fp else None) + + return (ChunkedIndexable(ChunkedFile(key), key.size), key.size) def get_size(self, location): """ @@ -361,11 +368,9 @@ class Store(glance.store.base.Store): tmpdir = self.s3_store_object_buffer_dir temp_file = tempfile.NamedTemporaryFile(dir=tmpdir) checksum = hashlib.md5() - chunk = image_file.read(self.CHUNKSIZE) - while chunk: + for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE): checksum.update(chunk) temp_file.write(chunk) - chunk = image_file.read(self.CHUNKSIZE) temp_file.flush() msg = _("Uploading temporary file to S3 for %s") % loc.get_uri() diff --git a/glance/store/swift.py b/glance/store/swift.py index d4ac71d267..616408cd2a 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -273,7 +273,15 @@ class Store(glance.store.base.Store): # "Expected %s byte file, Swift has %s bytes" % # (expected_size, obj_size)) - return (resp_body, resp_headers.get('content-length')) + class ResponseIndexable(glance.store.Indexable): + def another(self): + try: + return self.wrapped.next() + except StopIteration: + return '' + + length = resp_headers.get('content-length') + return (ResponseIndexable(resp_body, length), length) def get_size(self, location): """ diff --git a/glance/tests/functional/store_utils.py b/glance/tests/functional/store_utils.py new file mode 100644 index 0000000000..de2e4d0ab6 --- /dev/null +++ b/glance/tests/functional/store_utils.py @@ -0,0 +1,273 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# Copyright 2012 Red Hat, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utility methods to set testcases up for Swift and/or S3 tests. +""" + +import BaseHTTPServer +import ConfigParser +import httplib +import os +import thread + + +FIVE_KB = 5 * 1024 + + +class RemoteImageHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def do_HEAD(self): + """ + Respond to an image HEAD request fake metadata + """ + if 'images' in self.path: + self.send_response(200) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', FIVE_KB) + self.end_headers() + return + else: + self.send_error(404, 'File Not Found: %s' % self.path) + return + + def do_GET(self): + """ + Respond to an image GET request with fake image content. + """ + if 'images' in self.path: + self.send_response(200) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', FIVE_KB) + self.end_headers() + image_data = '*' * FIVE_KB + self.wfile.write(image_data) + self.wfile.close() + return + else: + self.send_error(404, 'File Not Found: %s' % self.path) + return + + +def setup_http(test): + server_class = BaseHTTPServer.HTTPServer + remote_server = server_class(('127.0.0.1', 0), RemoteImageHandler) + remote_ip, remote_port = remote_server.server_address + + def serve_requests(httpd): + httpd.serve_forever() + + thread.start_new_thread(serve_requests, (remote_server,)) + test.http_server = remote_server + test.http_ip = remote_ip + test.http_port = remote_port + + +def teardown_http(test): + test.http_server.shutdown() + + +def get_http_uri(test, image_id): + uri = 'http://%(http_ip)s:%(http_port)d/images/' % test.__dict__ + uri += image_id + return uri + + +def setup_swift(test): + # Test machines can set the GLANCE_TEST_SWIFT_CONF variable + # to override the location of the config file for migration testing + CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_SWIFT_CONF') + + if not CONFIG_FILE_PATH: + test.disabled_message = "GLANCE_TEST_SWIFT_CONF environ not set." + print "GLANCE_TEST_SWIFT_CONF environ not set." + test.disabled = True + return + + if os.path.exists(CONFIG_FILE_PATH): + cp = ConfigParser.RawConfigParser() + try: + cp.read(CONFIG_FILE_PATH) + defaults = cp.defaults() + for key, value in defaults.items(): + test.__dict__[key] = value + except ConfigParser.ParsingError, e: + test.disabled_message = ("Failed to read test_swift.conf " + "file. Got error: %s" % e) + test.disabled = True + return + + from swift.common import client as swift_client + + try: + swift_host = test.swift_store_auth_address + if not swift_host.startswith('http'): + swift_host = 'https://' + swift_host + user = test.swift_store_user + key = test.swift_store_key + container_name = test.swift_store_container + except AttributeError, e: + test.disabled_message = ("Failed to find required configuration " + "options for Swift store. " + "Got error: %s" % e) + test.disabled = True + return + + swift_conn = swift_client.Connection( + authurl=swift_host, user=user, key=key, snet=False, retries=1) + + try: + _resp_headers, containers = swift_conn.get_account() + except Exception, e: + test.disabled_message = ("Failed to get_account from Swift " + "Got error: %s" % e) + test.disabled = True + return + + try: + for container in containers: + if container == container_name: + swift_conn.delete_container(container) + except swift_client.ClientException, e: + test.disabled_message = ("Failed to delete container from Swift " + "Got error: %s" % e) + test.disabled = True + return + + test.swift_conn = swift_conn + + try: + swift_conn.put_container(container_name) + except swift_client.ClientException, e: + test.disabled_message = ("Failed to create container. " + "Got error: %s" % e) + test.disabled = True + return + + +def teardown_swift(test): + if not test.disabled: + from swift.common import client as swift_client + try: + test.swift_conn.delete_container(test.swift_store_container) + except swift_client.ClientException, e: + if e.http_status == httplib.CONFLICT: + pass + else: + raise + test.swift_conn.put_container(test.swift_store_container) + + +def get_swift_uri(test, image_id): + uri = ('swift+http://%(swift_store_user)s:%(swift_store_key)s' % + test.__dict__) + uri += ('@%(swift_store_auth_address)s/%(swift_store_container)s/' % + test.__dict__) + uri += image_id + return uri.replace('@http://', '@') + + +def setup_s3(test): + # Test machines can set the GLANCE_TEST_S3_CONF variable + # to override the location of the config file for S3 testing + CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_S3_CONF') + + if not CONFIG_FILE_PATH: + test.disabled_message = "GLANCE_TEST_S3_CONF environ not set." + test.disabled = True + return + + if os.path.exists(CONFIG_FILE_PATH): + cp = ConfigParser.RawConfigParser() + try: + cp.read(CONFIG_FILE_PATH) + defaults = cp.defaults() + for key, value in defaults.items(): + test.__dict__[key] = value + except ConfigParser.ParsingError, e: + test.disabled_message = ("Failed to read test_s3.conf config " + "file. Got error: %s" % e) + test.disabled = True + return + + from boto.s3.connection import S3Connection + from boto.exception import S3ResponseError + + try: + s3_host = test.s3_store_host + access_key = test.s3_store_access_key + secret_key = test.s3_store_secret_key + bucket_name = test.s3_store_bucket + except AttributeError, e: + test.disabled_message = ("Failed to find required configuration " + "options for S3 store. Got error: %s" % e) + test.disabled = True + return + + s3_conn = S3Connection(access_key, secret_key, host=s3_host) + + test.bucket = None + try: + buckets = s3_conn.get_all_buckets() + for bucket in buckets: + if bucket.name == bucket_name: + test.bucket = bucket + except S3ResponseError, e: + test.disabled_message = ("Failed to connect to S3 with " + "credentials, to find bucket. " + "Got error: %s" % e) + test.disabled = True + return + except TypeError, e: + # This hack is necessary because of a bug in boto 1.9b: + # http://code.google.com/p/boto/issues/detail?id=540 + test.disabled_message = ("Failed to connect to S3 with " + "credentials. Got error: %s" % e) + test.disabled = True + return + + test.s3_conn = s3_conn + + if not test.bucket: + try: + test.bucket = s3_conn.create_bucket(bucket_name) + except boto.exception.S3ResponseError, e: + test.disabled_message = ("Failed to create bucket. " + "Got error: %s" % e) + test.disabled = True + return + else: + for key in test.bucket.list(): + key.delete() + + +def teardown_s3(test): + if not test.disabled: + # It's not possible to simply clear a bucket. You + # need to loop over all the keys and delete them + # all first... + for key in test.bucket.list(): + key.delete() + + +def get_s3_uri(test, image_id): + uri = ('s3://%(s3_store_access_key)s:%(s3_store_secret_key)s' % + test.__dict__) + uri += '@%(s3_conn)s/' % test.__dict__ + uri += '%(s3_store_bucket)s/' % test.__dict__ + uri += image_id + return uri.replace('S3Connection:', '') diff --git a/glance/tests/functional/test_api.py b/glance/tests/functional/test_api.py index 93259fe1bb..8359324424 100644 --- a/glance/tests/functional/test_api.py +++ b/glance/tests/functional/test_api.py @@ -1307,7 +1307,7 @@ class TestApi(functional.FunctionalTest): to fail to start. """ self.cleanup() - self.api_server.default_store = 'shouldnotexist' + self.default_store = 'shouldnotexist' # ensure failure exit code is available to assert on self.api_server.server_control_options += ' --await-child=1' diff --git a/glance/tests/functional/test_bin_glance.py b/glance/tests/functional/test_bin_glance.py index 8a92dac24c..267f985999 100644 --- a/glance/tests/functional/test_bin_glance.py +++ b/glance/tests/functional/test_bin_glance.py @@ -23,7 +23,10 @@ import tempfile from glance.common import utils from glance.tests import functional -from glance.tests.utils import execute, minimal_add_command +from glance.tests.utils import execute, requires, minimal_add_command +from glance.tests.functional.store_utils import (setup_http, + teardown_http, + get_http_uri) class TestBinGlance(functional.FunctionalTest): @@ -80,6 +83,48 @@ class TestBinGlance(functional.FunctionalTest): self.assertEqual('0', size, "Expected image to be 0 bytes in size, " "but got %s. " % size) + @requires(setup_http, teardown_http) + def test_add_copying_from(self): + self.cleanup() + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # 0. Verify no public images + cmd = "bin/glance --port=%d index" % api_port + + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + self.assertEqual('', out.strip()) + + # 1. Add public image + suffix = 'copy_from=%s' % get_http_uri(self, 'foobar') + cmd = minimal_add_command(api_port, 'MyImage', suffix) + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + self.assertTrue(out.strip().startswith('Added new image with ID:')) + + # 2. Verify image added as public image + cmd = "bin/glance --port=%d index" % api_port + + exitcode, out, err = execute(cmd) + + self.assertEqual(0, exitcode) + lines = out.split("\n")[2:-1] + self.assertEqual(1, len(lines)) + + line = lines[0] + + image_id, name, disk_format, container_format, size = \ + [c.strip() for c in line.split()] + self.assertEqual('MyImage', name) + + self.assertEqual('5120', size, "Expected image to be 0 bytes in size," + " but got %s. " % size) + def test_add_with_location_and_stdin(self): self.cleanup() self.start_servers(**self.__dict__.copy()) diff --git a/glance/tests/functional/test_cache_middleware.py b/glance/tests/functional/test_cache_middleware.py index c3c9ab8211..b9f54b69f8 100644 --- a/glance/tests/functional/test_cache_middleware.py +++ b/glance/tests/functional/test_cache_middleware.py @@ -29,53 +29,22 @@ import shutil import thread import time -import BaseHTTPServer import httplib2 from glance.tests import functional from glance.tests.utils import (skip_if_disabled, + requires, execute, xattr_writes_supported, - minimal_headers, - ) + minimal_headers) +from glance.tests.functional.store_utils import (setup_http, + teardown_http, + get_http_uri) FIVE_KB = 5 * 1024 -class RemoteImageHandler(BaseHTTPServer.BaseHTTPRequestHandler): - def do_HEAD(self): - """ - Respond to an image HEAD request fake metadata - """ - if 'images' in self.path: - self.send_response(200) - self.send_header('Content-Type', 'application/octet-stream') - self.send_header('Content-Length', FIVE_KB) - self.end_headers() - return - else: - self.send_error(404, 'File Not Found: %s' % self.path) - return - - def do_GET(self): - """ - Respond to an image GET request with fake image content. - """ - if 'images' in self.path: - self.send_response(200) - self.send_header('Content-Type', 'application/octet-stream') - self.send_header('Content-Length', FIVE_KB) - self.end_headers() - image_data = '*' * FIVE_KB - self.wfile.write(image_data) - self.wfile.close() - return - else: - self.send_error(404, 'File Not Found: %s' % self.path) - return - - class BaseCacheMiddlewareTest(object): @skip_if_disabled @@ -153,6 +122,7 @@ class BaseCacheMiddlewareTest(object): self.stop_servers() + @requires(setup_http, teardown_http) @skip_if_disabled def test_cache_remote_image(self): """ @@ -164,18 +134,8 @@ class BaseCacheMiddlewareTest(object): api_port = self.api_port registry_port = self.registry_port - # set up "remote" image server - server_class = BaseHTTPServer.HTTPServer - remote_server = server_class(('127.0.0.1', 0), RemoteImageHandler) - remote_ip, remote_port = remote_server.server_address - - def serve_requests(httpd): - httpd.serve_forever() - - thread.start_new_thread(serve_requests, (remote_server,)) - # Add a remote image and verify a 201 Created is returned - remote_uri = 'http://%s:%d/images/2' % (remote_ip, remote_port) + remote_uri = get_http_uri(self, '2') headers = {'X-Image-Meta-Name': 'Image2', 'X-Image-Meta-disk_format': 'raw', 'X-Image-Meta-container_format': 'ovf', @@ -204,8 +164,6 @@ class BaseCacheMiddlewareTest(object): self.assertEqual(response.status, 200) self.assertEqual(int(response['content-length']), FIVE_KB) - remote_server.shutdown() - self.stop_servers() diff --git a/glance/tests/functional/test_copy_to_file.py b/glance/tests/functional/test_copy_to_file.py new file mode 100644 index 0000000000..e376154bd8 --- /dev/null +++ b/glance/tests/functional/test_copy_to_file.py @@ -0,0 +1,225 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# Copyright 2012 Red Hat, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests copying images to a Glance API server which uses a filesystem- +based storage backend. + +The from_swift testcase requires that a real Swift account is available. +It looks in a file GLANCE_TEST_SWIFT_CONF environ variable for the +credentials to use. + +Note that this test clears the entire container from the Swift account +for use by the test case, so make sure you supply credentials for +test accounts only. + +The from_s3 testcase requires that a real S3 account is available. +It looks in a file specified in the GLANCE_TEST_S3_CONF environ variable +for the credentials to use. + +Note that this test clears the entire bucket from the S3 account +for use by the test case, so make sure you supply credentials for +test accounts only. + +In either case, if a connection to the external store cannot be +established, then the relevant test case is skipped. +""" + +import hashlib +import httplib2 +import json + +from glance.tests import functional +from glance.tests.utils import skip_if_disabled, requires +from glance.tests.functional.store_utils import (setup_swift, + teardown_swift, + get_swift_uri, + setup_s3, + teardown_s3, + get_s3_uri, + setup_http, + teardown_http, + get_http_uri) + +FIVE_KB = 5 * 1024 + + +class TestCopyToFile(functional.FunctionalTest): + + """ + Functional tests for copying images from the Swift, S3 & HTTP storage + backends to file + """ + + def _do_test_copy_from(self, from_store, get_uri): + """ + Ensure we can copy from an external image in from_store. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # POST /images with public image to be stored in from_store, + # to stand in for the 'external' image + image_data = "*" * FIVE_KB + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'external', + 'X-Image-Meta-Store': from_store, + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + original_image_id = data['image']['id'] + + copy_from = get_uri(self, original_image_id) + + # POST /images with public image copied from_store (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE original image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + original_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + # GET image again to make sure the existence of the original + # image in from_store is not depended on + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() + + @requires(setup_swift, teardown_swift) + @skip_if_disabled + def test_copy_from_swift(self): + """ + Ensure we can copy from an external image in Swift. + """ + self._do_test_copy_from('swift', get_swift_uri) + + @requires(setup_s3, teardown_s3) + @skip_if_disabled + def test_copy_from_s3(self): + """ + Ensure we can copy from an external image in S3. + """ + self._do_test_copy_from('s3', get_s3_uri) + + @requires(setup_http, teardown_http) + @skip_if_disabled + def test_copy_from_http(self): + """ + Ensure we can copy from an external image in HTTP. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + copy_from = get_http_uri(self, 'foobar') + + # POST /images with public image copied HTTP (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() diff --git a/glance/tests/functional/test_s3.py b/glance/tests/functional/test_s3.py index abc2526407..79fdf066b8 100644 --- a/glance/tests/functional/test_s3.py +++ b/glance/tests/functional/test_s3.py @@ -30,10 +30,8 @@ If a connection cannot be established, all the test cases are skipped. """ -import ConfigParser import hashlib import json -import os import tempfile import unittest @@ -42,7 +40,19 @@ import httplib2 from glance.common import crypt from glance.common import utils from glance.tests.functional import test_api -from glance.tests.utils import execute, skip_if_disabled +from glance.tests.utils import (execute, + skip_if_disabled, + requires, + minimal_headers) +from glance.tests.functional.store_utils import (setup_s3, + teardown_s3, + get_s3_uri, + setup_swift, + teardown_swift, + get_swift_uri, + setup_http, + teardown_http, + get_http_uri) FIVE_KB = 5 * 1024 @@ -52,108 +62,25 @@ class TestS3(test_api.TestApi): """Functional tests for the S3 backend""" - # Test machines can set the GLANCE_TEST_S3_CONF variable - # to override the location of the config file for S3 testing - CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_S3_CONF') - def setUp(self): """ Test a connection to an S3 store using the credentials found in the environs or /tests/functional/test_s3.conf, if found. If the connection fails, mark all tests to skip. """ - self.inited = False - self.disabled = True - - if self.inited: + if self.disabled: return - if not self.CONFIG_FILE_PATH: - self.disabled_message = "GLANCE_TEST_S3_CONF environ not set." - self.inited = True - return + setup_s3(self) - if os.path.exists(TestS3.CONFIG_FILE_PATH): - cp = ConfigParser.RawConfigParser() - try: - cp.read(TestS3.CONFIG_FILE_PATH) - defaults = cp.defaults() - for key, value in defaults.items(): - self.__dict__[key] = value - except ConfigParser.ParsingError, e: - self.disabled_message = ("Failed to read test_s3.conf config " - "file. Got error: %s" % e) - self.inited = True - return - - from boto.s3.connection import S3Connection - from boto.exception import S3ResponseError - - try: - s3_host = self.s3_store_host - access_key = self.s3_store_access_key - secret_key = self.s3_store_secret_key - bucket_name = self.s3_store_bucket - except AttributeError, e: - self.disabled_message = ("Failed to find required configuration " - "options for S3 store. Got error: %s" % e) - self.inited = True - return - - s3_conn = S3Connection(access_key, secret_key, host=s3_host) - - self.bucket = None - try: - buckets = s3_conn.get_all_buckets() - for bucket in buckets: - if bucket.name == bucket_name: - self.bucket = bucket - except S3ResponseError, e: - self.disabled_message = ("Failed to connect to S3 with " - "credentials, to find bucket. " - "Got error: %s" % e) - self.inited = True - return - except TypeError, e: - # This hack is necessary because of a bug in boto 1.9b: - # http://code.google.com/p/boto/issues/detail?id=540 - self.disabled_message = ("Failed to connect to S3 with " - "credentials. Got error: %s" % e) - self.inited = True - return - - self.s3_conn = s3_conn - - if not self.bucket: - try: - self.bucket = s3_conn.create_bucket(bucket_name) - except boto.exception.S3ResponseError, e: - self.disabled_message = ("Failed to create bucket. " - "Got error: %s" % e) - self.inited = True - return - else: - self.clear_bucket() - - self.disabled = False - self.inited = True self.default_store = 's3' super(TestS3, self).setUp() def tearDown(self): - if not self.disabled: - self.clear_bucket() + teardown_s3(self) super(TestS3, self).tearDown() - def clear_bucket(self): - # It's not possible to simply clear a bucket. You - # need to loop over all the keys and delete them - # all first... - keys = self.bucket.list() - for key in keys: - key.delete() - @skip_if_disabled def test_remote_image(self): """Verify an image added using a 'Location' header can be retrieved""" @@ -162,9 +89,7 @@ class TestS3(test_api.TestApi): # 1. POST /images with public image named Image1 image_data = "*" * FIVE_KB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True'} + headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, @@ -204,11 +129,9 @@ class TestS3(test_api.TestApi): # 4. POST /images using location generated by Image1 image_id2 = utils.generate_uuid() image_data = "*" * FIVE_KB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Id': image_id2, - 'X-Image-Meta-Name': 'Image2', - 'X-Image-Meta-Is-Public': 'True', - 'X-Image-Meta-Location': s3_store_location} + headers = minimal_headers('Image2') + headers['X-Image-Meta-Id'] = image_id2 + headers['X-Image-Meta-Location'] = s3_store_location path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) @@ -245,3 +168,156 @@ class TestS3(test_api.TestApi): http.request(path % args, 'DELETE') self.stop_servers() + + def _do_test_copy_from(self, from_store, get_uri): + """ + Ensure we can copy from an external image in from_store. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # POST /images with public image to be stored in from_store, + # to stand in for the 'external' image + image_data = "*" * FIVE_KB + headers = minimal_headers('external') + headers['X-Image-Meta-Store'] = from_store + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + original_image_id = data['image']['id'] + + copy_from = get_uri(self, original_image_id) + + # POST /images with public image copied from_store (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-Is-Public': 'True', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE original image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + original_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + # GET image again to make sure the existence of the original + # image in from_store is not depended on + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() + + @skip_if_disabled + def test_copy_from_s3(self): + """ + Ensure we can copy from an external image in S3. + """ + self._do_test_copy_from('s3', get_s3_uri) + + @requires(setup_swift, teardown_swift) + @skip_if_disabled + def test_copy_from_swift(self): + """ + Ensure we can copy from an external image in Swift. + """ + self._do_test_copy_from('swift', get_swift_uri) + + @requires(setup_http, teardown_http) + @skip_if_disabled + def test_copy_from_http(self): + """ + Ensure we can copy from an external image in HTTP. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + copy_from = get_http_uri(self, 'foobar') + + # POST /images with public image copied HTTP (to S3) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() diff --git a/glance/tests/functional/test_swift.py b/glance/tests/functional/test_swift.py index 4d6de1ea95..4e36cc4034 100644 --- a/glance/tests/functional/test_swift.py +++ b/glance/tests/functional/test_swift.py @@ -30,18 +30,25 @@ If a connection cannot be established, all the test cases are skipped. """ -import ConfigParser import hashlib import httplib import httplib2 import json -import os from glance.common import crypt import glance.store.swift # Needed to register driver for location from glance.store.location import get_location_from_uri from glance.tests.functional import test_api -from glance.tests.utils import skip_if_disabled +from glance.tests.utils import skip_if_disabled, requires, minimal_headers +from glance.tests.functional.store_utils import (setup_swift, + teardown_swift, + get_swift_uri, + setup_s3, + teardown_s3, + get_s3_uri, + setup_http, + teardown_http, + get_http_uri) FIVE_KB = 5 * 1024 FIVE_MB = 5 * 1024 * 1024 @@ -51,89 +58,17 @@ class TestSwift(test_api.TestApi): """Functional tests for the Swift backend""" - # Test machines can set the GLANCE_TEST_SWIFT_CONF variable - # to override the location of the config file for migration testing - CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_SWIFT_CONF') - def setUp(self): """ Test a connection to an Swift store using the credentials found in the environs or /tests/functional/test_swift.conf, if found. If the connection fails, mark all tests to skip. """ - self.inited = False - self.disabled = True - - if self.inited: + if self.disabled: return - if not self.CONFIG_FILE_PATH: - self.disabled_message = "GLANCE_TEST_SWIFT_CONF environ not set." - self.inited = True - return + setup_swift(self) - if os.path.exists(TestSwift.CONFIG_FILE_PATH): - cp = ConfigParser.RawConfigParser() - try: - cp.read(TestSwift.CONFIG_FILE_PATH) - defaults = cp.defaults() - for key, value in defaults.items(): - self.__dict__[key] = value - except ConfigParser.ParsingError, e: - self.disabled_message = ("Failed to read test_swift.conf " - "file. Got error: %s" % e) - self.inited = True - return - - from swift.common import client as swift_client - - try: - swift_host = self.swift_store_auth_address - if not swift_host.startswith('http'): - swift_host = 'https://' + swift_host - user = self.swift_store_user - key = self.swift_store_key - container_name = self.swift_store_container - except AttributeError, e: - self.disabled_message = ("Failed to find required configuration " - "options for Swift store. " - "Got error: %s" % e) - self.inited = True - return - - self.swift_conn = swift_conn = swift_client.Connection( - authurl=swift_host, user=user, key=key, snet=False, retries=1) - - try: - _resp_headers, containers = swift_conn.get_account() - except Exception, e: - self.disabled_message = ("Failed to get_account from Swift " - "Got error: %s" % e) - self.inited = True - return - - try: - for container in containers: - if container == container_name: - swift_conn.delete_container(container) - except swift_client.ClientException, e: - self.disabled_message = ("Failed to delete container from Swift " - "Got error: %s" % e) - self.inited = True - return - - self.swift_conn = swift_conn - - try: - swift_conn.put_container(container_name) - except swift_client.ClientException, e: - self.disabled_message = ("Failed to create container. " - "Got error: %s" % e) - self.inited = True - return - - self.disabled = False - self.inited = True self.default_store = 'swift' super(TestSwift, self).setUp() @@ -185,9 +120,7 @@ class TestSwift(test_api.TestApi): # POST /images with public image named Image1 # attribute and no custom properties. Verify a 200 OK is returned image_data = "*" * FIVE_MB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True'} + headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, @@ -224,8 +157,8 @@ class TestSwift(test_api.TestApi): 'x-image-meta-name': 'Image1', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'active', - 'x-image-meta-disk_format': '', - 'x-image-meta-container_format': '', + 'x-image-meta-disk_format': 'raw', + 'x-image-meta-container_format': 'ovf', 'x-image-meta-size': str(FIVE_MB) } @@ -335,9 +268,7 @@ class TestSwift(test_api.TestApi): # 1. POST /images with public image named Image1 # attribute and no custom properties. Verify a 200 OK is returned image_data = "*" * FIVE_MB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True'} + headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, @@ -374,8 +305,8 @@ class TestSwift(test_api.TestApi): 'x-image-meta-name': 'Image1', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'active', - 'x-image-meta-disk_format': '', - 'x-image-meta-container_format': '', + 'x-image-meta-disk_format': 'raw', + 'x-image-meta-container_format': 'ovf', 'x-image-meta-size': str(FIVE_MB) } @@ -423,9 +354,7 @@ class TestSwift(test_api.TestApi): # POST /images with public image named Image1 image_data = "*" * FIVE_KB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True'} + headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, @@ -470,10 +399,8 @@ class TestSwift(test_api.TestApi): # POST /images with public image named Image1 without uploading data image_data = "*" * FIVE_KB - headers = {'Content-Type': 'application/octet-stream', - 'X-Image-Meta-Name': 'Image1', - 'X-Image-Meta-Is-Public': 'True', - 'X-Image-Meta-Location': swift_location} + headers = minimal_headers('Image1') + headers['X-Image-Meta-Location'] = swift_location path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) @@ -512,3 +439,156 @@ class TestSwift(test_api.TestApi): self.assertEqual(response.status, 200) self.stop_servers() + + def _do_test_copy_from(self, from_store, get_uri): + """ + Ensure we can copy from an external image in from_store. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + # POST /images with public image to be stored in from_store, + # to stand in for the 'external' image + image_data = "*" * FIVE_KB + headers = minimal_headers('external') + headers['X-Image-Meta-Store'] = from_store + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers, + body=image_data) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + original_image_id = data['image']['id'] + + copy_from = get_uri(self, original_image_id) + + # POST /images with public image copied from_store (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE original image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + original_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + # GET image again to make sure the existence of the original + # image in from_store is not depended on + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() + + @skip_if_disabled + def test_copy_from_swift(self): + """ + Ensure we can copy from an external image in Swift. + """ + self._do_test_copy_from('swift', get_swift_uri) + + @requires(setup_s3, teardown_s3) + @skip_if_disabled + def test_copy_from_s3(self): + """ + Ensure we can copy from an external image in S3. + """ + self._do_test_copy_from('s3', get_s3_uri) + + @requires(setup_http, teardown_http) + @skip_if_disabled + def test_copy_from_http(self): + """ + Ensure we can copy from an external image in HTTP. + """ + self.cleanup() + + self.start_servers(**self.__dict__.copy()) + + api_port = self.api_port + registry_port = self.registry_port + + copy_from = get_http_uri(self, 'foobar') + + # POST /images with public image copied HTTP (to Swift) + headers = {'X-Image-Meta-Name': 'copied', + 'X-Image-Meta-disk_format': 'raw', + 'X-Image-Meta-container_format': 'ovf', + 'X-Image-Meta-Is-Public': 'True', + 'X-Glance-API-Copy-From': copy_from} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201, content) + data = json.loads(content) + + copy_image_id = data['image']['id'] + + # GET image and make sure image content is as expected + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(response['content-length'], str(FIVE_KB)) + + self.assertEqual(content, "*" * FIVE_KB) + self.assertEqual(hashlib.md5(content).hexdigest(), + hashlib.md5("*" * FIVE_KB).hexdigest()) + self.assertEqual(data['image']['size'], FIVE_KB) + self.assertEqual(data['image']['name'], "copied") + + # DELETE copied image + path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, + copy_image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + self.stop_servers() diff --git a/glance/tests/unit/test_s3_store.py b/glance/tests/unit/test_s3_store.py index e68dfee402..b7da8af915 100644 --- a/glance/tests/unit/test_s3_store.py +++ b/glance/tests/unit/test_s3_store.py @@ -273,7 +273,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_s3, new_image_size) = self.store.get(loc) new_image_contents = new_image_s3.getvalue() - new_image_s3_size = new_image_s3.len + new_image_s3_size = len(new_image_s3) self.assertEquals(expected_s3_contents, new_image_contents) self.assertEquals(expected_s3_size, new_image_s3_size) diff --git a/glance/tests/unit/test_swift_store.py b/glance/tests/unit/test_swift_store.py index 905241f5df..303f1c3379 100644 --- a/glance/tests/unit/test_swift_store.py +++ b/glance/tests/unit/test_swift_store.py @@ -261,7 +261,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) @@ -318,7 +318,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) @@ -382,7 +382,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) @@ -430,7 +430,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) @@ -491,7 +491,7 @@ class TestStore(unittest.TestCase): loc = get_location_from_uri(expected_location) (new_image_swift, new_image_size) = self.store.get(loc) new_image_contents = new_image_swift.getvalue() - new_image_swift_size = new_image_swift.len + new_image_swift_size = len(new_image_swift) self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) diff --git a/glance/tests/utils.py b/glance/tests/utils.py index 2fb05a3153..d69bcbcc45 100644 --- a/glance/tests/utils.py +++ b/glance/tests/utils.py @@ -154,6 +154,23 @@ class skip_unless(object): return _skipper +class requires(object): + """Decorator that initiates additional test setup/teardown.""" + def __init__(self, setup, teardown=None): + self.setup = setup + self.teardown = teardown + + def __call__(self, func): + def _runner(*args, **kw): + self.setup(args[0]) + func(*args, **kw) + if self.teardown: + self.teardown(args[0]) + _runner.__name__ = func.__name__ + _runner.__doc__ = func.__doc__ + return _runner + + def skip_if_disabled(func): """Decorator that skips a test if test case is disabled.""" @functools.wraps(func)