diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index 1283ac8def..b5f7d226bc 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -43,7 +43,7 @@ from glance import utils logger = logging.getLogger('glance.api.v1.images') SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format', - 'size_min', 'size_max'] + 'size_min', 'size_max', 'is_public'] SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir') diff --git a/glance/common/exception.py b/glance/common/exception.py index 351c3e276a..87c0b612e8 100644 --- a/glance/common/exception.py +++ b/glance/common/exception.py @@ -54,6 +54,24 @@ class NotFound(Error): pass +class UnknownScheme(Error): + + msg = "Unknown scheme '%s' found in URI" + + def __init__(self, scheme): + msg = self.__class__.msg % scheme + super(UnknownScheme, self).__init__(msg) + + +class BadStoreUri(Error): + + msg = "The Store URI %s was malformed. Reason: %s" + + def __init__(self, uri, reason): + msg = self.__class__.msg % (uri, reason) + super(BadStoreUri, self).__init__(msg) + + class Duplicate(Error): pass diff --git a/glance/registry/db/api.py b/glance/registry/db/api.py index 45d01b9f3c..0f63ff524d 100644 --- a/glance/registry/db/api.py +++ b/glance/registry/db/api.py @@ -24,6 +24,7 @@ Defines interface for DB access import logging from sqlalchemy import asc, create_engine, desc +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import exc from sqlalchemy.orm import joinedload @@ -152,10 +153,10 @@ def image_get_all_pending_delete(context, delete_time=None, limit=None): return query.all() -def image_get_all_public(context, filters=None, marker=None, limit=None, - sort_key='created_at', sort_dir='desc'): - """Get all public images that match zero or more filters. - Get all public images that match zero or more filters. +def image_get_all(context, filters=None, marker=None, limit=None, + sort_key='created_at', sort_dir='desc'): + """ + Get all images that match zero or more filters. :param filters: dict of filter keys and values. If a 'properties' key is present, it is treated as a dict of key/value @@ -171,7 +172,6 @@ def image_get_all_public(context, filters=None, marker=None, limit=None, query = session.query(models.Image).\ options(joinedload(models.Image.properties)).\ filter_by(deleted=_deleted(context)).\ - filter_by(is_public=True).\ filter(models.Image.status != 'killed') sort_dir_func = { @@ -196,7 +196,8 @@ def image_get_all_public(context, filters=None, marker=None, limit=None, query = query.filter(models.Image.properties.any(name=k, value=v)) for (k, v) in filters.items(): - query = query.filter(getattr(models.Image, k) == v) + if v is not None: + query = query.filter(getattr(models.Image, k) == v) if marker != None: # images returned should be created before the image defined by marker @@ -297,7 +298,11 @@ def _image_update(context, values, image_id, purge_props=False): # idiotic. validate_image(image_ref.to_dict()) - image_ref.save(session=session) + try: + image_ref.save(session=session) + except IntegrityError, e: + raise exception.Duplicate("Image ID %s already exists!" + % values['id']) _set_properties_for_image(context, image_ref, properties, purge_props, session) diff --git a/glance/registry/server.py b/glance/registry/server.py index 5ca3f48909..9273d1ccb8 100644 --- a/glance/registry/server.py +++ b/glance/registry/server.py @@ -56,6 +56,16 @@ class Controller(object): self.options = options db_api.configure_db(options) + def _get_images(self, context, **params): + """ + Get images, wrapping in exception if necessary. + """ + try: + return db_api.image_get_all(None, **params) + except exception.NotFound, e: + msg = "Invalid marker. Image could not be found." + raise exc.HTTPBadRequest(explanation=msg) + def index(self, req): """ Return a basic filtered list of public, non-deleted images @@ -77,11 +87,7 @@ class Controller(object): } """ params = self._get_query_params(req) - try: - images = db_api.image_get_all_public(None, **params) - except exception.NotFound, e: - msg = "Invalid marker. Image could not be found." - raise exc.HTTPBadRequest(explanation=msg) + images = self._get_images(None, **params) results = [] for image in images: @@ -104,12 +110,8 @@ class Controller(object): all image model fields. """ params = self._get_query_params(req) - try: - images = db_api.image_get_all_public(None, **params) - except exception.NotFound, e: - msg = "Invalid marker. Image could not be found." - raise exc.HTTPBadRequest(explanation=msg) + images = self._get_images(None, **params) image_dicts = [make_image_dict(i) for i in images] return dict(images=image_dicts) @@ -144,6 +146,7 @@ class Controller(object): filters = {} properties = {} + filters['is_public'] = self._get_is_public(req) for param in req.str_params: if param in SUPPORTED_FILTERS: filters[param] = req.str_params.get(param) @@ -199,6 +202,24 @@ class Controller(object): raise exc.HTTPBadRequest(explanation=msg) return sort_dir + def _get_is_public(self, req): + """Parse is_public into something usable.""" + is_public = req.str_params.get('is_public', None) + + if is_public is None: + # NOTE(vish): This preserves the default value of showing only + # public images. + return True + is_public = is_public.lower() + if is_public == 'none': + return None + elif is_public == 'true' or is_public == '1': + return True + elif is_public == 'false' or is_public == '0': + return False + else: + raise exc.HTTPBadRequest("is_public must be None, True, or False") + def show(self, req, id): """Return data about the given image id.""" try: diff --git a/glance/store/__init__.py b/glance/store/__init__.py index c00380bc6e..efdcfe4e2e 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -22,6 +22,7 @@ import urlparse from glance import registry from glance.common import config, exception +from glance.store import location logger = logging.getLogger('glance.store') @@ -79,72 +80,32 @@ def get_backend_class(backend): def get_from_backend(uri, **kwargs): """Yields chunks of data from backend specified by uri""" - parsed_uri = urlparse.urlparse(uri) - scheme = parsed_uri.scheme + loc = location.get_location_from_uri(uri) + backend_class = get_backend_class(loc.store_name) - backend_class = get_backend_class(scheme) - - return backend_class.get(parsed_uri, **kwargs) + return backend_class.get(loc, **kwargs) def delete_from_backend(uri, **kwargs): """Removes chunks of data from backend specified by uri""" - parsed_uri = urlparse.urlparse(uri) - scheme = parsed_uri.scheme - - backend_class = get_backend_class(scheme) + loc = location.get_location_from_uri(uri) + backend_class = get_backend_class(loc.store_name) if hasattr(backend_class, 'delete'): - return backend_class.delete(parsed_uri, **kwargs) + return backend_class.delete(loc, **kwargs) -def get_store_from_location(location): +def get_store_from_location(uri): """ Given a location (assumed to be a URL), attempt to determine the store from the location. We use here a simple guess that the scheme of the parsed URL is the store... - :param location: Location to check for the store + :param uri: Location to check for the store """ - loc_pieces = urlparse.urlparse(location) - return loc_pieces.scheme - - -def parse_uri_tokens(parsed_uri, example_url): - """ - Given a URI and an example_url, attempt to parse the uri to assemble an - authurl. This method returns the user, key, authurl, referenced container, - and the object we're looking for in that container. - - Parsing the uri is three phases: - 1) urlparse to split the tokens - 2) use RE to split on @ and / - 3) reassemble authurl - """ - path = parsed_uri.path.lstrip('//') - netloc = parsed_uri.netloc - - try: - try: - creds, netloc = netloc.split('@') - except ValueError: - # Python 2.6.1 compat - # see lp659445 and Python issue7904 - creds, path = path.split('@') - user, key = creds.split(':') - path_parts = path.split('/') - obj = path_parts.pop() - container = path_parts.pop() - except (ValueError, IndexError): - raise BackendException( - "Expected four values to unpack in: %s:%s. " - "Should have received something like: %s." - % (parsed_uri.scheme, parsed_uri.path, example_url)) - - authurl = "https://%s" % '/'.join(path_parts) - - return user, key, authurl, container, obj + loc = location.get_location_from_uri(uri) + return loc.store_name def schedule_delete_from_backend(uri, options, id, **kwargs): diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py index 518d44d0b0..8041049377 100644 --- a/glance/store/filesystem.py +++ b/glance/store/filesystem.py @@ -26,9 +26,39 @@ import urlparse from glance.common import exception import glance.store +import glance.store.location logger = logging.getLogger('glance.store.filesystem') +glance.store.location.add_scheme_map({'file': 'filesystem'}) + + +class StoreLocation(glance.store.location.StoreLocation): + + """Class describing a Filesystem URI""" + + def process_specs(self): + self.scheme = self.specs.get('scheme', 'file') + self.path = self.specs.get('path') + + def get_uri(self): + return "file://%s" % self.path + + def parse_uri(self, uri): + """ + Parse URLs. This method fixes an issue where credentials specified + in the URL are interpreted differently in Python 2.6.1+ than prior + versions of Python. + """ + pieces = urlparse.urlparse(uri) + assert pieces.scheme == 'file' + self.scheme = pieces.scheme + path = (pieces.netloc + pieces.path).strip() + if path == '': + reason = "No path specified" + raise exception.BadStoreUri(uri, reason) + self.path = path + class ChunkedFile(object): @@ -64,13 +94,19 @@ class ChunkedFile(object): class FilesystemBackend(glance.store.Backend): @classmethod - def get(cls, parsed_uri, expected_size=None, options=None): + def get(cls, location, expected_size=None, options=None): """ - Filesystem-based backend + Takes a `glance.store.location.Location` object that indicates + where to find the image file, and returns a generator to use in + reading the image file. - file:///path/to/file.tar.gz.0 + :location `glance.store.location.Location` object, supplied + from glance.store.location.get_location_from_uri() + + :raises NotFound if file does not exist """ - filepath = parsed_uri.path + loc = location.store_location + filepath = loc.path if not os.path.exists(filepath): raise exception.NotFound("Image file %s not found" % filepath) else: @@ -79,17 +115,19 @@ class FilesystemBackend(glance.store.Backend): return ChunkedFile(filepath) @classmethod - def delete(cls, parsed_uri): + def delete(cls, location): """ - Removes a file from the filesystem backend. + Takes a `glance.store.location.Location` object that indicates + where to find the image file to delete - :param parsed_uri: Parsed pieces of URI in form of:: - file:///path/to/filename.ext + :location `glance.store.location.Location` object, supplied + from glance.store.location.get_location_from_uri() :raises NotFound if file does not exist :raises NotAuthorized if cannot delete because of permissions """ - fn = parsed_uri.path + loc = location.store_location + fn = loc.path if os.path.exists(fn): try: logger.debug("Deleting image at %s", fn) diff --git a/glance/store/http.py b/glance/store/http.py index 9ddb9ca746..beaa4641c0 100644 --- a/glance/store/http.py +++ b/glance/store/http.py @@ -16,31 +16,104 @@ # under the License. import httplib +import urlparse +from glance.common import exception import glance.store +import glance.store.location + +glance.store.location.add_scheme_map({'http': 'http', + 'https': 'http'}) + + +class StoreLocation(glance.store.location.StoreLocation): + + """Class describing an HTTP(S) URI""" + + def process_specs(self): + self.scheme = self.specs.get('scheme', 'http') + self.netloc = self.specs['netloc'] + self.user = self.specs.get('user') + self.password = self.specs.get('password') + self.path = self.specs.get('path') + + def _get_credstring(self): + if self.user: + return '%s:%s@' % (self.user, self.password) + return '' + + def get_uri(self): + return "%s://%s%s%s" % ( + self.scheme, + self._get_credstring(), + self.netloc, + self.path) + + def parse_uri(self, uri): + """ + Parse URLs. This method fixes an issue where credentials specified + in the URL are interpreted differently in Python 2.6.1+ than prior + versions of Python. + """ + pieces = urlparse.urlparse(uri) + assert pieces.scheme in ('https', 'http') + self.scheme = pieces.scheme + netloc = pieces.netloc + path = pieces.path + try: + if '@' in netloc: + creds, netloc = netloc.split('@') + else: + creds = None + except ValueError: + # Python 2.6.1 compat + # see lp659445 and Python issue7904 + if '@' in path: + creds, path = path.split('@') + else: + creds = None + if creds: + try: + self.user, self.password = creds.split(':') + except ValueError: + reason = ("Credentials '%s' not well-formatted." + % "".join(creds)) + raise exception.BadStoreUri(uri, reason) + else: + self.user = None + if netloc == '': + reason = "No address specified in HTTP URL" + raise exception.BadStoreUri(uri, reason) + self.netloc = netloc + self.path = path class HTTPBackend(glance.store.Backend): """ An implementation of the HTTP Backend Adapter """ @classmethod - def get(cls, parsed_uri, expected_size, options=None, conn_class=None): + def get(cls, location, expected_size, options=None, conn_class=None): """ - Takes a parsed uri for an HTTP resource, fetches it, and - yields the data. + Takes a `glance.store.location.Location` object that indicates + where to find the image file, and returns a generator from Swift + provided by Swift client's get_object() method. + + :location `glance.store.location.Location` object, supplied + from glance.store.location.get_location_from_uri() """ + loc = location.store_location if conn_class: pass # use the conn_class passed in - elif parsed_uri.scheme == "http": + elif loc.scheme == "http": conn_class = httplib.HTTPConnection - elif parsed_uri.scheme == "https": + elif loc.scheme == "https": conn_class = httplib.HTTPSConnection else: raise glance.store.BackendException( "scheme '%s' not supported for HTTPBackend") - conn = conn_class(parsed_uri.netloc) - conn.request("GET", parsed_uri.path, "", {}) + conn = conn_class(loc.netloc) + conn.request("GET", loc.path, "", {}) try: return glance.store._file_iter(conn.getresponse(), cls.CHUNKSIZE) diff --git a/glance/store/location.py b/glance/store/location.py new file mode 100644 index 0000000000..ffb3acd2b7 --- /dev/null +++ b/glance/store/location.py @@ -0,0 +1,182 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +A class that describes the location of an image in Glance. + +In Glance, an image can either be **stored** in Glance, or it can be +**registered** in Glance but actually be stored somewhere else. + +We needed a class that could support the various ways that Glance +describes where exactly an image is stored. + +An image in Glance has two location properties: the image URI +and the image storage URI. + +The image URI is essentially the permalink identifier for the image. +It is displayed in the output of various Glance API calls and, +while read-only, is entirely user-facing. It shall **not** contain any +security credential information at all. The Glance image URI shall +be the host:port of that Glance API server along with /images/. + +The Glance storage URI is an internal URI structure that Glance +uses to maintain critical information about how to access the images +that it stores in its storage backends. It **does contain** security +credentials and is **not** user-facing. +""" + +import logging +import urlparse + +from glance.common import exception +from glance.common import utils + +logger = logging.getLogger('glance.store.location') + +SCHEME_TO_STORE_MAP = {} + + +def get_location_from_uri(uri): + """ + Given a URI, return a Location object that has had an appropriate + store parse the URI. + + :param uri: A URI that could come from the end-user in the Location + attribute/header + + Example URIs: + https://user:pass@example.com:80/images/some-id + http://images.oracle.com/123456 + swift://user:account:pass@authurl.com/container/obj-id + swift+http://user:account:pass@authurl.com/container/obj-id + s3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id + s3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id + file:///var/lib/glance/images/1 + """ + # Add known stores to mapping... this gets past circular import + # issues. There's a better way to do this, but that's for another + # patch... + # TODO(jaypipes) Clear up these imports in refactor-stores blueprint + import glance.store.filesystem + import glance.store.http + import glance.store.s3 + import glance.store.swift + pieces = urlparse.urlparse(uri) + if pieces.scheme not in SCHEME_TO_STORE_MAP.keys(): + raise exception.UnknownScheme(pieces.scheme) + loc = Location(pieces.scheme, uri=uri) + return loc + + +def add_scheme_map(scheme_map): + """ + Given a mapping of 'scheme' to store_name, adds the mapping to the + known list of schemes. + + Each store should call this method and let Glance know about which + schemes to map to a store name. + """ + SCHEME_TO_STORE_MAP.update(scheme_map) + + +class Location(object): + + """ + Class describing the location of an image that Glance knows about + """ + + def __init__(self, store_name, uri=None, image_id=None, store_specs=None): + """ + Create a new Location object. + + :param store_name: The string identifier of the storage backend + :param image_id: The identifier of the image in whatever storage + backend is used. + :param uri: Optional URI to construct location from + :param store_specs: Dictionary of information about the location + of the image that is dependent on the backend + store + """ + self.store_name = store_name + self.image_id = image_id + self.store_specs = store_specs or {} + self.store_location = self._get_store_location() + if uri: + self.store_location.parse_uri(uri) + + def _get_store_location(self): + """ + We find the store module and then grab an instance of the store's + StoreLocation class which handles store-specific location information + """ + try: + cls = utils.import_class('glance.store.%s.StoreLocation' + % SCHEME_TO_STORE_MAP[self.store_name]) + return cls(self.store_specs) + except exception.NotFound: + logger.error("Unable to find StoreLocation class in store %s", + self.store_name) + return None + + def get_store_uri(self): + """ + Returns the Glance image URI, which is the host:port of the API server + along with /images/ + """ + return self.store_location.get_uri() + + def get_uri(self): + return None + + +class StoreLocation(object): + + """ + Base class that must be implemented by each store + """ + + def __init__(self, store_specs): + self.specs = store_specs + if self.specs: + self.process_specs() + + def process_specs(self): + """ + Subclasses should implement any processing of the self.specs collection + such as storing credentials and possibly establishing connections. + """ + pass + + def get_uri(self): + """ + Subclasses should implement a method that returns an internal URI that, + when supplied to the StoreLocation instance, can be interpreted by the + StoreLocation's parse_uri() method. The URI returned from this method + shall never be public and only used internally within Glance, so it is + fine to encode credentials in this URI. + """ + raise NotImplementedError("StoreLocation subclass must implement " + "get_uri()") + + def parse_uri(self, uri): + """ + Subclasses should implement a method that accepts a string URI and + sets appropriate internal fields such that a call to get_uri() will + return a proper internal URI + """ + raise NotImplementedError("StoreLocation subclass must implement " + "parse_uri()") diff --git a/glance/store/s3.py b/glance/store/s3.py index f235279bd4..a25a2e46ae 100644 --- a/glance/store/s3.py +++ b/glance/store/s3.py @@ -17,7 +17,101 @@ """The s3 backend adapter""" +import urlparse + +from glance.common import exception import glance.store +import glance.store.location + +glance.store.location.add_scheme_map({'s3': 's3', + 's3+http': 's3', + 's3+https': 's3'}) + + +class StoreLocation(glance.store.location.StoreLocation): + + """ + Class describing an S3 URI. An S3 URI can look like any of + the following: + + s3://accesskey:secretkey@s3service.com/bucket/key-id + s3+http://accesskey:secretkey@s3service.com/bucket/key-id + s3+https://accesskey:secretkey@s3service.com/bucket/key-id + + The s3+https:// URIs indicate there is an HTTPS s3service URL + """ + + def process_specs(self): + self.scheme = self.specs.get('scheme', 's3') + self.accesskey = self.specs.get('accesskey') + self.secretkey = self.specs.get('secretkey') + self.s3serviceurl = self.specs.get('s3serviceurl') + self.bucket = self.specs.get('bucket') + self.key = self.specs.get('key') + + def _get_credstring(self): + if self.accesskey: + return '%s:%s@' % (self.accesskey, self.secretkey) + return '' + + def get_uri(self): + return "%s://%s%s/%s/%s" % ( + self.scheme, + self._get_credstring(), + self.s3serviceurl, + self.bucket, + self.key) + + def parse_uri(self, uri): + """ + Parse URLs. This method fixes an issue where credentials specified + in the URL are interpreted differently in Python 2.6.1+ than prior + versions of Python. + + Note that an Amazon AWS secret key can contain the forward slash, + which is entirely retarded, and breaks urlparse miserably. + This function works around that issue. + """ + pieces = urlparse.urlparse(uri) + assert pieces.scheme in ('s3', 's3+http', 's3+https') + self.scheme = pieces.scheme + path = pieces.path.strip('/') + netloc = pieces.netloc.strip('/') + entire_path = (netloc + '/' + path).strip('/') + + if '@' in uri: + creds, path = entire_path.split('@') + cred_parts = creds.split(':') + + try: + access_key = cred_parts[0] + secret_key = cred_parts[1] + # NOTE(jaypipes): Need to encode to UTF-8 here because of a + # bug in the HMAC library that boto uses. + # See: http://bugs.python.org/issue5285 + # See: http://trac.edgewall.org/ticket/8083 + access_key = access_key.encode('utf-8') + secret_key = secret_key.encode('utf-8') + self.accesskey = access_key + self.secretkey = secret_key + except IndexError: + reason = "Badly formed S3 credentials %s" % creds + raise exception.BadStoreUri(uri, reason) + else: + self.accesskey = None + path = entire_path + try: + path_parts = path.split('/') + self.key = path_parts.pop() + self.bucket = path_parts.pop() + if len(path_parts) > 0: + self.s3serviceurl = '/'.join(path_parts) + else: + reason = "Badly formed S3 URI. Missing s3 service URL." + raise exception.BadStoreUri(uri, reason) + except IndexError: + reason = "Badly formed S3 URI" + raise exception.BadStoreUri(uri, reason) class S3Backend(glance.store.Backend): @@ -26,29 +120,30 @@ class S3Backend(glance.store.Backend): EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0" @classmethod - def get(cls, parsed_uri, expected_size, conn_class=None): - """ - Takes a parsed_uri in the format of: - s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects - to s3 and downloads the file. Returns the generator resp_body provided - by get_object. + def get(cls, location, expected_size, conn_class=None): """ + Takes a `glance.store.location.Location` object that indicates + where to find the image file, and returns a generator from S3 + provided by S3's key object + :location `glance.store.location.Location` object, supplied + from glance.store.location.get_location_from_uri() + """ if conn_class: pass else: import boto.s3.connection conn_class = boto.s3.connection.S3Connection - (access_key, secret_key, host, bucket, obj) = \ - cls._parse_s3_tokens(parsed_uri) + loc = location.store_location # Close the connection when we're through. - with conn_class(access_key, secret_key, host=host) as s3_conn: - bucket = cls._get_bucket(s3_conn, bucket) + with conn_class(loc.accesskey, loc.secretkey, + host=loc.s3serviceurl) as s3_conn: + bucket = cls._get_bucket(s3_conn, loc.bucket) # Close the key when we're through. - with cls._get_key(bucket, obj) as key: + with cls._get_key(bucket, loc.obj) as key: if not key.size == expected_size: raise glance.store.BackendException( "Expected %s bytes, got %s" % @@ -59,28 +154,28 @@ class S3Backend(glance.store.Backend): yield chunk @classmethod - def delete(cls, parsed_uri, conn_class=None): - """ - Takes a parsed_uri in the format of: - s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects - to s3 and deletes the file. Returns whatever boto.s3.key.Key.delete() - returns. + def delete(cls, location, conn_class=None): """ + Takes a `glance.store.location.Location` object that indicates + where to find the image file to delete + :location `glance.store.location.Location` object, supplied + from glance.store.location.get_location_from_uri() + """ if conn_class: pass else: conn_class = boto.s3.connection.S3Connection - (access_key, secret_key, host, bucket, obj) = \ - cls._parse_s3_tokens(parsed_uri) + loc = location.store_location # Close the connection when we're through. - with conn_class(access_key, secret_key, host=host) as s3_conn: - bucket = cls._get_bucket(s3_conn, bucket) + with conn_class(loc.accesskey, loc.secretkey, + host=loc.s3serviceurl) as s3_conn: + bucket = cls._get_bucket(s3_conn, loc.bucket) # Close the key when we're through. - with cls._get_key(bucket, obj) as key: + with cls._get_key(bucket, loc.obj) as key: return key.delete() @classmethod @@ -102,8 +197,3 @@ class S3Backend(glance.store.Backend): if not key: raise glance.store.BackendException("Could not get key: %s" % key) return key - - @classmethod - def _parse_s3_tokens(cls, parsed_uri): - """Parse tokens from the parsed_uri""" - return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL) diff --git a/glance/store/swift.py b/glance/store/swift.py index c972eed8c4..5be18c5db7 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -21,15 +21,114 @@ from __future__ import absolute_import import httplib import logging +import urlparse from glance.common import config from glance.common import exception import glance.store +import glance.store.location DEFAULT_SWIFT_CONTAINER = 'glance' logger = logging.getLogger('glance.store.swift') +glance.store.location.add_scheme_map({'swift': 'swift', + 'swift+http': 'swift', + 'swift+https': 'swift'}) + + +class StoreLocation(glance.store.location.StoreLocation): + + """ + Class describing a Swift URI. A Swift URI can look like any of + the following: + + swift://user:pass@authurl.com/container/obj-id + swift+http://user:pass@authurl.com/container/obj-id + swift+https://user:pass@authurl.com/container/obj-id + + The swift+https:// URIs indicate there is an HTTPS authentication URL + """ + + def process_specs(self): + self.scheme = self.specs.get('scheme', 'swift+https') + self.user = self.specs.get('user') + self.key = self.specs.get('key') + self.authurl = self.specs.get('authurl') + self.container = self.specs.get('container') + self.obj = self.specs.get('obj') + + def _get_credstring(self): + if self.user: + return '%s:%s@' % (self.user, self.key) + return '' + + def get_uri(self): + return "%s://%s%s/%s/%s" % ( + self.scheme, + self._get_credstring(), + self.authurl, + self.container, + self.obj) + + def parse_uri(self, uri): + """ + Parse URLs. This method fixes an issue where credentials specified + in the URL are interpreted differently in Python 2.6.1+ than prior + versions of Python. It also deals with the peculiarity that new-style + Swift URIs have where a username can contain a ':', like so: + + swift://account:user:pass@authurl.com/container/obj + """ + pieces = urlparse.urlparse(uri) + assert pieces.scheme in ('swift', 'swift+http', 'swift+https') + self.scheme = pieces.scheme + netloc = pieces.netloc + path = pieces.path.lstrip('/') + if netloc != '': + # > Python 2.6.1 + if '@' in netloc: + creds, netloc = netloc.split('@') + else: + creds = None + else: + # Python 2.6.1 compat + # see lp659445 and Python issue7904 + if '@' in path: + creds, path = path.split('@') + else: + creds = None + netloc = path[0:path.find('/')].strip('/') + path = path[path.find('/'):].strip('/') + if creds: + cred_parts = creds.split(':') + + # User can be account:user, in which case cred_parts[0:2] will be + # the account and user. Combine them into a single username of + # account:user + if len(cred_parts) == 1: + reason = "Badly formed credentials '%s' in Swift URI" % creds + raise exception.BadStoreUri(uri, reason) + elif len(cred_parts) == 3: + user = ':'.join(cred_parts[0:2]) + else: + user = cred_parts[0] + key = cred_parts[-1] + self.user = user + self.key = key + else: + self.user = None + path_parts = path.split('/') + try: + self.obj = path_parts.pop() + self.container = path_parts.pop() + self.authurl = netloc + if len(path_parts) > 0: + self.authurl = netloc + '/' + '/'.join(path_parts).strip('/') + except IndexError: + reason = "Badly formed Swift URI" + raise exception.BadStoreUri(uri, reason) + class SwiftBackend(glance.store.Backend): """An implementation of the swift backend adapter.""" @@ -39,31 +138,33 @@ class SwiftBackend(glance.store.Backend): CHUNKSIZE = 65536 @classmethod - def get(cls, parsed_uri, expected_size=None, options=None): + def get(cls, location, expected_size=None, options=None): """ - Takes a parsed_uri in the format of: - swift://user:password@auth_url/container/file.gz.0, connects to the - swift instance at auth_url and downloads the file. Returns the - generator resp_body provided by get_object. + Takes a `glance.store.location.Location` object that indicates + where to find the image file, and returns a generator from Swift + provided by Swift client's get_object() method. + + :location `glance.store.location.Location` object, supplied + from glance.store.location.get_location_from_uri() """ from swift.common import client as swift_client - (user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri) # TODO(sirp): snet=False for now, however, if the instance of # swift we're talking to is within our same region, we should set # snet=True + loc = location.store_location swift_conn = swift_client.Connection( - authurl=authurl, user=user, key=key, snet=False) + authurl=loc.authurl, user=loc.user, key=loc.key, snet=False) try: (resp_headers, resp_body) = swift_conn.get_object( - container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE) + container=loc.container, obj=loc.obj, + resp_chunk_size=cls.CHUNKSIZE) except swift_client.ClientException, e: if e.http_status == httplib.NOT_FOUND: - location = format_swift_location(user, key, authurl, - container, obj) + uri = location.get_store_uri() raise exception.NotFound("Swift could not find image at " - "location %(location)s" % locals()) + "uri %(uri)s" % locals()) if expected_size: obj_size = int(resp_headers['content-length']) @@ -98,6 +199,10 @@ class SwiftBackend(glance.store.Backend): = ``swift_store_container`` = The id of the image being added + :note Swift auth URLs by default use HTTPS. To specify an HTTP + auth URL, you can specify http://someurl.com for the + swift_store_auth_address config option + :param id: The opaque image identifier :param data: The image data to write, as a file-like object :param options: Conf mapping @@ -119,9 +224,14 @@ class SwiftBackend(glance.store.Backend): user = cls._option_get(options, 'swift_store_user') key = cls._option_get(options, 'swift_store_key') - full_auth_address = auth_address - if not full_auth_address.startswith('http'): - full_auth_address = 'https://' + full_auth_address + scheme = 'swift+https' + if auth_address.startswith('http://'): + scheme = 'swift+http' + full_auth_address = auth_address + elif auth_address.startswith('https://'): + full_auth_address = auth_address + else: + full_auth_address = 'https://' + auth_address # Defaults https swift_conn = swift_client.Connection( authurl=full_auth_address, user=user, key=key, snet=False) @@ -133,8 +243,13 @@ class SwiftBackend(glance.store.Backend): create_container_if_missing(container, swift_conn, options) obj_name = str(id) - location = format_swift_location(user, key, auth_address, - container, obj_name) + location = StoreLocation({'scheme': scheme, + 'container': container, + 'obj': obj_name, + 'authurl': auth_address, + 'user': user, + 'key': key}) + try: obj_etag = swift_conn.put_object(container, obj_name, data) @@ -152,7 +267,7 @@ class SwiftBackend(glance.store.Backend): # header keys are lowercased by Swift if 'content-length' in resp_headers: size = int(resp_headers['content-length']) - return (location, size, obj_etag) + return (location.get_uri(), size, obj_etag) except swift_client.ClientException, e: if e.http_status == httplib.CONFLICT: raise exception.Duplicate("Swift already has an image at " @@ -162,89 +277,34 @@ class SwiftBackend(glance.store.Backend): raise glance.store.BackendException(msg) @classmethod - def delete(cls, parsed_uri): + def delete(cls, location): """ - Deletes the swift object(s) at the parsed_uri location + Takes a `glance.store.location.Location` object that indicates + where to find the image file to delete + + :location `glance.store.location.Location` object, supplied + from glance.store.location.get_location_from_uri() """ from swift.common import client as swift_client - (user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri) # TODO(sirp): snet=False for now, however, if the instance of # swift we're talking to is within our same region, we should set # snet=True + loc = location.store_location swift_conn = swift_client.Connection( - authurl=authurl, user=user, key=key, snet=False) + authurl=loc.authurl, user=loc.user, key=loc.key, snet=False) try: - swift_conn.delete_object(container, obj) + swift_conn.delete_object(loc.container, loc.obj) except swift_client.ClientException, e: if e.http_status == httplib.NOT_FOUND: - location = format_swift_location(user, key, authurl, - container, obj) + uri = location.get_store_uri() raise exception.NotFound("Swift could not find image at " - "location %(location)s" % locals()) + "uri %(uri)s" % locals()) else: raise -def parse_swift_tokens(parsed_uri): - """ - Return the various tokens used by Swift. - - :param parsed_uri: The pieces of a URI returned by urlparse - :retval A tuple of (user, key, auth_address, container, obj_name) - """ - path = parsed_uri.path.lstrip('//') - netloc = parsed_uri.netloc - - try: - try: - creds, netloc = netloc.split('@') - path = '/'.join([netloc, path]) - except ValueError: - # Python 2.6.1 compat - # see lp659445 and Python issue7904 - creds, path = path.split('@') - - cred_parts = creds.split(':') - - # User can be account:user, in which case cred_parts[0:2] will be - # the account and user. Combine them into a single username of - # account:user - if len(cred_parts) == 3: - user = ':'.join(cred_parts[0:2]) - else: - user = cred_parts[0] - key = cred_parts[-1] - path_parts = path.split('/') - obj = path_parts.pop() - container = path_parts.pop() - except (ValueError, IndexError): - raise glance.store.BackendException( - "Expected four values to unpack in: swift:%s. " - "Should have received something like: %s." - % (parsed_uri.path, SwiftBackend.EXAMPLE_URL)) - - authurl = "https://%s" % '/'.join(path_parts) - - return user, key, authurl, container, obj - - -def format_swift_location(user, key, auth_address, container, obj_name): - """ - Returns the swift URI in the format: - swift://:@// - - :param user: The swift user to authenticate with - :param key: The auth key for the authenticating user - :param auth_address: The base URL for the authentication service - :param container: The name of the container - :param obj_name: The name of the object - """ - return "swift://%(user)s:%(key)s@%(auth_address)s/"\ - "%(container)s/%(obj_name)s" % locals() - - def create_container_if_missing(container, swift_conn, options): """ Creates a missing container in Swift if the diff --git a/tests/functional/test_curl_api.py b/tests/functional/test_curl_api.py index 1df2ca3e33..924754376d 100644 --- a/tests/functional/test_curl_api.py +++ b/tests/functional/test_curl_api.py @@ -1227,3 +1227,5 @@ class TestCurlApi(functional.FunctionalTest): self.assertEqual(images[0]['id'], 1) self.assertEqual(images[1]['id'], 3) self.assertEqual(images[2]['id'], 2) + + self.stop_servers() diff --git a/tests/functional/test_httplib2_api.py b/tests/functional/test_httplib2_api.py index 49570c986d..2d40221655 100644 --- a/tests/functional/test_httplib2_api.py +++ b/tests/functional/test_httplib2_api.py @@ -21,6 +21,7 @@ import hashlib import httplib2 import json import os +import tempfile from tests import functional from tests.utils import execute @@ -590,3 +591,488 @@ class TestApiHttplib2(functional.FunctionalTest): self.assertEqual(response['x-image-meta-is_public'], 'True') self.stop_servers() + + def test_traceback_not_consumed(self): + """ + A test that errors coming from the POST API do not + get consumed and print the actual error message, and + not something like <traceback object at 0x1918d40> + + :see https://bugs.launchpad.net/glance/+bug/755912 + """ + self.cleanup() + self.start_servers() + + # POST /images with binary data, but not setting + # Content-Type to application/octet-stream, verify a + # 400 returned and that the error is readable. + with tempfile.NamedTemporaryFile() as test_data_file: + test_data_file.write("XXX") + test_data_file.flush() + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', + body=test_data_file.name) + self.assertEqual(response.status, 400) + expected = "Content-Type must be application/octet-stream" + self.assertTrue(expected in content, + "Could not find '%s' in '%s'" % (expected, content)) + + self.stop_servers() + + def test_filtered_images(self): + """ + Set up four test images and ensure each query param filter works + """ + self.cleanup() + self.start_servers() + + # 0. GET /images + # Verify no public images + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(content, '{"images": []}') + + # 1. POST /images with three public images, and one private image + # with various attributes + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image1', + 'X-Image-Meta-Status': 'active', + 'X-Image-Meta-Container-Format': 'ovf', + 'X-Image-Meta-Disk-Format': 'vdi', + 'X-Image-Meta-Size': '19', + 'X-Image-Meta-Is-Public': 'True', + 'X-Image-Meta-Property-pants': 'are on'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + data = json.loads(content) + self.assertEqual(data['image']['properties']['pants'], "are on") + self.assertEqual(data['image']['is_public'], True) + + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'My Image!', + 'X-Image-Meta-Status': 'active', + 'X-Image-Meta-Container-Format': 'ovf', + 'X-Image-Meta-Disk-Format': 'vhd', + 'X-Image-Meta-Size': '20', + 'X-Image-Meta-Is-Public': 'True', + 'X-Image-Meta-Property-pants': 'are on'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + data = json.loads(content) + self.assertEqual(data['image']['properties']['pants'], "are on") + self.assertEqual(data['image']['is_public'], True) + + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'My Image!', + 'X-Image-Meta-Status': 'saving', + 'X-Image-Meta-Container-Format': 'ami', + 'X-Image-Meta-Disk-Format': 'ami', + 'X-Image-Meta-Size': '21', + 'X-Image-Meta-Is-Public': 'True', + 'X-Image-Meta-Property-pants': 'are off'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + data = json.loads(content) + self.assertEqual(data['image']['properties']['pants'], "are off") + self.assertEqual(data['image']['is_public'], True) + + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'My Private Image', + 'X-Image-Meta-Status': 'active', + 'X-Image-Meta-Container-Format': 'ami', + 'X-Image-Meta-Disk-Format': 'ami', + 'X-Image-Meta-Size': '22', + 'X-Image-Meta-Is-Public': 'False'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + data = json.loads(content) + self.assertEqual(data['image']['is_public'], False) + + # 2. GET /images + # Verify three public images + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 3) + + # 3. GET /images with name filter + # Verify correct images returned with name + params = "name=My%20Image!" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 2) + for image in data['images']: + self.assertEqual(image['name'], "My Image!") + + # 4. GET /images with status filter + # Verify correct images returned with status + params = "status=queued" + path = "http://%s:%d/v1/images/detail?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 3) + for image in data['images']: + self.assertEqual(image['status'], "queued") + + params = "status=active" + path = "http://%s:%d/v1/images/detail?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 0) + + # 5. GET /images with container_format filter + # Verify correct images returned with container_format + params = "container_format=ovf" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 2) + for image in data['images']: + self.assertEqual(image['container_format'], "ovf") + + # 6. GET /images with disk_format filter + # Verify correct images returned with disk_format + params = "disk_format=vdi" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 1) + for image in data['images']: + self.assertEqual(image['disk_format'], "vdi") + + # 7. GET /images with size_max filter + # Verify correct images returned with size <= expected + params = "size_max=20" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 2) + for image in data['images']: + self.assertTrue(image['size'] <= 20) + + # 8. GET /images with size_min filter + # Verify correct images returned with size >= expected + params = "size_min=20" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 2) + for image in data['images']: + self.assertTrue(image['size'] >= 20) + + # 9. Get /images with is_public=None filter + # Verify correct images returned with property + # Bug lp:803656 Support is_public in filtering + params = "is_public=None" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 4) + + # 10. Get /images with is_public=False filter + # Verify correct images returned with property + # Bug lp:803656 Support is_public in filtering + params = "is_public=False" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 1) + for image in data['images']: + self.assertEqual(image['name'], "My Private Image") + + # 11. Get /images with is_public=True filter + # Verify correct images returned with property + # Bug lp:803656 Support is_public in filtering + params = "is_public=True" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 3) + for image in data['images']: + self.assertNotEqual(image['name'], "My Private Image") + + # 12. GET /images with property filter + # Verify correct images returned with property + params = "property-pants=are%20on" + path = "http://%s:%d/v1/images/detail?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 2) + for image in data['images']: + self.assertEqual(image['properties']['pants'], "are on") + + # 13. GET /images with property filter and name filter + # Verify correct images returned with property and name + # Make sure you quote the url when using more than one param! + params = "name=My%20Image!&property-pants=are%20on" + path = "http://%s:%d/v1/images/detail?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 1) + for image in data['images']: + self.assertEqual(image['properties']['pants'], "are on") + self.assertEqual(image['name'], "My Image!") + + self.stop_servers() + + def test_limited_images(self): + """ + Ensure marker and limit query params work + """ + self.cleanup() + self.start_servers() + + # 0. GET /images + # Verify no public images + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(content, '{"images": []}') + + # 1. POST /images with three public images with various attributes + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image1', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image2', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image3', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + + # 2. GET /images with limit of 2 + # Verify only two images were returned + params = "limit=2" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 2) + self.assertEqual(data['images'][0]['id'], 3) + self.assertEqual(data['images'][1]['id'], 2) + + # 3. GET /images with marker + # Verify only two images were returned + params = "marker=3" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 2) + self.assertEqual(data['images'][0]['id'], 2) + self.assertEqual(data['images'][1]['id'], 1) + + # 4. GET /images with marker and limit + # Verify only one image was returned with the correct id + params = "limit=1&marker=2" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 1) + self.assertEqual(data['images'][0]['id'], 1) + + # 5. GET /images/detail with marker and limit + # Verify only one image was returned with the correct id + params = "limit=1&marker=3" + path = "http://%s:%d/v1/images?%s" % ( + "0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 1) + self.assertEqual(data['images'][0]['id'], 2) + + self.stop_servers() + + def test_ordered_images(self): + """ + Set up three test images and ensure each query param filter works + """ + self.cleanup() + self.start_servers() + + # 0. GET /images + # Verify no public images + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(content, '{"images": []}') + + # 1. POST /images with three public images with various attributes + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image1', + 'X-Image-Meta-Status': 'active', + 'X-Image-Meta-Container-Format': 'ovf', + 'X-Image-Meta-Disk-Format': 'vdi', + 'X-Image-Meta-Size': '19', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'ASDF', + 'X-Image-Meta-Status': 'active', + 'X-Image-Meta-Container-Format': 'bare', + 'X-Image-Meta-Disk-Format': 'iso', + 'X-Image-Meta-Size': '2', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'XYZ', + 'X-Image-Meta-Status': 'saving', + 'X-Image-Meta-Container-Format': 'ami', + 'X-Image-Meta-Disk-Format': 'ami', + 'X-Image-Meta-Size': '5', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + + # 2. GET /images with no query params + # Verify three public images sorted by created_at desc + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 3) + self.assertEqual(data['images'][0]['id'], 3) + self.assertEqual(data['images'][1]['id'], 2) + self.assertEqual(data['images'][2]['id'], 1) + + # 3. GET /images sorted by name asc + params = 'sort_key=name&sort_dir=asc' + path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 3) + self.assertEqual(data['images'][0]['id'], 2) + self.assertEqual(data['images'][1]['id'], 1) + self.assertEqual(data['images'][2]['id'], 3) + + # 4. GET /images sorted by size desc + params = 'sort_key=size&sort_dir=desc' + path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 3) + self.assertEqual(data['images'][0]['id'], 1) + self.assertEqual(data['images'][1]['id'], 3) + self.assertEqual(data['images'][2]['id'], 2) + + self.stop_servers() + + def test_duplicate_image_upload(self): + """ + Upload initial image, then attempt to upload duplicate image + """ + self.cleanup() + self.start_servers() + + # 0. GET /images + # Verify no public images + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + self.assertEqual(content, '{"images": []}') + + # 1. POST /images with public image named Image1 + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image1', + 'X-Image-Meta-Status': 'active', + 'X-Image-Meta-Container-Format': 'ovf', + 'X-Image-Meta-Disk-Format': 'vdi', + 'X-Image-Meta-Size': '19', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 201) + + # 2. POST /images with public image named Image1, and ID: 1 + headers = {'Content-Type': 'application/octet-stream', + 'X-Image-Meta-Name': 'Image1 Update', + 'X-Image-Meta-Status': 'active', + 'X-Image-Meta-Container-Format': 'ovf', + 'X-Image-Meta-Disk-Format': 'vdi', + 'X-Image-Meta-Size': '19', + 'X-Image-Meta-Id': '1', + 'X-Image-Meta-Is-Public': 'True'} + path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', headers=headers) + self.assertEqual(response.status, 409) + expected = "An image with identifier 1 already exists" + self.assertTrue(expected in content, + "Could not find '%s' in '%s'" % (expected, content)) + + self.stop_servers() diff --git a/tests/stubs.py b/tests/stubs.py index 56e7d20a5a..92d8170539 100644 --- a/tests/stubs.py +++ b/tests/stubs.py @@ -128,13 +128,9 @@ def stub_out_s3_backend(stubs): DATA = 'I am a teapot, short and stout\n' @classmethod - def get(cls, parsed_uri, expected_size, conn_class=None): + def get(cls, location, expected_size, conn_class=None): S3Backend = glance.store.s3.S3Backend - # raise BackendException if URI is bad. - (user, key, authurl, container, obj) = \ - S3Backend._parse_s3_tokens(parsed_uri) - def chunk_it(): for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE): yield cls.DATA[i:i + cls.CHUNK_SIZE] @@ -400,9 +396,9 @@ def stub_out_registry_db_image_api(stubs): f['deleted_at'] <= delete_time] return images - def image_get_all_public(self, _context, filters=None, marker=None, - limit=1000, sort_key=None, sort_dir=None): - images = [f for f in self.images if f['is_public'] == True] + def image_get_all(self, _context, filters=None, marker=None, + limit=1000, sort_key=None, sort_dir=None): + images = self.images if 'size_min' in filters: size_min = int(filters.pop('size_min')) @@ -424,7 +420,8 @@ def stub_out_registry_db_image_api(stubs): images = filter(_prop_filter(k, v), images) for k, v in filters.items(): - images = [f for f in images if f[k] == v] + if v is not None: + images = [f for f in images if f[k] == v] # sorted func expects func that compares in descending order def image_cmp(x, y): @@ -473,5 +470,5 @@ def stub_out_registry_db_image_api(stubs): fake_datastore.image_get) stubs.Set(glance.registry.db.api, 'image_get_all_pending_delete', fake_datastore.image_get_all_pending_delete) - stubs.Set(glance.registry.db.api, 'image_get_all_public', - fake_datastore.image_get_all_public) + stubs.Set(glance.registry.db.api, 'image_get_all', + fake_datastore.image_get_all) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 07bfc6b5bf..1a82c1e033 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -1077,6 +1077,84 @@ class TestRegistryAPI(unittest.TestCase): for image in images: self.assertEqual('v a', image['properties']['prop_123']) + def test_get_details_filter_public_none(self): + """ + Tests that the /images/detail registry API returns list of + all images if is_public none is passed + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': False, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'fake image #3', + 'size': 18, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images/detail?is_public=None') + res = req.get_response(self.api) + res_dict = json.loads(res.body) + self.assertEquals(res.status_int, 200) + + images = res_dict['images'] + self.assertEquals(len(images), 3) + + def test_get_details_filter_public_false(self): + """ + Tests that the /images/detail registry API returns list of + private images if is_public false is passed + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': False, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'fake image #3', + 'size': 18, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images/detail?is_public=False') + res = req.get_response(self.api) + res_dict = json.loads(res.body) + self.assertEquals(res.status_int, 200) + + images = res_dict['images'] + self.assertEquals(len(images), 2) + + for image in images: + self.assertEqual(False, image['is_public']) + + def test_get_details_filter_public_true(self): + """ + Tests that the /images/detail registry API returns list of + public images if is_public true is passed (same as default) + """ + extra_fixture = {'id': 3, + 'status': 'active', + 'is_public': False, + 'disk_format': 'vhd', + 'container_format': 'ovf', + 'name': 'fake image #3', + 'size': 18, + 'checksum': None} + + glance.registry.db.api.image_create(None, extra_fixture) + + req = webob.Request.blank('/images/detail?is_public=True') + res = req.get_response(self.api) + res_dict = json.loads(res.body) + self.assertEquals(res.status_int, 200) + + images = res_dict['images'] + self.assertEquals(len(images), 1) + + for image in images: + self.assertEqual(True, image['is_public']) + def test_get_details_sort_name_asc(self): """ Tests that the /images/details registry API returns list of diff --git a/tests/unit/test_filesystem_store.py b/tests/unit/test_filesystem_store.py index 97a1c6ef23..4b214ec177 100644 --- a/tests/unit/test_filesystem_store.py +++ b/tests/unit/test_filesystem_store.py @@ -20,11 +20,11 @@ import StringIO import hashlib import unittest -import urlparse import stubout from glance.common import exception +from glance.store.location import get_location_from_uri from glance.store.filesystem import FilesystemBackend, ChunkedFile from tests import stubs @@ -51,8 +51,8 @@ class TestFilesystemBackend(unittest.TestCase): def test_get(self): """Test a "normal" retrieval of an image in chunks""" - url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2") - image_file = FilesystemBackend.get(url_pieces) + loc = get_location_from_uri("file:///tmp/glance-tests/2") + image_file = FilesystemBackend.get(loc) expected_data = "chunk00000remainder" expected_num_chunks = 2 @@ -70,10 +70,10 @@ class TestFilesystemBackend(unittest.TestCase): Test that trying to retrieve a file that doesn't exist raises an error """ - url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing") + loc = get_location_from_uri("file:///tmp/glance-tests/non-existing") self.assertRaises(exception.NotFound, FilesystemBackend.get, - url_pieces) + loc) def test_add(self): """Test that we can add an image via the filesystem backend""" @@ -93,8 +93,8 @@ class TestFilesystemBackend(unittest.TestCase): self.assertEquals(expected_file_size, size) self.assertEquals(expected_checksum, checksum) - url_pieces = urlparse.urlparse("file:///tmp/glance-tests/42") - new_image_file = FilesystemBackend.get(url_pieces) + loc = get_location_from_uri("file:///tmp/glance-tests/42") + new_image_file = FilesystemBackend.get(loc) new_image_contents = "" new_image_file_size = 0 @@ -122,20 +122,19 @@ class TestFilesystemBackend(unittest.TestCase): """ Test we can delete an existing image in the filesystem store """ - url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2") - - FilesystemBackend.delete(url_pieces) + loc = get_location_from_uri("file:///tmp/glance-tests/2") + FilesystemBackend.delete(loc) self.assertRaises(exception.NotFound, FilesystemBackend.get, - url_pieces) + loc) def test_delete_non_existing(self): """ Test that trying to delete a file that doesn't exist raises an error """ - url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing") + loc = get_location_from_uri("file:///tmp/glance-tests/non-existing") self.assertRaises(exception.NotFound, FilesystemBackend.delete, - url_pieces) + loc) diff --git a/tests/unit/test_store_location.py b/tests/unit/test_store_location.py new file mode 100644 index 0000000000..a08f4905d0 --- /dev/null +++ b/tests/unit/test_store_location.py @@ -0,0 +1,243 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +from glance.common import exception +import glance.store.location as location +import glance.store.http +import glance.store.filesystem +import glance.store.swift +import glance.store.s3 + + +class TestStoreLocation(unittest.TestCase): + + def test_get_location_from_uri_back_to_uri(self): + """ + Test that for various URIs, the correct Location + object can be contructed and then the original URI + returned via the get_store_uri() method. + """ + good_store_uris = [ + 'https://user:pass@example.com:80/images/some-id', + 'http://images.oracle.com/123456', + 'swift://account:user:pass@authurl.com/container/obj-id', + 'swift+https://account:user:pass@authurl.com/container/obj-id', + 's3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id', + 's3://accesskey:secretwith/aslash@s3.amazonaws.com/bucket/key-id', + 's3+http://accesskey:secret@s3.amazonaws.com/bucket/key-id', + 's3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id', + 'file:///var/lib/glance/images/1'] + + for uri in good_store_uris: + loc = location.get_location_from_uri(uri) + # The get_store_uri() method *should* return an identical URI + # to the URI that is passed to get_location_from_uri() + self.assertEqual(loc.get_store_uri(), uri) + + def test_bad_store_scheme(self): + """ + Test that a URI with a non-existing scheme triggers exception + """ + bad_uri = 'unknown://user:pass@example.com:80/images/some-id' + + self.assertRaises(exception.UnknownScheme, + location.get_location_from_uri, + bad_uri) + + def test_filesystem_store_location(self): + """ + Test the specific StoreLocation for the Filesystem store + """ + uri = 'file:///var/lib/glance/images/1' + loc = glance.store.filesystem.StoreLocation({}) + loc.parse_uri(uri) + + self.assertEqual("file", loc.scheme) + self.assertEqual("/var/lib/glance/images/1", loc.path) + self.assertEqual(uri, loc.get_uri()) + + bad_uri = 'fil://' + self.assertRaises(Exception, loc.parse_uri, bad_uri) + + bad_uri = 'file://' + self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) + + def test_http_store_location(self): + """ + Test the specific StoreLocation for the HTTP store + """ + uri = 'http://example.com/images/1' + loc = glance.store.http.StoreLocation({}) + loc.parse_uri(uri) + + self.assertEqual("http", loc.scheme) + self.assertEqual("example.com", loc.netloc) + self.assertEqual("/images/1", loc.path) + self.assertEqual(uri, loc.get_uri()) + + uri = 'https://example.com:8080/images/container/1' + loc.parse_uri(uri) + + self.assertEqual("https", loc.scheme) + self.assertEqual("example.com:8080", loc.netloc) + self.assertEqual("/images/container/1", loc.path) + self.assertEqual(uri, loc.get_uri()) + + uri = 'https://user:password@example.com:8080/images/container/1' + loc.parse_uri(uri) + + self.assertEqual("https", loc.scheme) + self.assertEqual("example.com:8080", loc.netloc) + self.assertEqual("user", loc.user) + self.assertEqual("password", loc.password) + self.assertEqual("/images/container/1", loc.path) + self.assertEqual(uri, loc.get_uri()) + + uri = 'https://user:@example.com:8080/images/1' + loc.parse_uri(uri) + + self.assertEqual("https", loc.scheme) + self.assertEqual("example.com:8080", loc.netloc) + self.assertEqual("user", loc.user) + self.assertEqual("", loc.password) + self.assertEqual("/images/1", loc.path) + self.assertEqual(uri, loc.get_uri()) + + bad_uri = 'htt://' + self.assertRaises(Exception, loc.parse_uri, bad_uri) + + bad_uri = 'http://' + self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) + + bad_uri = 'http://user@example.com:8080/images/1' + self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) + + def test_swift_store_location(self): + """ + Test the specific StoreLocation for the Swift store + """ + uri = 'swift://example.com/images/1' + loc = glance.store.swift.StoreLocation({}) + loc.parse_uri(uri) + + self.assertEqual("swift", loc.scheme) + self.assertEqual("example.com", loc.authurl) + self.assertEqual("images", loc.container) + self.assertEqual("1", loc.obj) + self.assertEqual(None, loc.user) + self.assertEqual(uri, loc.get_uri()) + + uri = 'swift+https://user:pass@authurl.com/images/1' + loc.parse_uri(uri) + + self.assertEqual("swift+https", loc.scheme) + self.assertEqual("authurl.com", loc.authurl) + self.assertEqual("images", loc.container) + self.assertEqual("1", loc.obj) + self.assertEqual("user", loc.user) + self.assertEqual("pass", loc.key) + self.assertEqual(uri, loc.get_uri()) + + uri = 'swift+https://user:pass@authurl.com/v1/container/12345' + loc.parse_uri(uri) + + self.assertEqual("swift+https", loc.scheme) + self.assertEqual("authurl.com/v1", loc.authurl) + self.assertEqual("container", loc.container) + self.assertEqual("12345", loc.obj) + self.assertEqual("user", loc.user) + self.assertEqual("pass", loc.key) + self.assertEqual(uri, loc.get_uri()) + + uri = 'swift://account:user:pass@authurl.com/v1/container/12345' + loc.parse_uri(uri) + + self.assertEqual("swift", loc.scheme) + self.assertEqual("authurl.com/v1", loc.authurl) + self.assertEqual("container", loc.container) + self.assertEqual("12345", loc.obj) + self.assertEqual("account:user", loc.user) + self.assertEqual("pass", loc.key) + self.assertEqual(uri, loc.get_uri()) + + bad_uri = 'swif://' + self.assertRaises(Exception, loc.parse_uri, bad_uri) + + bad_uri = 'swift://' + self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) + + bad_uri = 'swift://user@example.com:8080/images/1' + self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) + + def test_s3_store_location(self): + """ + Test the specific StoreLocation for the S3 store + """ + uri = 's3://example.com/images/1' + loc = glance.store.s3.StoreLocation({}) + loc.parse_uri(uri) + + self.assertEqual("s3", loc.scheme) + self.assertEqual("example.com", loc.s3serviceurl) + self.assertEqual("images", loc.bucket) + self.assertEqual("1", loc.key) + self.assertEqual(None, loc.accesskey) + self.assertEqual(uri, loc.get_uri()) + + uri = 's3+https://accesskey:pass@s3serviceurl.com/images/1' + loc.parse_uri(uri) + + self.assertEqual("s3+https", loc.scheme) + self.assertEqual("s3serviceurl.com", loc.s3serviceurl) + self.assertEqual("images", loc.bucket) + self.assertEqual("1", loc.key) + self.assertEqual("accesskey", loc.accesskey) + self.assertEqual("pass", loc.secretkey) + self.assertEqual(uri, loc.get_uri()) + + uri = 's3+https://accesskey:pass@s3serviceurl.com/v1/bucket/12345' + loc.parse_uri(uri) + + self.assertEqual("s3+https", loc.scheme) + self.assertEqual("s3serviceurl.com/v1", loc.s3serviceurl) + self.assertEqual("bucket", loc.bucket) + self.assertEqual("12345", loc.key) + self.assertEqual("accesskey", loc.accesskey) + self.assertEqual("pass", loc.secretkey) + self.assertEqual(uri, loc.get_uri()) + + uri = 's3://accesskey:pass/withslash@s3serviceurl.com/v1/bucket/12345' + loc.parse_uri(uri) + + self.assertEqual("s3", loc.scheme) + self.assertEqual("s3serviceurl.com/v1", loc.s3serviceurl) + self.assertEqual("bucket", loc.bucket) + self.assertEqual("12345", loc.key) + self.assertEqual("accesskey", loc.accesskey) + self.assertEqual("pass/withslash", loc.secretkey) + self.assertEqual(uri, loc.get_uri()) + + bad_uri = 'swif://' + self.assertRaises(Exception, loc.parse_uri, bad_uri) + + bad_uri = 's3://' + self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) + + bad_uri = 's3://accesskey@example.com:8080/images/1' + self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) diff --git a/tests/unit/test_stores.py b/tests/unit/test_stores.py index 7308e6b4ac..acae9a2d1a 100644 --- a/tests/unit/test_stores.py +++ b/tests/unit/test_stores.py @@ -19,7 +19,6 @@ from StringIO import StringIO import stubout import unittest -import urlparse from glance.store.s3 import S3Backend from glance.store import Backend, BackendException, get_from_backend diff --git a/tests/unit/test_swift_store.py b/tests/unit/test_swift_store.py index 62b1e81246..044fc14a82 100644 --- a/tests/unit/test_swift_store.py +++ b/tests/unit/test_swift_store.py @@ -29,9 +29,8 @@ import swift.common.client from glance.common import exception from glance.store import BackendException -from glance.store.swift import (SwiftBackend, - format_swift_location, - parse_swift_tokens) +from glance.store.swift import SwiftBackend +from glance.store.location import get_location_from_uri FIVE_KB = (5 * 1024) SWIFT_OPTIONS = {'verbose': True, @@ -146,6 +145,18 @@ def stub_out_swift_common_client(stubs): 'http_connection', fake_http_connection) +def format_swift_location(user, key, authurl, container, obj): + """ + Helper method that returns a Swift store URI given + the component pieces. + """ + scheme = 'swift+https' + if authurl.startswith('http://'): + scheme = 'swift+http' + return "%s://%s:%s@%s/%s/%s" % (scheme, user, key, authurl, + container, obj) + + class TestSwiftBackend(unittest.TestCase): def setUp(self): @@ -157,46 +168,27 @@ class TestSwiftBackend(unittest.TestCase): """Clear the test environment""" self.stubs.UnsetAll() - def test_parse_swift_tokens(self): - """ - Test that the parse_swift_tokens function returns - user, key, authurl, container, and objname properly - """ - uri = "swift://user:key@localhost/v1.0/container/objname" - url_pieces = urlparse.urlparse(uri) - user, key, authurl, container, objname =\ - parse_swift_tokens(url_pieces) - self.assertEqual("user", user) - self.assertEqual("key", key) - self.assertEqual("https://localhost/v1.0", authurl) - self.assertEqual("container", container) - self.assertEqual("objname", objname) - - uri = "swift://user:key@localhost:9090/v1.0/container/objname" - url_pieces = urlparse.urlparse(uri) - user, key, authurl, container, objname =\ - parse_swift_tokens(url_pieces) - self.assertEqual("user", user) - self.assertEqual("key", key) - self.assertEqual("https://localhost:9090/v1.0", authurl) - self.assertEqual("container", container) - self.assertEqual("objname", objname) - - uri = "swift://account:user:key@localhost:9090/v1.0/container/objname" - url_pieces = urlparse.urlparse(uri) - user, key, authurl, container, objname =\ - parse_swift_tokens(url_pieces) - self.assertEqual("account:user", user) - self.assertEqual("key", key) - self.assertEqual("https://localhost:9090/v1.0", authurl) - self.assertEqual("container", container) - self.assertEqual("objname", objname) - def test_get(self): """Test a "normal" retrieval of an image in chunks""" - url_pieces = urlparse.urlparse( - "swift://user:key@auth_address/glance/2") - image_swift = SwiftBackend.get(url_pieces) + loc = get_location_from_uri("swift://user:key@auth_address/glance/2") + image_swift = SwiftBackend.get(loc) + + expected_data = "*" * FIVE_KB + data = "" + + for chunk in image_swift: + data += chunk + self.assertEqual(expected_data, data) + + def test_get_with_http_auth(self): + """ + Test a retrieval from Swift with an HTTP authurl. This is + specified either via a Location header with swift+http:// or using + http:// in the swift_store_auth_address config value + """ + loc = get_location_from_uri("swift+http://user:key@auth_address/" + "glance/2") + image_swift = SwiftBackend.get(loc) expected_data = "*" * FIVE_KB data = "" @@ -210,11 +202,10 @@ class TestSwiftBackend(unittest.TestCase): Test retrieval of an image with wrong expected_size param raises an exception """ - url_pieces = urlparse.urlparse( - "swift://user:key@auth_address/glance/2") + loc = get_location_from_uri("swift://user:key@auth_address/glance/2") self.assertRaises(BackendException, SwiftBackend.get, - url_pieces, + loc, {'expected_size': 42}) def test_get_non_existing(self): @@ -222,11 +213,10 @@ class TestSwiftBackend(unittest.TestCase): Test that trying to retrieve a swift that doesn't exist raises an error """ - url_pieces = urlparse.urlparse( - "swift://user:key@auth_address/noexist") + loc = get_location_from_uri("swift://user:key@authurl/glance/noexist") self.assertRaises(exception.NotFound, SwiftBackend.get, - url_pieces) + loc) def test_add(self): """Test that we can add an image via the swift backend""" @@ -249,14 +239,62 @@ class TestSwiftBackend(unittest.TestCase): self.assertEquals(expected_swift_size, size) self.assertEquals(expected_checksum, checksum) - url_pieces = urlparse.urlparse(expected_location) - new_image_swift = SwiftBackend.get(url_pieces) + loc = get_location_from_uri(expected_location) + new_image_swift = SwiftBackend.get(loc) new_image_contents = new_image_swift.getvalue() new_image_swift_size = new_image_swift.len self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_size, new_image_swift_size) + def test_add_auth_url_variations(self): + """ + Test that we can add an image via the swift backend with + a variety of different auth_address values + """ + variations = ['http://localhost:80', + 'http://localhost', + 'http://localhost/v1', + 'http://localhost/v1/', + 'https://localhost', + 'https://localhost:8080', + 'https://localhost/v1', + 'https://localhost/v1/', + 'localhost', + 'localhost:8080/v1'] + i = 42 + for variation in variations: + expected_image_id = i + expected_swift_size = FIVE_KB + expected_swift_contents = "*" * expected_swift_size + expected_checksum = \ + hashlib.md5(expected_swift_contents).hexdigest() + new_options = SWIFT_OPTIONS.copy() + new_options['swift_store_auth_address'] = variation + expected_location = format_swift_location( + new_options['swift_store_user'], + new_options['swift_store_key'], + new_options['swift_store_auth_address'], + new_options['swift_store_container'], + expected_image_id) + image_swift = StringIO.StringIO(expected_swift_contents) + + location, size, checksum = SwiftBackend.add(i, image_swift, + new_options) + + self.assertEquals(expected_location, location) + self.assertEquals(expected_swift_size, size) + self.assertEquals(expected_checksum, checksum) + + loc = get_location_from_uri(expected_location) + new_image_swift = SwiftBackend.get(loc) + new_image_contents = new_image_swift.getvalue() + new_image_swift_size = new_image_swift.len + + self.assertEquals(expected_swift_contents, new_image_contents) + self.assertEquals(expected_swift_size, new_image_swift_size) + i = i + 1 + def test_add_no_container_no_create(self): """ Tests that adding an image with a non-existing container @@ -306,8 +344,8 @@ class TestSwiftBackend(unittest.TestCase): self.assertEquals(expected_swift_size, size) self.assertEquals(expected_checksum, checksum) - url_pieces = urlparse.urlparse(expected_location) - new_image_swift = SwiftBackend.get(url_pieces) + loc = get_location_from_uri(expected_location) + new_image_swift = SwiftBackend.get(loc) new_image_contents = new_image_swift.getvalue() new_image_swift_size = new_image_swift.len @@ -356,22 +394,20 @@ class TestSwiftBackend(unittest.TestCase): """ Test we can delete an existing image in the swift store """ - url_pieces = urlparse.urlparse( - "swift://user:key@auth_address/glance/2") + loc = get_location_from_uri("swift://user:key@authurl/glance/2") - SwiftBackend.delete(url_pieces) + SwiftBackend.delete(loc) self.assertRaises(exception.NotFound, SwiftBackend.get, - url_pieces) + loc) def test_delete_non_existing(self): """ Test that trying to delete a swift that doesn't exist raises an error """ - url_pieces = urlparse.urlparse( - "swift://user:key@auth_address/noexist") + loc = get_location_from_uri("swift://user:key@authurl/glance/noexist") self.assertRaises(exception.NotFound, SwiftBackend.delete, - url_pieces) + loc)