openstack/glance
Code Issues Proposed changes

merge trunk

Browse Source
This commit is contained in:
Jason Koelker 2011-07-21 11:33:50 -05:00
commit f62d5b4ac6
18 changed files with 1565 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
glance/api/v1/images.py
View File

@ -43,7 +43,7 @@ from glance import utils
logger = logging.getLogger('glance.api.v1.images') logger = logging.getLogger('glance.api.v1.images')
SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format', SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format',
'size_min', 'size_max'] 'size_min', 'size_max', 'is_public']
SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir') SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir')

18
glance/common/exception.py
View File

@ -54,6 +54,24 @@ class NotFound(Error):
pass pass
class UnknownScheme(Error):
msg = "Unknown scheme '%s' found in URI"
def __init__(self, scheme):
msg = self.__class__.msg % scheme
super(UnknownScheme, self).__init__(msg)
class BadStoreUri(Error):
msg = "The Store URI %s was malformed. Reason: %s"
def __init__(self, uri, reason):
msg = self.__class__.msg % (uri, reason)
super(BadStoreUri, self).__init__(msg)
class Duplicate(Error): class Duplicate(Error):
pass pass

19
glance/registry/db/api.py
View File

@ -24,6 +24,7 @@ Defines interface for DB access
import logging import logging
from sqlalchemy import asc, create_engine, desc from sqlalchemy import asc, create_engine, desc
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import exc from sqlalchemy.orm import exc
from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload
@ -152,10 +153,10 @@ def image_get_all_pending_delete(context, delete_time=None, limit=None):
return query.all() return query.all()
def image_get_all_public(context, filters=None, marker=None, limit=None, def image_get_all(context, filters=None, marker=None, limit=None,
sort_key='created_at', sort_dir='desc'): sort_key='created_at', sort_dir='desc'):
"""Get all public images that match zero or more filters. """
Get all public images that match zero or more filters. Get all images that match zero or more filters.
:param filters: dict of filter keys and values. If a 'properties' :param filters: dict of filter keys and values. If a 'properties'
key is present, it is treated as a dict of key/value key is present, it is treated as a dict of key/value
@ -171,7 +172,6 @@ def image_get_all_public(context, filters=None, marker=None, limit=None,
query = session.query(models.Image).\ query = session.query(models.Image).\
options(joinedload(models.Image.properties)).\ options(joinedload(models.Image.properties)).\
filter_by(deleted=_deleted(context)).\ filter_by(deleted=_deleted(context)).\
filter_by(is_public=True).\
filter(models.Image.status != 'killed') filter(models.Image.status != 'killed')
sort_dir_func = { sort_dir_func = {
@ -196,7 +196,8 @@ def image_get_all_public(context, filters=None, marker=None, limit=None,
query = query.filter(models.Image.properties.any(name=k, value=v)) query = query.filter(models.Image.properties.any(name=k, value=v))
for (k, v) in filters.items(): for (k, v) in filters.items():
query = query.filter(getattr(models.Image, k) == v) if v is not None:
query = query.filter(getattr(models.Image, k) == v)
if marker != None: if marker != None:
# images returned should be created before the image defined by marker # images returned should be created before the image defined by marker
@ -297,7 +298,11 @@ def _image_update(context, values, image_id, purge_props=False):
# idiotic. # idiotic.
validate_image(image_ref.to_dict()) validate_image(image_ref.to_dict())
image_ref.save(session=session) try:
image_ref.save(session=session)
except IntegrityError, e:
raise exception.Duplicate("Image ID %s already exists!"
% values['id'])
_set_properties_for_image(context, image_ref, properties, purge_props, _set_properties_for_image(context, image_ref, properties, purge_props,
session) session)

41
glance/registry/server.py
View File

@ -56,6 +56,16 @@ class Controller(object):
self.options = options self.options = options
db_api.configure_db(options) db_api.configure_db(options)
def _get_images(self, context, **params):
"""
Get images, wrapping in exception if necessary.
"""
try:
return db_api.image_get_all(None, **params)
except exception.NotFound, e:
msg = "Invalid marker. Image could not be found."
raise exc.HTTPBadRequest(explanation=msg)
def index(self, req): def index(self, req):
""" """
Return a basic filtered list of public, non-deleted images Return a basic filtered list of public, non-deleted images
@ -77,11 +87,7 @@ class Controller(object):
} }
""" """
params = self._get_query_params(req) params = self._get_query_params(req)
try: images = self._get_images(None, **params)
images = db_api.image_get_all_public(None, **params)
except exception.NotFound, e:
msg = "Invalid marker. Image could not be found."
raise exc.HTTPBadRequest(explanation=msg)
results = [] results = []
for image in images: for image in images:
@ -104,12 +110,8 @@ class Controller(object):
all image model fields. all image model fields.
""" """
params = self._get_query_params(req) params = self._get_query_params(req)
try:
images = db_api.image_get_all_public(None, **params)
except exception.NotFound, e:
msg = "Invalid marker. Image could not be found."
raise exc.HTTPBadRequest(explanation=msg)
images = self._get_images(None, **params)
image_dicts = [make_image_dict(i) for i in images] image_dicts = [make_image_dict(i) for i in images]
return dict(images=image_dicts) return dict(images=image_dicts)
@ -144,6 +146,7 @@ class Controller(object):
filters = {} filters = {}
properties = {} properties = {}
filters['is_public'] = self._get_is_public(req)
for param in req.str_params: for param in req.str_params:
if param in SUPPORTED_FILTERS: if param in SUPPORTED_FILTERS:
filters[param] = req.str_params.get(param) filters[param] = req.str_params.get(param)
@ -199,6 +202,24 @@ class Controller(object):
raise exc.HTTPBadRequest(explanation=msg) raise exc.HTTPBadRequest(explanation=msg)
return sort_dir return sort_dir
def _get_is_public(self, req):
"""Parse is_public into something usable."""
is_public = req.str_params.get('is_public', None)
if is_public is None:
# NOTE(vish): This preserves the default value of showing only
# public images.
return True
is_public = is_public.lower()
if is_public == 'none':
return None
elif is_public == 'true' or is_public == '1':
return True
elif is_public == 'false' or is_public == '0':
return False
else:
raise exc.HTTPBadRequest("is_public must be None, True, or False")
def show(self, req, id): def show(self, req, id):
"""Return data about the given image id.""" """Return data about the given image id."""
try: try:

61
glance/store/__init__.py
View File

@ -22,6 +22,7 @@ import urlparse
from glance import registry from glance import registry
from glance.common import config, exception from glance.common import config, exception
from glance.store import location
logger = logging.getLogger('glance.store') logger = logging.getLogger('glance.store')
@ -79,72 +80,32 @@ def get_backend_class(backend):
def get_from_backend(uri, **kwargs): def get_from_backend(uri, **kwargs):
"""Yields chunks of data from backend specified by uri""" """Yields chunks of data from backend specified by uri"""
parsed_uri = urlparse.urlparse(uri) loc = location.get_location_from_uri(uri)
scheme = parsed_uri.scheme backend_class = get_backend_class(loc.store_name)
backend_class = get_backend_class(scheme) return backend_class.get(loc, **kwargs)
return backend_class.get(parsed_uri, **kwargs)
def delete_from_backend(uri, **kwargs): def delete_from_backend(uri, **kwargs):
"""Removes chunks of data from backend specified by uri""" """Removes chunks of data from backend specified by uri"""
parsed_uri = urlparse.urlparse(uri) loc = location.get_location_from_uri(uri)
scheme = parsed_uri.scheme backend_class = get_backend_class(loc.store_name)
backend_class = get_backend_class(scheme)
if hasattr(backend_class, 'delete'): if hasattr(backend_class, 'delete'):
return backend_class.delete(parsed_uri, **kwargs) return backend_class.delete(loc, **kwargs)
def get_store_from_location(location): def get_store_from_location(uri):
""" """
Given a location (assumed to be a URL), attempt to determine Given a location (assumed to be a URL), attempt to determine
the store from the location. We use here a simple guess that the store from the location. We use here a simple guess that
the scheme of the parsed URL is the store... the scheme of the parsed URL is the store...
:param location: Location to check for the store :param uri: Location to check for the store
""" """
loc_pieces = urlparse.urlparse(location) loc = location.get_location_from_uri(uri)
return loc_pieces.scheme return loc.store_name
def parse_uri_tokens(parsed_uri, example_url):
"""
Given a URI and an example_url, attempt to parse the uri to assemble an
authurl. This method returns the user, key, authurl, referenced container,
and the object we're looking for in that container.
Parsing the uri is three phases:
1) urlparse to split the tokens
2) use RE to split on @ and /
3) reassemble authurl
"""
path = parsed_uri.path.lstrip('//')
netloc = parsed_uri.netloc
try:
try:
creds, netloc = netloc.split('@')
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
creds, path = path.split('@')
user, key = creds.split(':')
path_parts = path.split('/')
obj = path_parts.pop()
container = path_parts.pop()
except (ValueError, IndexError):
raise BackendException(
"Expected four values to unpack in: %s:%s. "
"Should have received something like: %s."
% (parsed_uri.scheme, parsed_uri.path, example_url))
authurl = "https://%s" % '/'.join(path_parts)
return user, key, authurl, container, obj
def schedule_delete_from_backend(uri, options, id, **kwargs): def schedule_delete_from_backend(uri, options, id, **kwargs):

56
glance/store/filesystem.py
View File

@ -26,9 +26,39 @@ import urlparse
from glance.common import exception from glance.common import exception
import glance.store import glance.store
import glance.store.location
logger = logging.getLogger('glance.store.filesystem') logger = logging.getLogger('glance.store.filesystem')
glance.store.location.add_scheme_map({'file': 'filesystem'})
class StoreLocation(glance.store.location.StoreLocation):
"""Class describing a Filesystem URI"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 'file')
self.path = self.specs.get('path')
def get_uri(self):
return "file://%s" % self.path
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme == 'file'
self.scheme = pieces.scheme
path = (pieces.netloc + pieces.path).strip()
if path == '':
reason = "No path specified"
raise exception.BadStoreUri(uri, reason)
self.path = path
class ChunkedFile(object): class ChunkedFile(object):
@ -64,13 +94,19 @@ class ChunkedFile(object):
class FilesystemBackend(glance.store.Backend): class FilesystemBackend(glance.store.Backend):
@classmethod @classmethod
def get(cls, parsed_uri, expected_size=None, options=None): def get(cls, location, expected_size=None, options=None):
""" """
Filesystem-based backend Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator to use in
reading the image file.
file:///path/to/file.tar.gz.0 :location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if file does not exist
""" """
filepath = parsed_uri.path loc = location.store_location
filepath = loc.path
if not os.path.exists(filepath): if not os.path.exists(filepath):
raise exception.NotFound("Image file %s not found" % filepath) raise exception.NotFound("Image file %s not found" % filepath)
else: else:
@ -79,17 +115,19 @@ class FilesystemBackend(glance.store.Backend):
return ChunkedFile(filepath) return ChunkedFile(filepath)
@classmethod @classmethod
def delete(cls, parsed_uri): def delete(cls, location):
""" """
Removes a file from the filesystem backend. Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:param parsed_uri: Parsed pieces of URI in form of:: :location `glance.store.location.Location` object, supplied
file:///path/to/filename.ext from glance.store.location.get_location_from_uri()
:raises NotFound if file does not exist :raises NotFound if file does not exist
:raises NotAuthorized if cannot delete because of permissions :raises NotAuthorized if cannot delete because of permissions
""" """
fn = parsed_uri.path loc = location.store_location
fn = loc.path
if os.path.exists(fn): if os.path.exists(fn):
try: try:
logger.debug("Deleting image at %s", fn) logger.debug("Deleting image at %s", fn)

87
glance/store/http.py
View File

@ -16,31 +16,104 @@
# under the License. # under the License.
import httplib import httplib
import urlparse
from glance.common import exception
import glance.store import glance.store
import glance.store.location
glance.store.location.add_scheme_map({'http': 'http',
'https': 'http'})
class StoreLocation(glance.store.location.StoreLocation):
"""Class describing an HTTP(S) URI"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 'http')
self.netloc = self.specs['netloc']
self.user = self.specs.get('user')
self.password = self.specs.get('password')
self.path = self.specs.get('path')
def _get_credstring(self):
if self.user:
return '%s:%s@' % (self.user, self.password)
return ''
def get_uri(self):
return "%s://%s%s%s" % (
self.scheme,
self._get_credstring(),
self.netloc,
self.path)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('https', 'http')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path
try:
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
if creds:
try:
self.user, self.password = creds.split(':')
except ValueError:
reason = ("Credentials '%s' not well-formatted."
% "".join(creds))
raise exception.BadStoreUri(uri, reason)
else:
self.user = None
if netloc == '':
reason = "No address specified in HTTP URL"
raise exception.BadStoreUri(uri, reason)
self.netloc = netloc
self.path = path
class HTTPBackend(glance.store.Backend): class HTTPBackend(glance.store.Backend):
""" An implementation of the HTTP Backend Adapter """ """ An implementation of the HTTP Backend Adapter """
@classmethod @classmethod
def get(cls, parsed_uri, expected_size, options=None, conn_class=None): def get(cls, location, expected_size, options=None, conn_class=None):
""" """
Takes a parsed uri for an HTTP resource, fetches it, and Takes a `glance.store.location.Location` object that indicates
yields the data. where to find the image file, and returns a generator from Swift
provided by Swift client's get_object() method.
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
""" """
loc = location.store_location
if conn_class: if conn_class:
pass # use the conn_class passed in pass # use the conn_class passed in
elif parsed_uri.scheme == "http": elif loc.scheme == "http":
conn_class = httplib.HTTPConnection conn_class = httplib.HTTPConnection
elif parsed_uri.scheme == "https": elif loc.scheme == "https":
conn_class = httplib.HTTPSConnection conn_class = httplib.HTTPSConnection
else: else:
raise glance.store.BackendException( raise glance.store.BackendException(
"scheme '%s' not supported for HTTPBackend") "scheme '%s' not supported for HTTPBackend")
conn = conn_class(parsed_uri.netloc) conn = conn_class(loc.netloc)
conn.request("GET", parsed_uri.path, "", {}) conn.request("GET", loc.path, "", {})
try: try:
return glance.store._file_iter(conn.getresponse(), cls.CHUNKSIZE) return glance.store._file_iter(conn.getresponse(), cls.CHUNKSIZE)

182
glance/store/location.py Normal file
View File

@ -0,0 +1,182 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
A class that describes the location of an image in Glance.
In Glance, an image can either be **stored** in Glance, or it can be
**registered** in Glance but actually be stored somewhere else.
We needed a class that could support the various ways that Glance
describes where exactly an image is stored.
An image in Glance has two location properties: the image URI
and the image storage URI.
The image URI is essentially the permalink identifier for the image.
It is displayed in the output of various Glance API calls and,
while read-only, is entirely user-facing. It shall **not** contain any
security credential information at all. The Glance image URI shall
be the host:port of that Glance API server along with /images/<IMAGE_ID>.
The Glance storage URI is an internal URI structure that Glance
uses to maintain critical information about how to access the images
that it stores in its storage backends. It **does contain** security
credentials and is **not** user-facing.
"""
import logging
import urlparse
from glance.common import exception
from glance.common import utils
logger = logging.getLogger('glance.store.location')
SCHEME_TO_STORE_MAP = {}
def get_location_from_uri(uri):
"""
Given a URI, return a Location object that has had an appropriate
store parse the URI.
:param uri: A URI that could come from the end-user in the Location
attribute/header
Example URIs:
https://user:pass@example.com:80/images/some-id
http://images.oracle.com/123456
swift://user:account:pass@authurl.com/container/obj-id
swift+http://user:account:pass@authurl.com/container/obj-id
s3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id
s3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id
file:///var/lib/glance/images/1
"""
# Add known stores to mapping... this gets past circular import
# issues. There's a better way to do this, but that's for another
# patch...
# TODO(jaypipes) Clear up these imports in refactor-stores blueprint
import glance.store.filesystem
import glance.store.http
import glance.store.s3
import glance.store.swift
pieces = urlparse.urlparse(uri)
if pieces.scheme not in SCHEME_TO_STORE_MAP.keys():
raise exception.UnknownScheme(pieces.scheme)
loc = Location(pieces.scheme, uri=uri)
return loc
def add_scheme_map(scheme_map):
"""
Given a mapping of 'scheme' to store_name, adds the mapping to the
known list of schemes.
Each store should call this method and let Glance know about which
schemes to map to a store name.
"""
SCHEME_TO_STORE_MAP.update(scheme_map)
class Location(object):
"""
Class describing the location of an image that Glance knows about
"""
def __init__(self, store_name, uri=None, image_id=None, store_specs=None):
"""
Create a new Location object.
:param store_name: The string identifier of the storage backend
:param image_id: The identifier of the image in whatever storage
backend is used.
:param uri: Optional URI to construct location from
:param store_specs: Dictionary of information about the location
of the image that is dependent on the backend
store
"""
self.store_name = store_name
self.image_id = image_id
self.store_specs = store_specs or {}
self.store_location = self._get_store_location()
if uri:
self.store_location.parse_uri(uri)
def _get_store_location(self):
"""
We find the store module and then grab an instance of the store's
StoreLocation class which handles store-specific location information
"""
try:
cls = utils.import_class('glance.store.%s.StoreLocation'
% SCHEME_TO_STORE_MAP[self.store_name])
return cls(self.store_specs)
except exception.NotFound:
logger.error("Unable to find StoreLocation class in store %s",
self.store_name)
return None
def get_store_uri(self):
"""
Returns the Glance image URI, which is the host:port of the API server
along with /images/<IMAGE_ID>
"""
return self.store_location.get_uri()
def get_uri(self):
return None
class StoreLocation(object):
"""
Base class that must be implemented by each store
"""
def __init__(self, store_specs):
self.specs = store_specs
if self.specs:
self.process_specs()
def process_specs(self):
"""
Subclasses should implement any processing of the self.specs collection
such as storing credentials and possibly establishing connections.
"""
pass
def get_uri(self):
"""
Subclasses should implement a method that returns an internal URI that,
when supplied to the StoreLocation instance, can be interpreted by the
StoreLocation's parse_uri() method. The URI returned from this method
shall never be public and only used internally within Glance, so it is
fine to encode credentials in this URI.
"""
raise NotImplementedError("StoreLocation subclass must implement "
"get_uri()")
def parse_uri(self, uri):
"""
Subclasses should implement a method that accepts a string URI and
sets appropriate internal fields such that a call to get_uri() will
return a proper internal URI
"""
raise NotImplementedError("StoreLocation subclass must implement "
"parse_uri()")

144
glance/store/s3.py
View File

@ -17,7 +17,101 @@
"""The s3 backend adapter""" """The s3 backend adapter"""
import urlparse
from glance.common import exception
import glance.store import glance.store
import glance.store.location
glance.store.location.add_scheme_map({'s3': 's3',
's3+http': 's3',
's3+https': 's3'})
class StoreLocation(glance.store.location.StoreLocation):
"""
Class describing an S3 URI. An S3 URI can look like any of
the following:
s3://accesskey:secretkey@s3service.com/bucket/key-id
s3+http://accesskey:secretkey@s3service.com/bucket/key-id
s3+https://accesskey:secretkey@s3service.com/bucket/key-id
The s3+https:// URIs indicate there is an HTTPS s3service URL
"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 's3')
self.accesskey = self.specs.get('accesskey')
self.secretkey = self.specs.get('secretkey')
self.s3serviceurl = self.specs.get('s3serviceurl')
self.bucket = self.specs.get('bucket')
self.key = self.specs.get('key')
def _get_credstring(self):
if self.accesskey:
return '%s:%s@' % (self.accesskey, self.secretkey)
return ''
def get_uri(self):
return "%s://%s%s/%s/%s" % (
self.scheme,
self._get_credstring(),
self.s3serviceurl,
self.bucket,
self.key)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
Note that an Amazon AWS secret key can contain the forward slash,
which is entirely retarded, and breaks urlparse miserably.
This function works around that issue.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('s3', 's3+http', 's3+https')
self.scheme = pieces.scheme
path = pieces.path.strip('/')
netloc = pieces.netloc.strip('/')
entire_path = (netloc + '/' + path).strip('/')
if '@' in uri:
creds, path = entire_path.split('@')
cred_parts = creds.split(':')
try:
access_key = cred_parts[0]
secret_key = cred_parts[1]
# NOTE(jaypipes): Need to encode to UTF-8 here because of a
# bug in the HMAC library that boto uses.
# See: http://bugs.python.org/issue5285
# See: http://trac.edgewall.org/ticket/8083
access_key = access_key.encode('utf-8')
secret_key = secret_key.encode('utf-8')
self.accesskey = access_key
self.secretkey = secret_key
except IndexError:
reason = "Badly formed S3 credentials %s" % creds
raise exception.BadStoreUri(uri, reason)
else:
self.accesskey = None
path = entire_path
try:
path_parts = path.split('/')
self.key = path_parts.pop()
self.bucket = path_parts.pop()
if len(path_parts) > 0:
self.s3serviceurl = '/'.join(path_parts)
else:
reason = "Badly formed S3 URI. Missing s3 service URL."
raise exception.BadStoreUri(uri, reason)
except IndexError:
reason = "Badly formed S3 URI"
raise exception.BadStoreUri(uri, reason)
class S3Backend(glance.store.Backend): class S3Backend(glance.store.Backend):
@ -26,29 +120,30 @@ class S3Backend(glance.store.Backend):
EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0" EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0"
@classmethod @classmethod
def get(cls, parsed_uri, expected_size, conn_class=None): def get(cls, location, expected_size, conn_class=None):
"""
Takes a parsed_uri in the format of:
s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects
to s3 and downloads the file. Returns the generator resp_body provided
by get_object.
""" """
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator from S3
provided by S3's key object
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
"""
if conn_class: if conn_class:
pass pass
else: else:
import boto.s3.connection import boto.s3.connection
conn_class = boto.s3.connection.S3Connection conn_class = boto.s3.connection.S3Connection
(access_key, secret_key, host, bucket, obj) = \ loc = location.store_location
cls._parse_s3_tokens(parsed_uri)
# Close the connection when we're through. # Close the connection when we're through.
with conn_class(access_key, secret_key, host=host) as s3_conn: with conn_class(loc.accesskey, loc.secretkey,
bucket = cls._get_bucket(s3_conn, bucket) host=loc.s3serviceurl) as s3_conn:
bucket = cls._get_bucket(s3_conn, loc.bucket)
# Close the key when we're through. # Close the key when we're through.
with cls._get_key(bucket, obj) as key: with cls._get_key(bucket, loc.obj) as key:
if not key.size == expected_size: if not key.size == expected_size:
raise glance.store.BackendException( raise glance.store.BackendException(
"Expected %s bytes, got %s" % "Expected %s bytes, got %s" %
@ -59,28 +154,28 @@ class S3Backend(glance.store.Backend):
yield chunk yield chunk
@classmethod @classmethod
def delete(cls, parsed_uri, conn_class=None): def delete(cls, location, conn_class=None):
"""
Takes a parsed_uri in the format of:
s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects
to s3 and deletes the file. Returns whatever boto.s3.key.Key.delete()
returns.
""" """
Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
"""
if conn_class: if conn_class:
pass pass
else: else:
conn_class = boto.s3.connection.S3Connection conn_class = boto.s3.connection.S3Connection
(access_key, secret_key, host, bucket, obj) = \ loc = location.store_location
cls._parse_s3_tokens(parsed_uri)
# Close the connection when we're through. # Close the connection when we're through.
with conn_class(access_key, secret_key, host=host) as s3_conn: with conn_class(loc.accesskey, loc.secretkey,
bucket = cls._get_bucket(s3_conn, bucket) host=loc.s3serviceurl) as s3_conn:
bucket = cls._get_bucket(s3_conn, loc.bucket)
# Close the key when we're through. # Close the key when we're through.
with cls._get_key(bucket, obj) as key: with cls._get_key(bucket, loc.obj) as key:
return key.delete() return key.delete()
@classmethod @classmethod
@ -102,8 +197,3 @@ class S3Backend(glance.store.Backend):
if not key: if not key:
raise glance.store.BackendException("Could not get key: %s" % key) raise glance.store.BackendException("Could not get key: %s" % key)
return key return key
@classmethod
def _parse_s3_tokens(cls, parsed_uri):
"""Parse tokens from the parsed_uri"""
return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL)

226
glance/store/swift.py
View File

@ -21,15 +21,114 @@ from __future__ import absolute_import
import httplib import httplib
import logging import logging
import urlparse
from glance.common import config from glance.common import config
from glance.common import exception from glance.common import exception
import glance.store import glance.store
import glance.store.location
DEFAULT_SWIFT_CONTAINER = 'glance' DEFAULT_SWIFT_CONTAINER = 'glance'
logger = logging.getLogger('glance.store.swift') logger = logging.getLogger('glance.store.swift')
glance.store.location.add_scheme_map({'swift': 'swift',
'swift+http': 'swift',
'swift+https': 'swift'})
class StoreLocation(glance.store.location.StoreLocation):
"""
Class describing a Swift URI. A Swift URI can look like any of
the following:
swift://user:pass@authurl.com/container/obj-id
swift+http://user:pass@authurl.com/container/obj-id
swift+https://user:pass@authurl.com/container/obj-id
The swift+https:// URIs indicate there is an HTTPS authentication URL
"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 'swift+https')
self.user = self.specs.get('user')
self.key = self.specs.get('key')
self.authurl = self.specs.get('authurl')
self.container = self.specs.get('container')
self.obj = self.specs.get('obj')
def _get_credstring(self):
if self.user:
return '%s:%s@' % (self.user, self.key)
return ''
def get_uri(self):
return "%s://%s%s/%s/%s" % (
self.scheme,
self._get_credstring(),
self.authurl,
self.container,
self.obj)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python. It also deals with the peculiarity that new-style
Swift URIs have where a username can contain a ':', like so:
swift://account:user:pass@authurl.com/container/obj
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path.lstrip('/')
if netloc != '':
# > Python 2.6.1
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
else:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
netloc = path[0:path.find('/')].strip('/')
path = path[path.find('/'):].strip('/')
if creds:
cred_parts = creds.split(':')
# User can be account:user, in which case cred_parts[0:2] will be
# the account and user. Combine them into a single username of
# account:user
if len(cred_parts) == 1:
reason = "Badly formed credentials '%s' in Swift URI" % creds
raise exception.BadStoreUri(uri, reason)
elif len(cred_parts) == 3:
user = ':'.join(cred_parts[0:2])
else:
user = cred_parts[0]
key = cred_parts[-1]
self.user = user
self.key = key
else:
self.user = None
path_parts = path.split('/')
try:
self.obj = path_parts.pop()
self.container = path_parts.pop()
self.authurl = netloc
if len(path_parts) > 0:
self.authurl = netloc + '/' + '/'.join(path_parts).strip('/')
except IndexError:
reason = "Badly formed Swift URI"
raise exception.BadStoreUri(uri, reason)
class SwiftBackend(glance.store.Backend): class SwiftBackend(glance.store.Backend):
"""An implementation of the swift backend adapter.""" """An implementation of the swift backend adapter."""
@ -39,31 +138,33 @@ class SwiftBackend(glance.store.Backend):
CHUNKSIZE = 65536 CHUNKSIZE = 65536
@classmethod @classmethod
def get(cls, parsed_uri, expected_size=None, options=None): def get(cls, location, expected_size=None, options=None):
""" """
Takes a parsed_uri in the format of: Takes a `glance.store.location.Location` object that indicates
swift://user:password@auth_url/container/file.gz.0, connects to the where to find the image file, and returns a generator from Swift
swift instance at auth_url and downloads the file. Returns the provided by Swift client's get_object() method.
generator resp_body provided by get_object.
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
""" """
from swift.common import client as swift_client from swift.common import client as swift_client
(user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri)
# TODO(sirp): snet=False for now, however, if the instance of # TODO(sirp): snet=False for now, however, if the instance of
# swift we're talking to is within our same region, we should set # swift we're talking to is within our same region, we should set
# snet=True # snet=True
loc = location.store_location
swift_conn = swift_client.Connection( swift_conn = swift_client.Connection(
authurl=authurl, user=user, key=key, snet=False) authurl=loc.authurl, user=loc.user, key=loc.key, snet=False)
try: try:
(resp_headers, resp_body) = swift_conn.get_object( (resp_headers, resp_body) = swift_conn.get_object(
container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE) container=loc.container, obj=loc.obj,
resp_chunk_size=cls.CHUNKSIZE)
except swift_client.ClientException, e: except swift_client.ClientException, e:
if e.http_status == httplib.NOT_FOUND: if e.http_status == httplib.NOT_FOUND:
location = format_swift_location(user, key, authurl, uri = location.get_store_uri()
container, obj)
raise exception.NotFound("Swift could not find image at " raise exception.NotFound("Swift could not find image at "
"location %(location)s" % locals()) "uri %(uri)s" % locals())
if expected_size: if expected_size:
obj_size = int(resp_headers['content-length']) obj_size = int(resp_headers['content-length'])
@ -98,6 +199,10 @@ class SwiftBackend(glance.store.Backend):
<CONTAINER> = ``swift_store_container`` <CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added <ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:param id: The opaque image identifier :param id: The opaque image identifier
:param data: The image data to write, as a file-like object :param data: The image data to write, as a file-like object
:param options: Conf mapping :param options: Conf mapping
@ -119,9 +224,14 @@ class SwiftBackend(glance.store.Backend):
user = cls._option_get(options, 'swift_store_user') user = cls._option_get(options, 'swift_store_user')
key = cls._option_get(options, 'swift_store_key') key = cls._option_get(options, 'swift_store_key')
full_auth_address = auth_address scheme = 'swift+https'
if not full_auth_address.startswith('http'): if auth_address.startswith('http://'):
full_auth_address = 'https://' + full_auth_address scheme = 'swift+http'
full_auth_address = auth_address
elif auth_address.startswith('https://'):
full_auth_address = auth_address
else:
full_auth_address = 'https://' + auth_address # Defaults https
swift_conn = swift_client.Connection( swift_conn = swift_client.Connection(
authurl=full_auth_address, user=user, key=key, snet=False) authurl=full_auth_address, user=user, key=key, snet=False)
@ -133,8 +243,13 @@ class SwiftBackend(glance.store.Backend):
create_container_if_missing(container, swift_conn, options) create_container_if_missing(container, swift_conn, options)
obj_name = str(id) obj_name = str(id)
location = format_swift_location(user, key, auth_address, location = StoreLocation({'scheme': scheme,
container, obj_name) 'container': container,
'obj': obj_name,
'authurl': auth_address,
'user': user,
'key': key})
try: try:
obj_etag = swift_conn.put_object(container, obj_name, data) obj_etag = swift_conn.put_object(container, obj_name, data)
@ -152,7 +267,7 @@ class SwiftBackend(glance.store.Backend):
# header keys are lowercased by Swift # header keys are lowercased by Swift
if 'content-length' in resp_headers: if 'content-length' in resp_headers:
size = int(resp_headers['content-length']) size = int(resp_headers['content-length'])
return (location, size, obj_etag) return (location.get_uri(), size, obj_etag)
except swift_client.ClientException, e: except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT: if e.http_status == httplib.CONFLICT:
raise exception.Duplicate("Swift already has an image at " raise exception.Duplicate("Swift already has an image at "
@ -162,89 +277,34 @@ class SwiftBackend(glance.store.Backend):
raise glance.store.BackendException(msg) raise glance.store.BackendException(msg)
@classmethod @classmethod
def delete(cls, parsed_uri): def delete(cls, location):
""" """
Deletes the swift object(s) at the parsed_uri location Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
""" """
from swift.common import client as swift_client from swift.common import client as swift_client
(user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri)
# TODO(sirp): snet=False for now, however, if the instance of # TODO(sirp): snet=False for now, however, if the instance of
# swift we're talking to is within our same region, we should set # swift we're talking to is within our same region, we should set
# snet=True # snet=True
loc = location.store_location
swift_conn = swift_client.Connection( swift_conn = swift_client.Connection(
authurl=authurl, user=user, key=key, snet=False) authurl=loc.authurl, user=loc.user, key=loc.key, snet=False)
try: try:
swift_conn.delete_object(container, obj) swift_conn.delete_object(loc.container, loc.obj)
except swift_client.ClientException, e: except swift_client.ClientException, e:
if e.http_status == httplib.NOT_FOUND: if e.http_status == httplib.NOT_FOUND:
location = format_swift_location(user, key, authurl, uri = location.get_store_uri()
container, obj)
raise exception.NotFound("Swift could not find image at " raise exception.NotFound("Swift could not find image at "
"location %(location)s" % locals()) "uri %(uri)s" % locals())
else: else:
raise raise
def parse_swift_tokens(parsed_uri):
"""
Return the various tokens used by Swift.
:param parsed_uri: The pieces of a URI returned by urlparse
:retval A tuple of (user, key, auth_address, container, obj_name)
"""
path = parsed_uri.path.lstrip('//')
netloc = parsed_uri.netloc
try:
try:
creds, netloc = netloc.split('@')
path = '/'.join([netloc, path])
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
creds, path = path.split('@')
cred_parts = creds.split(':')
# User can be account:user, in which case cred_parts[0:2] will be
# the account and user. Combine them into a single username of
# account:user
if len(cred_parts) == 3:
user = ':'.join(cred_parts[0:2])
else:
user = cred_parts[0]
key = cred_parts[-1]
path_parts = path.split('/')
obj = path_parts.pop()
container = path_parts.pop()
except (ValueError, IndexError):
raise glance.store.BackendException(
"Expected four values to unpack in: swift:%s. "
"Should have received something like: %s."
% (parsed_uri.path, SwiftBackend.EXAMPLE_URL))
authurl = "https://%s" % '/'.join(path_parts)
return user, key, authurl, container, obj
def format_swift_location(user, key, auth_address, container, obj_name):
"""
Returns the swift URI in the format:
swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<OBJNAME>
:param user: The swift user to authenticate with
:param key: The auth key for the authenticating user
:param auth_address: The base URL for the authentication service
:param container: The name of the container
:param obj_name: The name of the object
"""
return "swift://%(user)s:%(key)s@%(auth_address)s/"\
"%(container)s/%(obj_name)s" % locals()
def create_container_if_missing(container, swift_conn, options): def create_container_if_missing(container, swift_conn, options):
""" """
Creates a missing container in Swift if the Creates a missing container in Swift if the

2
tests/functional/test_curl_api.py
View File

@ -1227,3 +1227,5 @@ class TestCurlApi(functional.FunctionalTest):
self.assertEqual(images[0]['id'], 1) self.assertEqual(images[0]['id'], 1)
self.assertEqual(images[1]['id'], 3) self.assertEqual(images[1]['id'], 3)
self.assertEqual(images[2]['id'], 2) self.assertEqual(images[2]['id'], 2)
self.stop_servers()

486
tests/functional/test_httplib2_api.py
View File

@ -21,6 +21,7 @@ import hashlib
import httplib2 import httplib2
import json import json
import os import os
import tempfile
from tests import functional from tests import functional
from tests.utils import execute from tests.utils import execute
@ -590,3 +591,488 @@ class TestApiHttplib2(functional.FunctionalTest):
self.assertEqual(response['x-image-meta-is_public'], 'True') self.assertEqual(response['x-image-meta-is_public'], 'True')
self.stop_servers() self.stop_servers()
def test_traceback_not_consumed(self):
"""
A test that errors coming from the POST API do not
get consumed and print the actual error message, and
not something like &lt;traceback object at 0x1918d40&gt;
:see https://bugs.launchpad.net/glance/+bug/755912
"""
self.cleanup()
self.start_servers()
# POST /images with binary data, but not setting
# Content-Type to application/octet-stream, verify a
# 400 returned and that the error is readable.
with tempfile.NamedTemporaryFile() as test_data_file:
test_data_file.write("XXX")
test_data_file.flush()
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST',
body=test_data_file.name)
self.assertEqual(response.status, 400)
expected = "Content-Type must be application/octet-stream"
self.assertTrue(expected in content,
"Could not find '%s' in '%s'" % (expected, content))
self.stop_servers()
def test_filtered_images(self):
"""
Set up four test images and ensure each query param filter works
"""
self.cleanup()
self.start_servers()
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with three public images, and one private image
# with various attributes
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vdi',
'X-Image-Meta-Size': '19',
'X-Image-Meta-Is-Public': 'True',
'X-Image-Meta-Property-pants': 'are on'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['properties']['pants'], "are on")
self.assertEqual(data['image']['is_public'], True)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'My Image!',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vhd',
'X-Image-Meta-Size': '20',
'X-Image-Meta-Is-Public': 'True',
'X-Image-Meta-Property-pants': 'are on'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['properties']['pants'], "are on")
self.assertEqual(data['image']['is_public'], True)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'My Image!',
'X-Image-Meta-Status': 'saving',
'X-Image-Meta-Container-Format': 'ami',
'X-Image-Meta-Disk-Format': 'ami',
'X-Image-Meta-Size': '21',
'X-Image-Meta-Is-Public': 'True',
'X-Image-Meta-Property-pants': 'are off'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['properties']['pants'], "are off")
self.assertEqual(data['image']['is_public'], True)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'My Private Image',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ami',
'X-Image-Meta-Disk-Format': 'ami',
'X-Image-Meta-Size': '22',
'X-Image-Meta-Is-Public': 'False'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['is_public'], False)
# 2. GET /images
# Verify three public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
# 3. GET /images with name filter
# Verify correct images returned with name
params = "name=My%20Image!"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertEqual(image['name'], "My Image!")
# 4. GET /images with status filter
# Verify correct images returned with status
params = "status=queued"
path = "http://%s:%d/v1/images/detail?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
for image in data['images']:
self.assertEqual(image['status'], "queued")
params = "status=active"
path = "http://%s:%d/v1/images/detail?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 0)
# 5. GET /images with container_format filter
# Verify correct images returned with container_format
params = "container_format=ovf"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertEqual(image['container_format'], "ovf")
# 6. GET /images with disk_format filter
# Verify correct images returned with disk_format
params = "disk_format=vdi"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
for image in data['images']:
self.assertEqual(image['disk_format'], "vdi")
# 7. GET /images with size_max filter
# Verify correct images returned with size <= expected
params = "size_max=20"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertTrue(image['size'] <= 20)
# 8. GET /images with size_min filter
# Verify correct images returned with size >= expected
params = "size_min=20"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertTrue(image['size'] >= 20)
# 9. Get /images with is_public=None filter
# Verify correct images returned with property
# Bug lp:803656 Support is_public in filtering
params = "is_public=None"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 4)
# 10. Get /images with is_public=False filter
# Verify correct images returned with property
# Bug lp:803656 Support is_public in filtering
params = "is_public=False"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
for image in data['images']:
self.assertEqual(image['name'], "My Private Image")
# 11. Get /images with is_public=True filter
# Verify correct images returned with property
# Bug lp:803656 Support is_public in filtering
params = "is_public=True"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
for image in data['images']:
self.assertNotEqual(image['name'], "My Private Image")
# 12. GET /images with property filter
# Verify correct images returned with property
params = "property-pants=are%20on"
path = "http://%s:%d/v1/images/detail?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertEqual(image['properties']['pants'], "are on")
# 13. GET /images with property filter and name filter
# Verify correct images returned with property and name
# Make sure you quote the url when using more than one param!
params = "name=My%20Image!&property-pants=are%20on"
path = "http://%s:%d/v1/images/detail?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
for image in data['images']:
self.assertEqual(image['properties']['pants'], "are on")
self.assertEqual(image['name'], "My Image!")
self.stop_servers()
def test_limited_images(self):
"""
Ensure marker and limit query params work
"""
self.cleanup()
self.start_servers()
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with three public images with various attributes
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image2',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image3',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
# 2. GET /images with limit of 2
# Verify only two images were returned
params = "limit=2"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
self.assertEqual(data['images'][0]['id'], 3)
self.assertEqual(data['images'][1]['id'], 2)
# 3. GET /images with marker
# Verify only two images were returned
params = "marker=3"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
self.assertEqual(data['images'][0]['id'], 2)
self.assertEqual(data['images'][1]['id'], 1)
# 4. GET /images with marker and limit
# Verify only one image was returned with the correct id
params = "limit=1&marker=2"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
self.assertEqual(data['images'][0]['id'], 1)
# 5. GET /images/detail with marker and limit
# Verify only one image was returned with the correct id
params = "limit=1&marker=3"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
self.assertEqual(data['images'][0]['id'], 2)
self.stop_servers()
def test_ordered_images(self):
"""
Set up three test images and ensure each query param filter works
"""
self.cleanup()
self.start_servers()
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with three public images with various attributes
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vdi',
'X-Image-Meta-Size': '19',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'ASDF',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'bare',
'X-Image-Meta-Disk-Format': 'iso',
'X-Image-Meta-Size': '2',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'XYZ',
'X-Image-Meta-Status': 'saving',
'X-Image-Meta-Container-Format': 'ami',
'X-Image-Meta-Disk-Format': 'ami',
'X-Image-Meta-Size': '5',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
# 2. GET /images with no query params
# Verify three public images sorted by created_at desc
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
self.assertEqual(data['images'][0]['id'], 3)
self.assertEqual(data['images'][1]['id'], 2)
self.assertEqual(data['images'][2]['id'], 1)
# 3. GET /images sorted by name asc
params = 'sort_key=name&sort_dir=asc'
path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
self.assertEqual(data['images'][0]['id'], 2)
self.assertEqual(data['images'][1]['id'], 1)
self.assertEqual(data['images'][2]['id'], 3)
# 4. GET /images sorted by size desc
params = 'sort_key=size&sort_dir=desc'
path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
self.assertEqual(data['images'][0]['id'], 1)
self.assertEqual(data['images'][1]['id'], 3)
self.assertEqual(data['images'][2]['id'], 2)
self.stop_servers()
def test_duplicate_image_upload(self):
"""
Upload initial image, then attempt to upload duplicate image
"""
self.cleanup()
self.start_servers()
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with public image named Image1
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vdi',
'X-Image-Meta-Size': '19',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
# 2. POST /images with public image named Image1, and ID: 1
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1 Update',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vdi',
'X-Image-Meta-Size': '19',
'X-Image-Meta-Id': '1',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 409)
expected = "An image with identifier 1 already exists"
self.assertTrue(expected in content,
"Could not find '%s' in '%s'" % (expected, content))
self.stop_servers()

19
tests/stubs.py
View File

@ -128,13 +128,9 @@ def stub_out_s3_backend(stubs):
DATA = 'I am a teapot, short and stout\n' DATA = 'I am a teapot, short and stout\n'
@classmethod @classmethod
def get(cls, parsed_uri, expected_size, conn_class=None): def get(cls, location, expected_size, conn_class=None):
S3Backend = glance.store.s3.S3Backend S3Backend = glance.store.s3.S3Backend
# raise BackendException if URI is bad.
(user, key, authurl, container, obj) = \
S3Backend._parse_s3_tokens(parsed_uri)
def chunk_it(): def chunk_it():
for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE): for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE):
yield cls.DATA[i:i + cls.CHUNK_SIZE] yield cls.DATA[i:i + cls.CHUNK_SIZE]
@ -400,9 +396,9 @@ def stub_out_registry_db_image_api(stubs):
f['deleted_at'] <= delete_time] f['deleted_at'] <= delete_time]
return images return images
def image_get_all_public(self, _context, filters=None, marker=None, def image_get_all(self, _context, filters=None, marker=None,
limit=1000, sort_key=None, sort_dir=None): limit=1000, sort_key=None, sort_dir=None):
images = [f for f in self.images if f['is_public'] == True] images = self.images
if 'size_min' in filters: if 'size_min' in filters:
size_min = int(filters.pop('size_min')) size_min = int(filters.pop('size_min'))
@ -424,7 +420,8 @@ def stub_out_registry_db_image_api(stubs):
images = filter(_prop_filter(k, v), images) images = filter(_prop_filter(k, v), images)
for k, v in filters.items(): for k, v in filters.items():
images = [f for f in images if f[k] == v] if v is not None:
images = [f for f in images if f[k] == v]
# sorted func expects func that compares in descending order # sorted func expects func that compares in descending order
def image_cmp(x, y): def image_cmp(x, y):
@ -473,5 +470,5 @@ def stub_out_registry_db_image_api(stubs):
fake_datastore.image_get) fake_datastore.image_get)
stubs.Set(glance.registry.db.api, 'image_get_all_pending_delete', stubs.Set(glance.registry.db.api, 'image_get_all_pending_delete',
fake_datastore.image_get_all_pending_delete) fake_datastore.image_get_all_pending_delete)
stubs.Set(glance.registry.db.api, 'image_get_all_public', stubs.Set(glance.registry.db.api, 'image_get_all',
fake_datastore.image_get_all_public) fake_datastore.image_get_all)

78
tests/unit/test_api.py
View File

@ -1077,6 +1077,84 @@ class TestRegistryAPI(unittest.TestCase):
for image in images: for image in images:
self.assertEqual('v a', image['properties']['prop_123']) self.assertEqual('v a', image['properties']['prop_123'])
def test_get_details_filter_public_none(self):
"""
Tests that the /images/detail registry API returns list of
all images if is_public none is passed
"""
extra_fixture = {'id': 3,
'status': 'active',
'is_public': False,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
glance.registry.db.api.image_create(None, extra_fixture)
req = webob.Request.blank('/images/detail?is_public=None')
res = req.get_response(self.api)
res_dict = json.loads(res.body)
self.assertEquals(res.status_int, 200)
images = res_dict['images']
self.assertEquals(len(images), 3)
def test_get_details_filter_public_false(self):
"""
Tests that the /images/detail registry API returns list of
private images if is_public false is passed
"""
extra_fixture = {'id': 3,
'status': 'active',
'is_public': False,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
glance.registry.db.api.image_create(None, extra_fixture)
req = webob.Request.blank('/images/detail?is_public=False')
res = req.get_response(self.api)
res_dict = json.loads(res.body)
self.assertEquals(res.status_int, 200)
images = res_dict['images']
self.assertEquals(len(images), 2)
for image in images:
self.assertEqual(False, image['is_public'])
def test_get_details_filter_public_true(self):
"""
Tests that the /images/detail registry API returns list of
public images if is_public true is passed (same as default)
"""
extra_fixture = {'id': 3,
'status': 'active',
'is_public': False,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
glance.registry.db.api.image_create(None, extra_fixture)
req = webob.Request.blank('/images/detail?is_public=True')
res = req.get_response(self.api)
res_dict = json.loads(res.body)
self.assertEquals(res.status_int, 200)
images = res_dict['images']
self.assertEquals(len(images), 1)
for image in images:
self.assertEqual(True, image['is_public'])
def test_get_details_sort_name_asc(self): def test_get_details_sort_name_asc(self):
""" """
Tests that the /images/details registry API returns list of Tests that the /images/details registry API returns list of

25
tests/unit/test_filesystem_store.py
View File

@ -20,11 +20,11 @@
import StringIO import StringIO
import hashlib import hashlib
import unittest import unittest
import urlparse
import stubout import stubout
from glance.common import exception from glance.common import exception
from glance.store.location import get_location_from_uri
from glance.store.filesystem import FilesystemBackend, ChunkedFile from glance.store.filesystem import FilesystemBackend, ChunkedFile
from tests import stubs from tests import stubs
@ -51,8 +51,8 @@ class TestFilesystemBackend(unittest.TestCase):
def test_get(self): def test_get(self):
"""Test a "normal" retrieval of an image in chunks""" """Test a "normal" retrieval of an image in chunks"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2") loc = get_location_from_uri("file:///tmp/glance-tests/2")
image_file = FilesystemBackend.get(url_pieces) image_file = FilesystemBackend.get(loc)
expected_data = "chunk00000remainder" expected_data = "chunk00000remainder"
expected_num_chunks = 2 expected_num_chunks = 2
@ -70,10 +70,10 @@ class TestFilesystemBackend(unittest.TestCase):
Test that trying to retrieve a file that doesn't exist Test that trying to retrieve a file that doesn't exist
raises an error raises an error
""" """
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing") loc = get_location_from_uri("file:///tmp/glance-tests/non-existing")
self.assertRaises(exception.NotFound, self.assertRaises(exception.NotFound,
FilesystemBackend.get, FilesystemBackend.get,
url_pieces) loc)
def test_add(self): def test_add(self):
"""Test that we can add an image via the filesystem backend""" """Test that we can add an image via the filesystem backend"""
@ -93,8 +93,8 @@ class TestFilesystemBackend(unittest.TestCase):
self.assertEquals(expected_file_size, size) self.assertEquals(expected_file_size, size)
self.assertEquals(expected_checksum, checksum) self.assertEquals(expected_checksum, checksum)
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/42") loc = get_location_from_uri("file:///tmp/glance-tests/42")
new_image_file = FilesystemBackend.get(url_pieces) new_image_file = FilesystemBackend.get(loc)
new_image_contents = "" new_image_contents = ""
new_image_file_size = 0 new_image_file_size = 0
@ -122,20 +122,19 @@ class TestFilesystemBackend(unittest.TestCase):
""" """
Test we can delete an existing image in the filesystem store Test we can delete an existing image in the filesystem store
""" """
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2") loc = get_location_from_uri("file:///tmp/glance-tests/2")
FilesystemBackend.delete(loc)
FilesystemBackend.delete(url_pieces)
self.assertRaises(exception.NotFound, self.assertRaises(exception.NotFound,
FilesystemBackend.get, FilesystemBackend.get,
url_pieces) loc)
def test_delete_non_existing(self): def test_delete_non_existing(self):
""" """
Test that trying to delete a file that doesn't exist Test that trying to delete a file that doesn't exist
raises an error raises an error
""" """
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing") loc = get_location_from_uri("file:///tmp/glance-tests/non-existing")
self.assertRaises(exception.NotFound, self.assertRaises(exception.NotFound,
FilesystemBackend.delete, FilesystemBackend.delete,
url_pieces) loc)

243
tests/unit/test_store_location.py Normal file
View File

@ -0,0 +1,243 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from glance.common import exception
import glance.store.location as location
import glance.store.http
import glance.store.filesystem
import glance.store.swift
import glance.store.s3
class TestStoreLocation(unittest.TestCase):
def test_get_location_from_uri_back_to_uri(self):
"""
Test that for various URIs, the correct Location
object can be contructed and then the original URI
returned via the get_store_uri() method.
"""
good_store_uris = [
'https://user:pass@example.com:80/images/some-id',
'http://images.oracle.com/123456',
'swift://account:user:pass@authurl.com/container/obj-id',
'swift+https://account:user:pass@authurl.com/container/obj-id',
's3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id',
's3://accesskey:secretwith/aslash@s3.amazonaws.com/bucket/key-id',
's3+http://accesskey:secret@s3.amazonaws.com/bucket/key-id',
's3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id',
'file:///var/lib/glance/images/1']
for uri in good_store_uris:
loc = location.get_location_from_uri(uri)
# The get_store_uri() method *should* return an identical URI
# to the URI that is passed to get_location_from_uri()
self.assertEqual(loc.get_store_uri(), uri)
def test_bad_store_scheme(self):
"""
Test that a URI with a non-existing scheme triggers exception
"""
bad_uri = 'unknown://user:pass@example.com:80/images/some-id'
self.assertRaises(exception.UnknownScheme,
location.get_location_from_uri,
bad_uri)
def test_filesystem_store_location(self):
"""
Test the specific StoreLocation for the Filesystem store
"""
uri = 'file:///var/lib/glance/images/1'
loc = glance.store.filesystem.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("file", loc.scheme)
self.assertEqual("/var/lib/glance/images/1", loc.path)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'fil://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 'file://'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
def test_http_store_location(self):
"""
Test the specific StoreLocation for the HTTP store
"""
uri = 'http://example.com/images/1'
loc = glance.store.http.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("http", loc.scheme)
self.assertEqual("example.com", loc.netloc)
self.assertEqual("/images/1", loc.path)
self.assertEqual(uri, loc.get_uri())
uri = 'https://example.com:8080/images/container/1'
loc.parse_uri(uri)
self.assertEqual("https", loc.scheme)
self.assertEqual("example.com:8080", loc.netloc)
self.assertEqual("/images/container/1", loc.path)
self.assertEqual(uri, loc.get_uri())
uri = 'https://user:password@example.com:8080/images/container/1'
loc.parse_uri(uri)
self.assertEqual("https", loc.scheme)
self.assertEqual("example.com:8080", loc.netloc)
self.assertEqual("user", loc.user)
self.assertEqual("password", loc.password)
self.assertEqual("/images/container/1", loc.path)
self.assertEqual(uri, loc.get_uri())
uri = 'https://user:@example.com:8080/images/1'
loc.parse_uri(uri)
self.assertEqual("https", loc.scheme)
self.assertEqual("example.com:8080", loc.netloc)
self.assertEqual("user", loc.user)
self.assertEqual("", loc.password)
self.assertEqual("/images/1", loc.path)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'htt://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 'http://'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
bad_uri = 'http://user@example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
def test_swift_store_location(self):
"""
Test the specific StoreLocation for the Swift store
"""
uri = 'swift://example.com/images/1'
loc = glance.store.swift.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("swift", loc.scheme)
self.assertEqual("example.com", loc.authurl)
self.assertEqual("images", loc.container)
self.assertEqual("1", loc.obj)
self.assertEqual(None, loc.user)
self.assertEqual(uri, loc.get_uri())
uri = 'swift+https://user:pass@authurl.com/images/1'
loc.parse_uri(uri)
self.assertEqual("swift+https", loc.scheme)
self.assertEqual("authurl.com", loc.authurl)
self.assertEqual("images", loc.container)
self.assertEqual("1", loc.obj)
self.assertEqual("user", loc.user)
self.assertEqual("pass", loc.key)
self.assertEqual(uri, loc.get_uri())
uri = 'swift+https://user:pass@authurl.com/v1/container/12345'
loc.parse_uri(uri)
self.assertEqual("swift+https", loc.scheme)
self.assertEqual("authurl.com/v1", loc.authurl)
self.assertEqual("container", loc.container)
self.assertEqual("12345", loc.obj)
self.assertEqual("user", loc.user)
self.assertEqual("pass", loc.key)
self.assertEqual(uri, loc.get_uri())
uri = 'swift://account:user:pass@authurl.com/v1/container/12345'
loc.parse_uri(uri)
self.assertEqual("swift", loc.scheme)
self.assertEqual("authurl.com/v1", loc.authurl)
self.assertEqual("container", loc.container)
self.assertEqual("12345", loc.obj)
self.assertEqual("account:user", loc.user)
self.assertEqual("pass", loc.key)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'swif://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 'swift://'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
bad_uri = 'swift://user@example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
def test_s3_store_location(self):
"""
Test the specific StoreLocation for the S3 store
"""
uri = 's3://example.com/images/1'
loc = glance.store.s3.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("s3", loc.scheme)
self.assertEqual("example.com", loc.s3serviceurl)
self.assertEqual("images", loc.bucket)
self.assertEqual("1", loc.key)
self.assertEqual(None, loc.accesskey)
self.assertEqual(uri, loc.get_uri())
uri = 's3+https://accesskey:pass@s3serviceurl.com/images/1'
loc.parse_uri(uri)
self.assertEqual("s3+https", loc.scheme)
self.assertEqual("s3serviceurl.com", loc.s3serviceurl)
self.assertEqual("images", loc.bucket)
self.assertEqual("1", loc.key)
self.assertEqual("accesskey", loc.accesskey)
self.assertEqual("pass", loc.secretkey)
self.assertEqual(uri, loc.get_uri())
uri = 's3+https://accesskey:pass@s3serviceurl.com/v1/bucket/12345'
loc.parse_uri(uri)
self.assertEqual("s3+https", loc.scheme)
self.assertEqual("s3serviceurl.com/v1", loc.s3serviceurl)
self.assertEqual("bucket", loc.bucket)
self.assertEqual("12345", loc.key)
self.assertEqual("accesskey", loc.accesskey)
self.assertEqual("pass", loc.secretkey)
self.assertEqual(uri, loc.get_uri())
uri = 's3://accesskey:pass/withslash@s3serviceurl.com/v1/bucket/12345'
loc.parse_uri(uri)
self.assertEqual("s3", loc.scheme)
self.assertEqual("s3serviceurl.com/v1", loc.s3serviceurl)
self.assertEqual("bucket", loc.bucket)
self.assertEqual("12345", loc.key)
self.assertEqual("accesskey", loc.accesskey)
self.assertEqual("pass/withslash", loc.secretkey)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'swif://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 's3://'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
bad_uri = 's3://accesskey@example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)

1
tests/unit/test_stores.py
View File

@ -19,7 +19,6 @@ from StringIO import StringIO
import stubout import stubout
import unittest import unittest
import urlparse
from glance.store.s3 import S3Backend from glance.store.s3 import S3Backend
from glance.store import Backend, BackendException, get_from_backend from glance.store import Backend, BackendException, get_from_backend

152
tests/unit/test_swift_store.py
View File

@ -29,9 +29,8 @@ import swift.common.client
from glance.common import exception from glance.common import exception
from glance.store import BackendException from glance.store import BackendException
from glance.store.swift import (SwiftBackend, from glance.store.swift import SwiftBackend
format_swift_location, from glance.store.location import get_location_from_uri
parse_swift_tokens)
FIVE_KB = (5 * 1024) FIVE_KB = (5 * 1024)
SWIFT_OPTIONS = {'verbose': True, SWIFT_OPTIONS = {'verbose': True,
@ -146,6 +145,18 @@ def stub_out_swift_common_client(stubs):
'http_connection', fake_http_connection) 'http_connection', fake_http_connection)
def format_swift_location(user, key, authurl, container, obj):
"""
Helper method that returns a Swift store URI given
the component pieces.
"""
scheme = 'swift+https'
if authurl.startswith('http://'):
scheme = 'swift+http'
return "%s://%s:%s@%s/%s/%s" % (scheme, user, key, authurl,
container, obj)
class TestSwiftBackend(unittest.TestCase): class TestSwiftBackend(unittest.TestCase):
def setUp(self): def setUp(self):
@ -157,46 +168,27 @@ class TestSwiftBackend(unittest.TestCase):
"""Clear the test environment""" """Clear the test environment"""
self.stubs.UnsetAll() self.stubs.UnsetAll()
def test_parse_swift_tokens(self):
"""
Test that the parse_swift_tokens function returns
user, key, authurl, container, and objname properly
"""
uri = "swift://user:key@localhost/v1.0/container/objname"
url_pieces = urlparse.urlparse(uri)
user, key, authurl, container, objname =\
parse_swift_tokens(url_pieces)
self.assertEqual("user", user)
self.assertEqual("key", key)
self.assertEqual("https://localhost/v1.0", authurl)
self.assertEqual("container", container)
self.assertEqual("objname", objname)
uri = "swift://user:key@localhost:9090/v1.0/container/objname"
url_pieces = urlparse.urlparse(uri)
user, key, authurl, container, objname =\
parse_swift_tokens(url_pieces)
self.assertEqual("user", user)
self.assertEqual("key", key)
self.assertEqual("https://localhost:9090/v1.0", authurl)
self.assertEqual("container", container)
self.assertEqual("objname", objname)
uri = "swift://account:user:key@localhost:9090/v1.0/container/objname"
url_pieces = urlparse.urlparse(uri)
user, key, authurl, container, objname =\
parse_swift_tokens(url_pieces)
self.assertEqual("account:user", user)
self.assertEqual("key", key)
self.assertEqual("https://localhost:9090/v1.0", authurl)
self.assertEqual("container", container)
self.assertEqual("objname", objname)
def test_get(self): def test_get(self):
"""Test a "normal" retrieval of an image in chunks""" """Test a "normal" retrieval of an image in chunks"""
url_pieces = urlparse.urlparse( loc = get_location_from_uri("swift://user:key@auth_address/glance/2")
"swift://user:key@auth_address/glance/2") image_swift = SwiftBackend.get(loc)
image_swift = SwiftBackend.get(url_pieces)
expected_data = "*" * FIVE_KB
data = ""
for chunk in image_swift:
data += chunk
self.assertEqual(expected_data, data)
def test_get_with_http_auth(self):
"""
Test a retrieval from Swift with an HTTP authurl. This is
specified either via a Location header with swift+http:// or using
http:// in the swift_store_auth_address config value
"""
loc = get_location_from_uri("swift+http://user:key@auth_address/"
"glance/2")
image_swift = SwiftBackend.get(loc)
expected_data = "*" * FIVE_KB expected_data = "*" * FIVE_KB
data = "" data = ""
@ -210,11 +202,10 @@ class TestSwiftBackend(unittest.TestCase):
Test retrieval of an image with wrong expected_size param Test retrieval of an image with wrong expected_size param
raises an exception raises an exception
""" """
url_pieces = urlparse.urlparse( loc = get_location_from_uri("swift://user:key@auth_address/glance/2")
"swift://user:key@auth_address/glance/2")
self.assertRaises(BackendException, self.assertRaises(BackendException,
SwiftBackend.get, SwiftBackend.get,
url_pieces, loc,
{'expected_size': 42}) {'expected_size': 42})
def test_get_non_existing(self): def test_get_non_existing(self):
@ -222,11 +213,10 @@ class TestSwiftBackend(unittest.TestCase):
Test that trying to retrieve a swift that doesn't exist Test that trying to retrieve a swift that doesn't exist
raises an error raises an error
""" """
url_pieces = urlparse.urlparse( loc = get_location_from_uri("swift://user:key@authurl/glance/noexist")
"swift://user:key@auth_address/noexist")
self.assertRaises(exception.NotFound, self.assertRaises(exception.NotFound,
SwiftBackend.get, SwiftBackend.get,
url_pieces) loc)
def test_add(self): def test_add(self):
"""Test that we can add an image via the swift backend""" """Test that we can add an image via the swift backend"""
@ -249,14 +239,62 @@ class TestSwiftBackend(unittest.TestCase):
self.assertEquals(expected_swift_size, size) self.assertEquals(expected_swift_size, size)
self.assertEquals(expected_checksum, checksum) self.assertEquals(expected_checksum, checksum)
url_pieces = urlparse.urlparse(expected_location) loc = get_location_from_uri(expected_location)
new_image_swift = SwiftBackend.get(url_pieces) new_image_swift = SwiftBackend.get(loc)
new_image_contents = new_image_swift.getvalue() new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len new_image_swift_size = new_image_swift.len
self.assertEquals(expected_swift_contents, new_image_contents) self.assertEquals(expected_swift_contents, new_image_contents)
self.assertEquals(expected_swift_size, new_image_swift_size) self.assertEquals(expected_swift_size, new_image_swift_size)
def test_add_auth_url_variations(self):
"""
Test that we can add an image via the swift backend with
a variety of different auth_address values
"""
variations = ['http://localhost:80',
'http://localhost',
'http://localhost/v1',
'http://localhost/v1/',
'https://localhost',
'https://localhost:8080',
'https://localhost/v1',
'https://localhost/v1/',
'localhost',
'localhost:8080/v1']
i = 42
for variation in variations:
expected_image_id = i
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = \
hashlib.md5(expected_swift_contents).hexdigest()
new_options = SWIFT_OPTIONS.copy()
new_options['swift_store_auth_address'] = variation
expected_location = format_swift_location(
new_options['swift_store_user'],
new_options['swift_store_key'],
new_options['swift_store_auth_address'],
new_options['swift_store_container'],
expected_image_id)
image_swift = StringIO.StringIO(expected_swift_contents)
location, size, checksum = SwiftBackend.add(i, image_swift,
new_options)
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_swift = SwiftBackend.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
self.assertEquals(expected_swift_contents, new_image_contents)
self.assertEquals(expected_swift_size, new_image_swift_size)
i = i + 1
def test_add_no_container_no_create(self): def test_add_no_container_no_create(self):
""" """
Tests that adding an image with a non-existing container Tests that adding an image with a non-existing container
@ -306,8 +344,8 @@ class TestSwiftBackend(unittest.TestCase):
self.assertEquals(expected_swift_size, size) self.assertEquals(expected_swift_size, size)
self.assertEquals(expected_checksum, checksum) self.assertEquals(expected_checksum, checksum)
url_pieces = urlparse.urlparse(expected_location) loc = get_location_from_uri(expected_location)
new_image_swift = SwiftBackend.get(url_pieces) new_image_swift = SwiftBackend.get(loc)
new_image_contents = new_image_swift.getvalue() new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len new_image_swift_size = new_image_swift.len
@ -356,22 +394,20 @@ class TestSwiftBackend(unittest.TestCase):
""" """
Test we can delete an existing image in the swift store Test we can delete an existing image in the swift store
""" """
url_pieces = urlparse.urlparse( loc = get_location_from_uri("swift://user:key@authurl/glance/2")
"swift://user:key@auth_address/glance/2")
SwiftBackend.delete(url_pieces) SwiftBackend.delete(loc)
self.assertRaises(exception.NotFound, self.assertRaises(exception.NotFound,
SwiftBackend.get, SwiftBackend.get,
url_pieces) loc)
def test_delete_non_existing(self): def test_delete_non_existing(self):
""" """
Test that trying to delete a swift that doesn't exist Test that trying to delete a swift that doesn't exist
raises an error raises an error
""" """
url_pieces = urlparse.urlparse( loc = get_location_from_uri("swift://user:key@authurl/glance/noexist")
"swift://user:key@auth_address/noexist")
self.assertRaises(exception.NotFound, self.assertRaises(exception.NotFound,
SwiftBackend.delete, SwiftBackend.delete,
url_pieces) loc)