openstack/glance
Code Issues Proposed changes

Merging trunk

Browse Source
This commit is contained in:
Rick Harris 2011-07-25 12:53:01 -05:00
commit 957e4d1831
36 changed files with 2505 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Authors
View File

@ -7,11 +7,13 @@ Donal Lafferty <donal.lafferty@citrix.com>
Eldar Nugaev <enugaev@griddynamics.com>
Ewan Mellor <ewan.mellor@citrix.com>
Isaku Yamahata <yamahata@valinux.co.jp>
Jason Koelker <jason@koelker.net>
Jay Pipes <jaypipes@gmail.com>
Jinwoo 'Joseph' Suh <jsuh@isi.edu>
Josh Kearney <josh@jk0.org>
Justin Shepherd <jshepher@rackspace.com>
Ken Pepple <ken.pepple@gmail.com>
Kevin L. Mitchell <kevin.mitchell@rackspace.com>
Matt Dietz <matt.dietz@rackspace.com>
Monty Taylor <mordred@inaugust.com>
Rick Clark <rick@openstack.org>

7
bin/glance-control
View File

@ -44,15 +44,16 @@ from glance.common import config
ALL_COMMANDS = ['start', 'stop', 'shutdown', 'restart',
'reload', 'force-reload']
ALL_SERVERS = ['glance-api', 'glance-registry']
GRACEFUL_SHUTDOWN_SERVERS = ['glance-api', 'glance-registry']
ALL_SERVERS = ['glance-api', 'glance-registry', 'glance-scrubber']
GRACEFUL_SHUTDOWN_SERVERS = ['glance-api', 'glance-registry',
'glance-scrubber']
MAX_DESCRIPTORS = 32768
MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB
USAGE = """%prog [options] <SERVER> <COMMAND> [CONFPATH]
Where <SERVER> is one of:
all, api, registry
all, api, registry, scrubber
And command is one of:

80
bin/glance-scrubber Executable file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Glance Scrub Service
"""
import optparse
import os
import sys
# If ../glance/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir,
os.pardir))
if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
sys.path.insert(0, possible_topdir)
from glance import version
from glance.common import config
from glance.store import scrubber
def create_options(parser):
"""
Sets up the CLI and config-file options that may be
parsed and program commands.
:param parser: The option parser
"""
config.add_common_options(parser)
config.add_log_options(parser)
parser.add_option("-D", "--daemon", default=False, dest="daemon",
action="store_true",
help="Run as a long-running process. When not "
"specified (the default) run the scrub "
"operation once and then exits. When specified "
"do not exit and run scrub on wakeup_time "
"interval as specified in the config file.")
if __name__ == '__main__':
oparser = optparse.OptionParser(version='%%prog %s'
% version.version_string())
create_options(oparser)
(options, args) = config.parse_options(oparser)
try:
conf, app = config.load_paste_app('glance-scrubber', options, args)
daemon = options.get('daemon') or \
config.get_option(conf, 'daemon', type='bool',
default=False)
if daemon:
wakeup_time = int(conf.get('wakeup_time', 300))
server = scrubber.Daemon(wakeup_time)
server.start(app)
server.wait()
else:
import eventlet
pool = eventlet.greenpool.GreenPool(1000)
scrubber = app.run(pool)
except RuntimeError, e:
sys.exit("ERROR: %s" % e)

10
etc/glance-api.conf
View File

@ -61,8 +61,13 @@ image_cache_datadir = /var/lib/glance/image-cache/
# stalled and eligible for reaping
image_cache_stall_timeout = 86400
# ============ Delayed Delete Options =============================
# Turn on/off delayed delete
delayed_delete = False
[pipeline:glance-api]
pipeline = versionnegotiation apiv1app
pipeline = versionnegotiation context apiv1app
# To enable Image Cache Management API replace pipeline with below:
# pipeline = versionnegotiation imagecache apiv1app
@ -81,3 +86,6 @@ paste.filter_factory = glance.api.middleware.version_negotiation:filter_factory
[filter:imagecache]
paste.filter_factory = glance.api.middleware.image_cache:filter_factory
[filter:context]
paste.filter_factory = glance.common.context:filter_factory

8
etc/glance-registry.conf
View File

@ -29,5 +29,11 @@ sql_connection = sqlite:///glance.sqlite
# before MySQL can drop the connection.
sql_idle_timeout = 3600
[app:glance-registry]
[pipeline:glance-registry]
pipeline = context registryapp
[app:registryapp]
paste.app_factory = glance.registry.server:app_factory
[filter:context]
paste.filter_factory = glance.common.context:filter_factory

36
etc/glance-scrubber.conf Normal file
View File

@ -0,0 +1,36 @@
[DEFAULT]
# Show more verbose log output (sets INFO log level output)
verbose = True
# Show debugging output in logs (sets DEBUG log level output)
debug = False
# Log to this file. Make sure you do not set the same log
# file for both the API and registry servers!
log_file = /var/log/glance/scrubber.log
# Delayed delete time in seconds
scrub_time = 43200
# Should we run our own loop or rely on cron/scheduler to run us
daemon = False
# Loop time between checking the db for new items to schedule for delete
wakeup_time = 300
# SQLAlchemy connection string for the reference implementation
# registry server. Any valid SQLAlchemy connection string is fine.
# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
sql_connection = sqlite:///glance.sqlite
# Period in seconds after which SQLAlchemy should reestablish its connection
# to the database.
#
# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
# notice this, you can lower this value to ensure that SQLAlchemy reconnects
# before MySQL can drop the connection.
sql_idle_timeout = 3600
[app:glance-scrubber]
paste.app_factory = glance.store.scrubber:app_factory

5
glance/api/__init__.py
View File

@ -38,3 +38,8 @@ class BaseController(object):
logger.debug(msg)
raise webob.exc.HTTPNotFound(
msg, request=request, content_type='text/plain')
except exception.NotAuthorized:
msg = "Unauthorized image access"
logger.debug(msg)
raise webob.exc.HTTPForbidden(msg, request=request,
content_type='text/plain')

67
glance/api/v1/images.py
View File

@ -27,14 +27,15 @@ import sys
import webob
from webob.exc import (HTTPNotFound,
HTTPConflict,
HTTPBadRequest)
HTTPBadRequest,
HTTPForbidden)
from glance import api
from glance import image_cache
from glance.common import exception
from glance.common import wsgi
from glance.store import (get_from_backend,
delete_from_backend,
schedule_delete_from_backend,
get_store_from_location,
get_backend_class,
UnsupportedBackend)
@ -45,7 +46,7 @@ from glance import utils
logger = logging.getLogger('glance.api.v1.images')
SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format',
'size_min', 'size_max']
'size_min', 'size_max', 'is_public']
SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir')
@ -97,7 +98,8 @@ class Controller(api.BaseController):
"""
params = self._get_query_params(req)
try:
images = registry.get_images_list(self.options, **params)
images = registry.get_images_list(self.options, req.context,
**params)
except exception.Invalid, e:
raise HTTPBadRequest(explanation=str(e))
@ -127,7 +129,8 @@ class Controller(api.BaseController):
"""
params = self._get_query_params(req)
try:
images = registry.get_images_detail(self.options, **params)
images = registry.get_images_detail(self.options, req.context,
**params)
except exception.Invalid, e:
raise HTTPBadRequest(explanation=str(e))
return dict(images=images)
@ -272,6 +275,7 @@ class Controller(api.BaseController):
try:
image_meta = registry.add_image_metadata(self.options,
req.context,
image_meta)
return image_meta
except exception.Duplicate:
@ -284,6 +288,11 @@ class Controller(api.BaseController):
for line in msg.split('\n'):
logger.error(line)
raise HTTPBadRequest(msg, request=req, content_type="text/plain")
except exception.NotAuthorized:
msg = "Not authorized to reserve image."
logger.error(msg)
raise HTTPForbidden(msg, request=req,
content_type="text/plain")
def _upload(self, req, image_meta):
"""
@ -313,7 +322,7 @@ class Controller(api.BaseController):
image_id = image_meta['id']
logger.debug("Setting image %s to status 'saving'" % image_id)
registry.update_image_metadata(self.options, image_id,
registry.update_image_metadata(self.options, req.context, image_id,
{'status': 'saving'})
try:
logger.debug("Uploading image data for image %(image_id)s "
@ -340,7 +349,8 @@ class Controller(api.BaseController):
logger.debug("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d" % locals())
registry.update_image_metadata(self.options, image_id,
registry.update_image_metadata(self.options, req.context,
image_id,
{'checksum': checksum,
'size': size})
@ -352,6 +362,13 @@ class Controller(api.BaseController):
self._safe_kill(req, image_id)
raise HTTPConflict(msg, request=req)
except exception.NotAuthorized, e:
msg = ("Unauthorized upload attempt: %s") % str(e)
logger.error(msg)
self._safe_kill(req, image_id)
raise HTTPForbidden(msg, request=req,
content_type='text/plain')
except Exception, e:
msg = ("Error uploading image: %s") % str(e)
logger.error(msg)
@ -371,6 +388,7 @@ class Controller(api.BaseController):
image_meta['location'] = location
image_meta['status'] = 'active'
return registry.update_image_metadata(self.options,
req.context,
image_id,
image_meta)
@ -382,6 +400,7 @@ class Controller(api.BaseController):
:param image_id: Opaque image identifier
"""
registry.update_image_metadata(self.options,
req.context,
image_id,
{'status': 'killed'})
@ -450,6 +469,12 @@ class Controller(api.BaseController):
and the request body is not application/octet-stream
image data.
"""
if req.context.read_only:
msg = "Read-only access"
logger.debug(msg)
raise HTTPForbidden(msg, request=req,
content_type="text/plain")
image_meta = self._reserve(req, image_meta)
image_id = image_meta['id']
@ -471,6 +496,12 @@ class Controller(api.BaseController):
:retval Returns the updated image information as a mapping
"""
if req.context.read_only:
msg = "Read-only access"
logger.debug(msg)
raise HTTPForbidden(msg, request=req,
content_type="text/plain")
orig_image_meta = self.get_image_meta_or_404(req, id)
orig_status = orig_image_meta['status']
@ -478,8 +509,9 @@ class Controller(api.BaseController):
raise HTTPConflict("Cannot upload to an unqueued image")
try:
image_meta = registry.update_image_metadata(self.options, id,
image_meta, True)
image_meta = registry.update_image_metadata(self.options,
req.context, id,
image_meta, True)
if image_data is not None:
image_meta = self._upload_and_activate(req, image_meta)
except exception.Invalid, e:
@ -503,6 +535,12 @@ class Controller(api.BaseController):
:raises HttpNotAuthorized if image or any chunk is not
deleteable by the requesting user
"""
if req.context.read_only:
msg = "Read-only access"
logger.debug(msg)
raise HTTPForbidden(msg, request=req,
content_type="text/plain")
image = self.get_image_meta_or_404(req, id)
# The image's location field may be None in the case
@ -510,14 +548,9 @@ class Controller(api.BaseController):
# to delete the image if the backend doesn't yet store it.
# See https://bugs.launchpad.net/glance/+bug/747799
if image['location']:
try:
delete_from_backend(image['location'])
except (UnsupportedBackend, exception.NotFound):
msg = "Failed to delete image from store (%s). " + \
"Continuing with deletion from registry."
logger.error(msg % (image['location'],))
registry.delete_image_metadata(self.options, id)
schedule_delete_from_backend(image['location'], self.options,
req.context, id)
registry.delete_image_metadata(self.options, req.context, id)
def get_store_or_400(self, request, store_name):
"""

6
glance/client.py
View File

@ -35,7 +35,8 @@ class V1Client(base_client.BaseClient):
DEFAULT_PORT = 9292
def __init__(self, host, port=None, use_ssl=False, doc_root="/v1"):
def __init__(self, host, port=None, use_ssl=False, doc_root="/v1",
auth_tok=None):
"""
Creates a new client to a Glance API service.
@ -43,10 +44,11 @@ class V1Client(base_client.BaseClient):
:param port: The port where Glance resides (defaults to 9292)
:param use_ssl: Should we use HTTPS? (defaults to False)
:param doc_root: Prefix for all URLs we request from host
:param auth_tok: The auth token to pass to the server
"""
port = port or self.DEFAULT_PORT
self.doc_root = doc_root
super(Client, self).__init__(host, port, use_ssl)
super(Client, self).__init__(host, port, use_ssl, auth_tok)
def do_request(self, method, action, body=None, headers=None, params=None):
action = "%s/%s" % (self.doc_root, action.lstrip("/"))

6
glance/common/client.py
View File

@ -41,17 +41,19 @@ class BaseClient(object):
CHUNKSIZE = 65536
def __init__(self, host, port, use_ssl):
def __init__(self, host, port, use_ssl, auth_tok):
"""
Creates a new client to some service.
:param host: The host where service resides
:param port: The port where service resides
:param use_ssl: Should we use HTTPS?
:param auth_tok: The auth token to pass to the server
"""
self.host = host
self.port = port
self.use_ssl = use_ssl
self.auth_tok = auth_tok
self.connection = None
def get_connection_type(self):
@ -99,6 +101,8 @@ class BaseClient(object):
try:
connection_type = self.get_connection_type()
headers = headers or {}
if 'x-auth-token' not in headers and self.auth_tok:
headers['x-auth-token'] = self.auth_tok
c = connection_type(self.host, self.port)
# Do a simple request or a chunked request, depending

97
glance/common/context.py Normal file
View File

@ -0,0 +1,97 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from glance.common import utils
from glance.common import wsgi
class RequestContext(object):
"""
Stores information about the security context under which the user
accesses the system, as well as additional request information.
"""
def __init__(self, auth_tok=None, user=None, tenant=None, is_admin=False,
read_only=False, show_deleted=False):
self.auth_tok = auth_tok
self.user = user
self.tenant = tenant
self.is_admin = is_admin
self.read_only = read_only
self.show_deleted = show_deleted
def is_image_visible(self, image):
"""Return True if the image is visible in this context."""
# Is admin == image visible
if self.is_admin:
return True
# No owner == image visible
if image.owner is None:
return True
# Image is_public == image visible
if image.is_public:
return True
# Private image
return self.owner is not None and self.owner == image.owner
@property
def owner(self):
"""Return the owner to correlate with an image."""
return self.tenant
class ContextMiddleware(wsgi.Middleware):
def __init__(self, app, options):
self.options = options
super(ContextMiddleware, self).__init__(app)
def make_context(self, *args, **kwargs):
"""
Create a context with the given arguments.
"""
# Determine the context class to use
ctxcls = RequestContext
if 'context_class' in self.options:
ctxcls = utils.import_class(self.options['context_class'])
return ctxcls(*args, **kwargs)
def process_request(self, req):
"""
Extract any authentication information in the request and
construct an appropriate context from it.
"""
# Use the default empty context, with admin turned on for
# backwards compatibility
req.context = self.make_context(is_admin=True)
def filter_factory(global_conf, **local_conf):
"""
Factory method for paste.deploy
"""
conf = global_conf.copy()
conf.update(local_conf)
def filter(app):
return ContextMiddleware(app, conf)
return filter

18
glance/common/exception.py
View File

@ -54,6 +54,24 @@ class NotFound(Error):
pass
class UnknownScheme(Error):
msg = "Unknown scheme '%s' found in URI"
def __init__(self, scheme):
msg = self.__class__.msg % scheme
super(UnknownScheme, self).__init__(msg)
class BadStoreUri(Error):
msg = "The Store URI %s was malformed. Reason: %s"
def __init__(self, uri, reason):
msg = self.__class__.msg % (uri, reason)
super(BadStoreUri, self).__init__(msg)
class Duplicate(Error):
pass

29
glance/registry/__init__.py
View File

@ -26,33 +26,33 @@ from glance.registry import client
logger = logging.getLogger('glance.registry')
def get_registry_client(options):
def get_registry_client(options, cxt):
host = options['registry_host']
port = int(options['registry_port'])
return client.RegistryClient(host, port)
return client.RegistryClient(host, port, auth_tok=cxt.auth_tok)
def get_images_list(options, **kwargs):
c = get_registry_client(options)
def get_images_list(options, context, **kwargs):
c = get_registry_client(options, context)
return c.get_images(**kwargs)
def get_images_detail(options, **kwargs):
c = get_registry_client(options)
def get_images_detail(options, context, **kwargs):
c = get_registry_client(options, context)
return c.get_images_detailed(**kwargs)
def get_image_metadata(options, image_id):
c = get_registry_client(options)
def get_image_metadata(options, context, image_id):
c = get_registry_client(options, context)
return c.get_image(image_id)
def add_image_metadata(options, image_meta):
def add_image_metadata(options, context, image_meta):
if options['debug']:
logger.debug("Adding image metadata...")
_debug_print_metadata(image_meta)
c = get_registry_client(options)
c = get_registry_client(options, context)
new_image_meta = c.add_image(image_meta)
if options['debug']:
@ -63,12 +63,13 @@ def add_image_metadata(options, image_meta):
return new_image_meta
def update_image_metadata(options, image_id, image_meta, purge_props=False):
def update_image_metadata(options, context, image_id, image_meta,
purge_props=False):
if options['debug']:
logger.debug("Updating image metadata for image %s...", image_id)
_debug_print_metadata(image_meta)
c = get_registry_client(options)
c = get_registry_client(options, context)
new_image_meta = c.update_image(image_id, image_meta, purge_props)
if options['debug']:
@ -79,9 +80,9 @@ def update_image_metadata(options, image_id, image_meta, purge_props=False):
return new_image_meta
def delete_image_metadata(options, image_id):
def delete_image_metadata(options, context, image_id):
logger.debug("Deleting image metadata for image %s...", image_id)
c = get_registry_client(options)
c = get_registry_client(options, context)
return c.delete_image(image_id)

5
glance/registry/client.py
View File

@ -33,16 +33,17 @@ class RegistryClient(BaseClient):
DEFAULT_PORT = 9191
def __init__(self, host, port=None, use_ssl=False):
def __init__(self, host, port=None, use_ssl=False, auth_tok=None):
"""
Creates a new client to a Glance Registry service.
:param host: The host where Glance resides
:param port: The port where Glance resides (defaults to 9191)
:param use_ssl: Should we use HTTPS? (defaults to False)
:param auth_tok: The auth token to pass to the server
"""
port = port or self.DEFAULT_PORT
super(RegistryClient, self).__init__(host, port, use_ssl)
super(RegistryClient, self).__init__(host, port, use_ssl, auth_tok)
def get_images(self, **kwargs):
"""

63
glance/registry/db/api.py
View File

@ -24,6 +24,7 @@ Defines interface for DB access
import logging
from sqlalchemy import asc, create_engine, desc
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import exc
from sqlalchemy.orm import joinedload
@ -45,12 +46,14 @@ BASE_MODEL_ATTRS = set(['id', 'created_at', 'updated_at', 'deleted_at',
IMAGE_ATTRS = BASE_MODEL_ATTRS | set(['name', 'status', 'size',
'disk_format', 'container_format',
'is_public', 'location', 'checksum'])
'is_public', 'location', 'checksum',
'owner'])
CONTAINER_FORMATS = ['ami', 'ari', 'aki', 'bare', 'ovf']
DISK_FORMATS = ['ami', 'ari', 'aki', 'vhd', 'vmdk', 'raw', 'qcow2', 'vdi',
'iso']
STATUSES = ['active', 'saving', 'queued', 'killed']
STATUSES = ['active', 'saving', 'queued', 'killed', 'pending_delete',
'deleted']
def configure_db(options):
@ -119,7 +122,7 @@ def image_get(context, image_id, session=None):
"""Get an image or raise if it does not exist."""
session = session or get_session()
try:
return session.query(models.Image).\
image = session.query(models.Image).\
options(joinedload(models.Image.properties)).\
filter_by(deleted=_deleted(context)).\
filter_by(id=image_id).\
@ -127,11 +130,40 @@ def image_get(context, image_id, session=None):
except exc.NoResultFound:
raise exception.NotFound("No image found with ID %s" % image_id)
# Make sure they can look at it
if not context.is_image_visible(image):
raise exception.NotAuthorized("Image not visible to you")
def image_get_all_public(context, filters=None, marker=None, limit=None,
sort_key='created_at', sort_dir='desc'):
return image
def image_get_all_pending_delete(context, delete_time=None, limit=None):
"""Get all images that are pending deletion
:param limit: maximum number of images to return
"""
Get all public images that match zero or more filters.
session = get_session()
query = session.query(models.Image).\
options(joinedload(models.Image.properties)).\
filter_by(deleted=True).\
filter(models.Image.status == 'pending_delete')
if delete_time:
query = query.filter(models.Image.deleted_at <= delete_time)
query = query.order_by(desc(models.Image.deleted_at)).\
order_by(desc(models.Image.id))
if limit:
query = query.limit(limit)
return query.all()
def image_get_all(context, filters=None, marker=None, limit=None,
sort_key='created_at', sort_dir='desc'):
"""
Get all images that match zero or more filters.
:param filters: dict of filter keys and values. If a 'properties'
key is present, it is treated as a dict of key/value
@ -147,7 +179,6 @@ def image_get_all_public(context, filters=None, marker=None, limit=None,
query = session.query(models.Image).\
options(joinedload(models.Image.properties)).\
filter_by(deleted=_deleted(context)).\
filter_by(is_public=True).\
filter(models.Image.status != 'killed')
sort_dir_func = {
@ -168,11 +199,19 @@ def image_get_all_public(context, filters=None, marker=None, limit=None,
query = query.filter(models.Image.size <= filters['size_max'])
del filters['size_max']
if 'is_public' in filters and filters['is_public'] is not None:
the_filter = models.Image.is_public == filters['is_public']
if filters['is_public'] and context.owner is not None:
the_filter = or_(the_filter, models.Image.owner == context.owner)
query = query.filter(the_filter)
del filters['is_public']
for (k, v) in filters.pop('properties', {}).items():
query = query.filter(models.Image.properties.any(name=k, value=v))
for (k, v) in filters.items():
query = query.filter(getattr(models.Image, k) == v)
if v is not None:
query = query.filter(getattr(models.Image, k) == v)
if marker != None:
# images returned should be created before the image defined by marker
@ -273,7 +312,11 @@ def _image_update(context, values, image_id, purge_props=False):
# idiotic.
validate_image(image_ref.to_dict())
image_ref.save(session=session)
try:
image_ref.save(session=session)
except IntegrityError, e:
raise exception.Duplicate("Image ID %s already exists!"
% values['id'])
_set_properties_for_image(context, image_ref, properties, purge_props,
session)
@ -350,6 +393,8 @@ def _deleted(context):
Calculates whether to include deleted objects based on context.
Currently just looks for a flag called deleted in the context dict.
"""
if hasattr(context, 'show_deleted'):
return context.show_deleted
if not hasattr(context, 'get'):
return False
return context.get('deleted', False)

82
glance/registry/db/migrate_repo/versions/007_add_owner.py Normal file
View File

@ -0,0 +1,82 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.changeset import *
from sqlalchemy import *
from sqlalchemy.sql import and_, not_
from glance.registry.db.migrate_repo.schema import (
Boolean, DateTime, BigInteger, Integer, String,
Text, from_migration_import)
def get_images_table(meta):
"""
Returns the Table object for the images table that corresponds to
the images table definition of this version.
"""
images = Table('images', meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', String(255)),
Column('disk_format', String(20)),
Column('container_format', String(20)),
Column('size', BigInteger()),
Column('status', String(30), nullable=False),
Column('is_public', Boolean(), nullable=False, default=False,
index=True),
Column('location', Text()),
Column('created_at', DateTime(), nullable=False),
Column('updated_at', DateTime()),
Column('deleted_at', DateTime()),
Column('deleted', Boolean(), nullable=False, default=False,
index=True),
Column('checksum', String(32)),
Column('owner', String(255)),
mysql_engine='InnoDB',
useexisting=True)
return images
def get_image_properties_table(meta):
"""
No changes to the image properties table from 006...
"""
(get_image_properties_table,) = from_migration_import(
'006_key_to_name', ['get_image_properties_table'])
image_properties = get_image_properties_table(meta)
return image_properties
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
images = get_images_table(meta)
owner = Column('owner', String(255))
owner.create(images)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
images = get_images_table(meta)
images.columns['owner'].drop()

1
glance/registry/db/models.py
View File

@ -105,6 +105,7 @@ class Image(BASE, ModelBase):
is_public = Column(Boolean, nullable=False, default=False)
location = Column(Text)
checksum = Column(String(32))
owner = Column(String(255))
class ImageProperty(BASE, ModelBase):

98
glance/registry/server.py
View File

@ -56,6 +56,16 @@ class Controller(object):
self.options = options
db_api.configure_db(options)
def _get_images(self, context, **params):
"""
Get images, wrapping in exception if necessary.
"""
try:
return db_api.image_get_all(context, **params)
except exception.NotFound, e:
msg = "Invalid marker. Image could not be found."
raise exc.HTTPBadRequest(explanation=msg)
def index(self, req):
"""
Return a basic filtered list of public, non-deleted images
@ -77,11 +87,7 @@ class Controller(object):
}
"""
params = self._get_query_params(req)
try:
images = db_api.image_get_all_public(None, **params)
except exception.NotFound, e:
msg = "Invalid marker. Image could not be found."
raise exc.HTTPBadRequest(explanation=msg)
images = self._get_images(req.context, **params)
results = []
for image in images:
@ -104,12 +110,8 @@ class Controller(object):
all image model fields.
"""
params = self._get_query_params(req)
try:
images = db_api.image_get_all_public(None, **params)
except exception.NotFound, e:
msg = "Invalid marker. Image could not be found."
raise exc.HTTPBadRequest(explanation=msg)
images = self._get_images(req.context, **params)
image_dicts = [make_image_dict(i) for i in images]
return dict(images=image_dicts)
@ -144,6 +146,11 @@ class Controller(object):
filters = {}
properties = {}
if req.context.is_admin:
# Only admin gets to look for non-public images
filters['is_public'] = self._get_is_public(req)
else:
filters['is_public'] = True
for param in req.str_params:
if param in SUPPORTED_FILTERS:
filters[param] = req.str_params.get(param)
@ -199,12 +206,36 @@ class Controller(object):
raise exc.HTTPBadRequest(explanation=msg)
return sort_dir
def _get_is_public(self, req):
"""Parse is_public into something usable."""
is_public = req.str_params.get('is_public', None)
if is_public is None:
# NOTE(vish): This preserves the default value of showing only
# public images.
return True
is_public = is_public.lower()
if is_public == 'none':
return None
elif is_public == 'true' or is_public == '1':
return True
elif is_public == 'false' or is_public == '0':
return False
else:
raise exc.HTTPBadRequest("is_public must be None, True, or False")
def show(self, req, id):
"""Return data about the given image id."""
try:
image = db_api.image_get(None, id)
image = db_api.image_get(req.context, id)
except exception.NotFound:
raise exc.HTTPNotFound()
except exception.NotAuthorized:
# If it's private and doesn't belong to them, don't let on
# that it exists
logger.info("Access by %s to image %s denied" %
(req.context.user, id))
raise exc.HTTPNotFound()
return dict(image=make_image_dict(image))
@ -217,11 +248,19 @@ class Controller(object):
:retval Returns 200 if delete was successful, a fault if not.
"""
context = None
if req.context.read_only:
raise exc.HTTPForbidden()
try:
db_api.image_destroy(context, id)
db_api.image_destroy(req.context, id)
except exception.NotFound:
return exc.HTTPNotFound()
except exception.NotAuthorized:
# If it's private and doesn't belong to them, don't let on
# that it exists
logger.info("Access by %s to image %s denied" %
(req.context.user, id))
raise exc.HTTPNotFound()
def create(self, req, body):
"""
@ -234,14 +273,20 @@ class Controller(object):
which will include the newly-created image's internal id
in the 'id' field
"""
if req.context.read_only:
raise exc.HTTPForbidden()
image_data = body['image']
# Ensure the image has a status set
image_data.setdefault('status', 'active')
context = None
# Set up the image owner
if not req.context.is_admin or 'owner' not in image_data:
image_data['owner'] = req.context.owner
try:
image_data = db_api.image_create(context, image_data)
image_data = db_api.image_create(req.context, image_data)
return dict(image=make_image_dict(image_data))
except exception.Duplicate:
msg = ("Image with identifier %s already exists!" % id)
@ -262,18 +307,25 @@ class Controller(object):
:retval Returns the updated image information as a mapping,
"""
if req.context.read_only:
raise exc.HTTPForbidden()
image_data = body['image']
# Prohibit modification of 'owner'
if not req.context.is_admin and 'owner' in image_data:
del image_data['owner']
purge_props = req.headers.get("X-Glance-Registry-Purge-Props", "false")
context = None
try:
logger.debug("Updating image %(id)s with metadata: %(image_data)r"
% locals())
if purge_props == "true":
updated_image = db_api.image_update(context, id, image_data,
True)
updated_image = db_api.image_update(req.context, id,
image_data, True)
else:
updated_image = db_api.image_update(context, id, image_data)
updated_image = db_api.image_update(req.context, id,
image_data)
return dict(image=make_image_dict(updated_image))
except exception.Invalid, e:
msg = ("Failed to update image metadata. "
@ -284,6 +336,14 @@ class Controller(object):
raise exc.HTTPNotFound(body='Image not found',
request=req,
content_type='text/plain')
except exception.NotAuthorized:
# If it's private and doesn't belong to them, don't let on
# that it exists
logger.info("Access by %s to image %s denied" %
(req.context.user, id))
raise exc.HTTPNotFound(body='Image not found',
request=req,
content_type='text/plain')
def create_resource(options):

75
glance/store/__init__.py
View File

@ -15,11 +15,17 @@
# License for the specific language governing permissions and limitations
# under the License.
import logging
import optparse
import os
import urlparse
from glance.common import exception
from glance import registry
from glance.common import config, exception
from glance.store import location
logger = logging.getLogger('glance.store')
# TODO(sirp): should this be moved out to common/utils.py ?
@ -74,69 +80,48 @@ def get_backend_class(backend):
def get_from_backend(uri, **kwargs):
"""Yields chunks of data from backend specified by uri"""
parsed_uri = urlparse.urlparse(uri)
scheme = parsed_uri.scheme
loc = location.get_location_from_uri(uri)
backend_class = get_backend_class(loc.store_name)
backend_class = get_backend_class(scheme)
return backend_class.get(parsed_uri, **kwargs)
return backend_class.get(loc, **kwargs)
def delete_from_backend(uri, **kwargs):
"""Removes chunks of data from backend specified by uri"""
parsed_uri = urlparse.urlparse(uri)
scheme = parsed_uri.scheme
backend_class = get_backend_class(scheme)
loc = location.get_location_from_uri(uri)
backend_class = get_backend_class(loc.store_name)
if hasattr(backend_class, 'delete'):
return backend_class.delete(parsed_uri, **kwargs)
return backend_class.delete(loc, **kwargs)
def get_store_from_location(location):
def get_store_from_location(uri):
"""
Given a location (assumed to be a URL), attempt to determine
the store from the location. We use here a simple guess that
the scheme of the parsed URL is the store...
:param location: Location to check for the store
:param uri: Location to check for the store
"""
loc_pieces = urlparse.urlparse(location)
return loc_pieces.scheme
loc = location.get_location_from_uri(uri)
return loc.store_name
def parse_uri_tokens(parsed_uri, example_url):
def schedule_delete_from_backend(uri, options, context, id, **kwargs):
"""
Given a URI and an example_url, attempt to parse the uri to assemble an
authurl. This method returns the user, key, authurl, referenced container,
and the object we're looking for in that container.
Parsing the uri is three phases:
1) urlparse to split the tokens
2) use RE to split on @ and /
3) reassemble authurl
Given a uri and a time, schedule the deletion of an image.
"""
path = parsed_uri.path.lstrip('//')
netloc = parsed_uri.netloc
try:
use_delay = config.get_option(options, 'delayed_delete', type='bool',
default=False)
if not use_delay:
registry.update_image_metadata(options, context, id,
{'status': 'deleted'})
try:
creds, netloc = netloc.split('@')
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
creds, path = path.split('@')
user, key = creds.split(':')
path_parts = path.split('/')
obj = path_parts.pop()
container = path_parts.pop()
except (ValueError, IndexError):
raise BackendException(
"Expected four values to unpack in: %s:%s. "
"Should have received something like: %s."
% (parsed_uri.scheme, parsed_uri.path, example_url))
return delete_from_backend(uri, **kwargs)
except (UnsupportedBackend, exception.NotFound):
msg = "Failed to delete image from store (%s). "
logger.error(msg % uri)
authurl = "https://%s" % '/'.join(path_parts)
return user, key, authurl, container, obj
registry.update_image_metadata(options, context, id,
{'status': 'pending_delete'})

56
glance/store/filesystem.py
View File

@ -26,9 +26,39 @@ import urlparse
from glance.common import exception
import glance.store
import glance.store.location
logger = logging.getLogger('glance.store.filesystem')
glance.store.location.add_scheme_map({'file': 'filesystem'})
class StoreLocation(glance.store.location.StoreLocation):
"""Class describing a Filesystem URI"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 'file')
self.path = self.specs.get('path')
def get_uri(self):
return "file://%s" % self.path
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme == 'file'
self.scheme = pieces.scheme
path = (pieces.netloc + pieces.path).strip()
if path == '':
reason = "No path specified"
raise exception.BadStoreUri(uri, reason)
self.path = path
class ChunkedFile(object):
@ -64,13 +94,19 @@ class ChunkedFile(object):
class FilesystemBackend(glance.store.Backend):
@classmethod
def get(cls, parsed_uri, expected_size=None, options=None):
def get(cls, location, expected_size=None, options=None):
"""
Filesystem-based backend
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator to use in
reading the image file.
file:///path/to/file.tar.gz.0
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if file does not exist
"""
filepath = parsed_uri.path
loc = location.store_location
filepath = loc.path
if not os.path.exists(filepath):
raise exception.NotFound("Image file %s not found" % filepath)
else:
@ -79,17 +115,19 @@ class FilesystemBackend(glance.store.Backend):
return ChunkedFile(filepath)
@classmethod
def delete(cls, parsed_uri):
def delete(cls, location):
"""
Removes a file from the filesystem backend.
Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:param parsed_uri: Parsed pieces of URI in form of::
file:///path/to/filename.ext
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if file does not exist
:raises NotAuthorized if cannot delete because of permissions
"""
fn = parsed_uri.path
loc = location.store_location
fn = loc.path
if os.path.exists(fn):
try:
logger.debug("Deleting image at %s", fn)

87
glance/store/http.py
View File

@ -16,31 +16,104 @@
# under the License.
import httplib
import urlparse
from glance.common import exception
import glance.store
import glance.store.location
glance.store.location.add_scheme_map({'http': 'http',
'https': 'http'})
class StoreLocation(glance.store.location.StoreLocation):
"""Class describing an HTTP(S) URI"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 'http')
self.netloc = self.specs['netloc']
self.user = self.specs.get('user')
self.password = self.specs.get('password')
self.path = self.specs.get('path')
def _get_credstring(self):
if self.user:
return '%s:%s@' % (self.user, self.password)
return ''
def get_uri(self):
return "%s://%s%s%s" % (
self.scheme,
self._get_credstring(),
self.netloc,
self.path)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('https', 'http')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path
try:
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
if creds:
try:
self.user, self.password = creds.split(':')
except ValueError:
reason = ("Credentials '%s' not well-formatted."
% "".join(creds))
raise exception.BadStoreUri(uri, reason)
else:
self.user = None
if netloc == '':
reason = "No address specified in HTTP URL"
raise exception.BadStoreUri(uri, reason)
self.netloc = netloc
self.path = path
class HTTPBackend(glance.store.Backend):
""" An implementation of the HTTP Backend Adapter """
@classmethod
def get(cls, parsed_uri, expected_size, options=None, conn_class=None):
def get(cls, location, expected_size, options=None, conn_class=None):
"""
Takes a parsed uri for an HTTP resource, fetches it, and
yields the data.
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator from Swift
provided by Swift client's get_object() method.
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
"""
loc = location.store_location
if conn_class:
pass # use the conn_class passed in
elif parsed_uri.scheme == "http":
elif loc.scheme == "http":
conn_class = httplib.HTTPConnection
elif parsed_uri.scheme == "https":
elif loc.scheme == "https":
conn_class = httplib.HTTPSConnection
else:
raise glance.store.BackendException(
"scheme '%s' not supported for HTTPBackend")
conn = conn_class(parsed_uri.netloc)
conn.request("GET", parsed_uri.path, "", {})
conn = conn_class(loc.netloc)
conn.request("GET", loc.path, "", {})
try:
return glance.store._file_iter(conn.getresponse(), cls.CHUNKSIZE)

182
glance/store/location.py Normal file
View File

@ -0,0 +1,182 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
A class that describes the location of an image in Glance.
In Glance, an image can either be **stored** in Glance, or it can be
**registered** in Glance but actually be stored somewhere else.
We needed a class that could support the various ways that Glance
describes where exactly an image is stored.
An image in Glance has two location properties: the image URI
and the image storage URI.
The image URI is essentially the permalink identifier for the image.
It is displayed in the output of various Glance API calls and,
while read-only, is entirely user-facing. It shall **not** contain any
security credential information at all. The Glance image URI shall
be the host:port of that Glance API server along with /images/<IMAGE_ID>.
The Glance storage URI is an internal URI structure that Glance
uses to maintain critical information about how to access the images
that it stores in its storage backends. It **does contain** security
credentials and is **not** user-facing.
"""
import logging
import urlparse
from glance.common import exception
from glance.common import utils
logger = logging.getLogger('glance.store.location')
SCHEME_TO_STORE_MAP = {}
def get_location_from_uri(uri):
"""
Given a URI, return a Location object that has had an appropriate
store parse the URI.
:param uri: A URI that could come from the end-user in the Location
attribute/header
Example URIs:
https://user:pass@example.com:80/images/some-id
http://images.oracle.com/123456
swift://user:account:pass@authurl.com/container/obj-id
swift+http://user:account:pass@authurl.com/container/obj-id
s3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id
s3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id
file:///var/lib/glance/images/1
"""
# Add known stores to mapping... this gets past circular import
# issues. There's a better way to do this, but that's for another
# patch...
# TODO(jaypipes) Clear up these imports in refactor-stores blueprint
import glance.store.filesystem
import glance.store.http
import glance.store.s3
import glance.store.swift
pieces = urlparse.urlparse(uri)
if pieces.scheme not in SCHEME_TO_STORE_MAP.keys():
raise exception.UnknownScheme(pieces.scheme)
loc = Location(pieces.scheme, uri=uri)
return loc
def add_scheme_map(scheme_map):
"""
Given a mapping of 'scheme' to store_name, adds the mapping to the
known list of schemes.
Each store should call this method and let Glance know about which
schemes to map to a store name.
"""
SCHEME_TO_STORE_MAP.update(scheme_map)
class Location(object):
"""
Class describing the location of an image that Glance knows about
"""
def __init__(self, store_name, uri=None, image_id=None, store_specs=None):
"""
Create a new Location object.
:param store_name: The string identifier of the storage backend
:param image_id: The identifier of the image in whatever storage
backend is used.
:param uri: Optional URI to construct location from
:param store_specs: Dictionary of information about the location
of the image that is dependent on the backend
store
"""
self.store_name = store_name
self.image_id = image_id
self.store_specs = store_specs or {}
self.store_location = self._get_store_location()
if uri:
self.store_location.parse_uri(uri)
def _get_store_location(self):
"""
We find the store module and then grab an instance of the store's
StoreLocation class which handles store-specific location information
"""
try:
cls = utils.import_class('glance.store.%s.StoreLocation'
% SCHEME_TO_STORE_MAP[self.store_name])
return cls(self.store_specs)
except exception.NotFound:
logger.error("Unable to find StoreLocation class in store %s",
self.store_name)
return None
def get_store_uri(self):
"""
Returns the Glance image URI, which is the host:port of the API server
along with /images/<IMAGE_ID>
"""
return self.store_location.get_uri()
def get_uri(self):
return None
class StoreLocation(object):
"""
Base class that must be implemented by each store
"""
def __init__(self, store_specs):
self.specs = store_specs
if self.specs:
self.process_specs()
def process_specs(self):
"""
Subclasses should implement any processing of the self.specs collection
such as storing credentials and possibly establishing connections.
"""
pass
def get_uri(self):
"""
Subclasses should implement a method that returns an internal URI that,
when supplied to the StoreLocation instance, can be interpreted by the
StoreLocation's parse_uri() method. The URI returned from this method
shall never be public and only used internally within Glance, so it is
fine to encode credentials in this URI.
"""
raise NotImplementedError("StoreLocation subclass must implement "
"get_uri()")
def parse_uri(self, uri):
"""
Subclasses should implement a method that accepts a string URI and
sets appropriate internal fields such that a call to get_uri() will
return a proper internal URI
"""
raise NotImplementedError("StoreLocation subclass must implement "
"parse_uri()")

144
glance/store/s3.py
View File

@ -17,7 +17,101 @@
"""The s3 backend adapter"""
import urlparse
from glance.common import exception
import glance.store
import glance.store.location
glance.store.location.add_scheme_map({'s3': 's3',
's3+http': 's3',
's3+https': 's3'})
class StoreLocation(glance.store.location.StoreLocation):
"""
Class describing an S3 URI. An S3 URI can look like any of
the following:
s3://accesskey:secretkey@s3service.com/bucket/key-id
s3+http://accesskey:secretkey@s3service.com/bucket/key-id
s3+https://accesskey:secretkey@s3service.com/bucket/key-id
The s3+https:// URIs indicate there is an HTTPS s3service URL
"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 's3')
self.accesskey = self.specs.get('accesskey')
self.secretkey = self.specs.get('secretkey')
self.s3serviceurl = self.specs.get('s3serviceurl')
self.bucket = self.specs.get('bucket')
self.key = self.specs.get('key')
def _get_credstring(self):
if self.accesskey:
return '%s:%s@' % (self.accesskey, self.secretkey)
return ''
def get_uri(self):
return "%s://%s%s/%s/%s" % (
self.scheme,
self._get_credstring(),
self.s3serviceurl,
self.bucket,
self.key)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
Note that an Amazon AWS secret key can contain the forward slash,
which is entirely retarded, and breaks urlparse miserably.
This function works around that issue.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('s3', 's3+http', 's3+https')
self.scheme = pieces.scheme
path = pieces.path.strip('/')
netloc = pieces.netloc.strip('/')
entire_path = (netloc + '/' + path).strip('/')
if '@' in uri:
creds, path = entire_path.split('@')
cred_parts = creds.split(':')
try:
access_key = cred_parts[0]
secret_key = cred_parts[1]
# NOTE(jaypipes): Need to encode to UTF-8 here because of a
# bug in the HMAC library that boto uses.
# See: http://bugs.python.org/issue5285
# See: http://trac.edgewall.org/ticket/8083
access_key = access_key.encode('utf-8')
secret_key = secret_key.encode('utf-8')
self.accesskey = access_key
self.secretkey = secret_key
except IndexError:
reason = "Badly formed S3 credentials %s" % creds
raise exception.BadStoreUri(uri, reason)
else:
self.accesskey = None
path = entire_path
try:
path_parts = path.split('/')
self.key = path_parts.pop()
self.bucket = path_parts.pop()
if len(path_parts) > 0:
self.s3serviceurl = '/'.join(path_parts)
else:
reason = "Badly formed S3 URI. Missing s3 service URL."
raise exception.BadStoreUri(uri, reason)
except IndexError:
reason = "Badly formed S3 URI"
raise exception.BadStoreUri(uri, reason)
class S3Backend(glance.store.Backend):
@ -26,29 +120,30 @@ class S3Backend(glance.store.Backend):
EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0"
@classmethod
def get(cls, parsed_uri, expected_size, conn_class=None):
"""
Takes a parsed_uri in the format of:
s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects
to s3 and downloads the file. Returns the generator resp_body provided
by get_object.
def get(cls, location, expected_size, conn_class=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator from S3
provided by S3's key object
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
"""
if conn_class:
pass
else:
import boto.s3.connection
conn_class = boto.s3.connection.S3Connection
(access_key, secret_key, host, bucket, obj) = \
cls._parse_s3_tokens(parsed_uri)
loc = location.store_location
# Close the connection when we're through.
with conn_class(access_key, secret_key, host=host) as s3_conn:
bucket = cls._get_bucket(s3_conn, bucket)
with conn_class(loc.accesskey, loc.secretkey,
host=loc.s3serviceurl) as s3_conn:
bucket = cls._get_bucket(s3_conn, loc.bucket)
# Close the key when we're through.
with cls._get_key(bucket, obj) as key:
with cls._get_key(bucket, loc.obj) as key:
if not key.size == expected_size:
raise glance.store.BackendException(
"Expected %s bytes, got %s" %
@ -59,28 +154,28 @@ class S3Backend(glance.store.Backend):
yield chunk
@classmethod
def delete(cls, parsed_uri, conn_class=None):
"""
Takes a parsed_uri in the format of:
s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects
to s3 and deletes the file. Returns whatever boto.s3.key.Key.delete()
returns.
def delete(cls, location, conn_class=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
"""
if conn_class:
pass
else:
conn_class = boto.s3.connection.S3Connection
(access_key, secret_key, host, bucket, obj) = \
cls._parse_s3_tokens(parsed_uri)
loc = location.store_location
# Close the connection when we're through.
with conn_class(access_key, secret_key, host=host) as s3_conn:
bucket = cls._get_bucket(s3_conn, bucket)
with conn_class(loc.accesskey, loc.secretkey,
host=loc.s3serviceurl) as s3_conn:
bucket = cls._get_bucket(s3_conn, loc.bucket)
# Close the key when we're through.
with cls._get_key(bucket, obj) as key:
with cls._get_key(bucket, loc.obj) as key:
return key.delete()
@classmethod
@ -102,8 +197,3 @@ class S3Backend(glance.store.Backend):
if not key:
raise glance.store.BackendException("Could not get key: %s" % key)
return key
@classmethod
def _parse_s3_tokens(cls, parsed_uri):
"""Parse tokens from the parsed_uri"""
return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL)

90
glance/store/scrubber.py Normal file
View File

@ -0,0 +1,90 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import eventlet
import logging
from glance import registry
from glance import store
from glance.common import config
from glance.common import context
from glance.common import exception
from glance.registry.db import api as db_api
logger = logging.getLogger('glance.store.scrubber')
class Daemon(object):
def __init__(self, wakeup_time=300, threads=1000):
logger.info("Starting Daemon: " +
"wakeup_time=%s threads=%s" % (wakeup_time, threads))
self.wakeup_time = wakeup_time
self.event = eventlet.event.Event()
self.pool = eventlet.greenpool.GreenPool(threads)
def start(self, application):
self._run(application)
def wait(self):
try:
self.event.wait()
except KeyboardInterrupt:
logger.info("Daemon Shutdown on KeyboardInterrupt")
def _run(self, application):
logger.debug("Runing application")
self.pool.spawn_n(application.run, self.pool, self.event)
eventlet.spawn_after(self.wakeup_time, self._run, application)
logger.debug("Next run scheduled in %s seconds" % self.wakeup_time)
class Scrubber(object):
def __init__(self, options):
logger.info("Initializing scrubber with options: %s" % options)
self.options = options
scrub_time = config.get_option(options, 'scrub_time', type='int',
default=0)
logger.info("Scrub interval set to %s seconds" % scrub_time)
self.scrub_time = datetime.timedelta(seconds=scrub_time)
db_api.configure_db(options)
def run(self, pool, event=None):
delete_time = datetime.datetime.utcnow() - self.scrub_time
logger.info("Getting images deleted before %s" % delete_time)
pending = db_api.image_get_all_pending_delete(None, delete_time)
logger.info("Deleting %s images" % len(pending))
delete_work = [(p['id'], p['location']) for p in pending]
pool.starmap(self._delete, delete_work)
def _delete(self, id, location):
try:
logger.debug("Deleting %s" % location)
store.delete_from_backend(location)
except (store.UnsupportedBackend, exception.NotFound):
msg = "Failed to delete image from store (%s). "
logger.error(msg % uri)
ctx = context.RequestContext(is_admin=True, show_deleted=True)
db_api.image_update(ctx, id, {'status': 'deleted'})
def app_factory(global_config, **local_conf):
conf = global_config.copy()
conf.update(local_conf)
return Scrubber(conf)

226
glance/store/swift.py
View File

@ -21,15 +21,114 @@ from __future__ import absolute_import
import httplib
import logging
import urlparse
from glance.common import config
from glance.common import exception
import glance.store
import glance.store.location
DEFAULT_SWIFT_CONTAINER = 'glance'
logger = logging.getLogger('glance.store.swift')
glance.store.location.add_scheme_map({'swift': 'swift',
'swift+http': 'swift',
'swift+https': 'swift'})
class StoreLocation(glance.store.location.StoreLocation):
"""
Class describing a Swift URI. A Swift URI can look like any of
the following:
swift://user:pass@authurl.com/container/obj-id
swift+http://user:pass@authurl.com/container/obj-id
swift+https://user:pass@authurl.com/container/obj-id
The swift+https:// URIs indicate there is an HTTPS authentication URL
"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 'swift+https')
self.user = self.specs.get('user')
self.key = self.specs.get('key')
self.authurl = self.specs.get('authurl')
self.container = self.specs.get('container')
self.obj = self.specs.get('obj')
def _get_credstring(self):
if self.user:
return '%s:%s@' % (self.user, self.key)
return ''
def get_uri(self):
return "%s://%s%s/%s/%s" % (
self.scheme,
self._get_credstring(),
self.authurl,
self.container,
self.obj)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python. It also deals with the peculiarity that new-style
Swift URIs have where a username can contain a ':', like so:
swift://account:user:pass@authurl.com/container/obj
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path.lstrip('/')
if netloc != '':
# > Python 2.6.1
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
else:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
netloc = path[0:path.find('/')].strip('/')
path = path[path.find('/'):].strip('/')
if creds:
cred_parts = creds.split(':')
# User can be account:user, in which case cred_parts[0:2] will be
# the account and user. Combine them into a single username of
# account:user
if len(cred_parts) == 1:
reason = "Badly formed credentials '%s' in Swift URI" % creds
raise exception.BadStoreUri(uri, reason)
elif len(cred_parts) == 3:
user = ':'.join(cred_parts[0:2])
else:
user = cred_parts[0]
key = cred_parts[-1]
self.user = user
self.key = key
else:
self.user = None
path_parts = path.split('/')
try:
self.obj = path_parts.pop()
self.container = path_parts.pop()
self.authurl = netloc
if len(path_parts) > 0:
self.authurl = netloc + '/' + '/'.join(path_parts).strip('/')
except IndexError:
reason = "Badly formed Swift URI"
raise exception.BadStoreUri(uri, reason)
class SwiftBackend(glance.store.Backend):
"""An implementation of the swift backend adapter."""
@ -39,31 +138,33 @@ class SwiftBackend(glance.store.Backend):
CHUNKSIZE = 65536
@classmethod
def get(cls, parsed_uri, expected_size=None, options=None):
def get(cls, location, expected_size=None, options=None):
"""
Takes a parsed_uri in the format of:
swift://user:password@auth_url/container/file.gz.0, connects to the
swift instance at auth_url and downloads the file. Returns the
generator resp_body provided by get_object.
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator from Swift
provided by Swift client's get_object() method.
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
"""
from swift.common import client as swift_client
(user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri)
# TODO(sirp): snet=False for now, however, if the instance of
# swift we're talking to is within our same region, we should set
# snet=True
loc = location.store_location
swift_conn = swift_client.Connection(
authurl=authurl, user=user, key=key, snet=False)
authurl=loc.authurl, user=loc.user, key=loc.key, snet=False)
try:
(resp_headers, resp_body) = swift_conn.get_object(
container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE)
container=loc.container, obj=loc.obj,
resp_chunk_size=cls.CHUNKSIZE)
except swift_client.ClientException, e:
if e.http_status == httplib.NOT_FOUND:
location = format_swift_location(user, key, authurl,
container, obj)
uri = location.get_store_uri()
raise exception.NotFound("Swift could not find image at "
"location %(location)s" % locals())
"uri %(uri)s" % locals())
if expected_size:
obj_size = int(resp_headers['content-length'])
@ -98,6 +199,10 @@ class SwiftBackend(glance.store.Backend):
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:param id: The opaque image identifier
:param data: The image data to write, as a file-like object
:param options: Conf mapping
@ -119,9 +224,14 @@ class SwiftBackend(glance.store.Backend):
user = cls._option_get(options, 'swift_store_user')
key = cls._option_get(options, 'swift_store_key')
full_auth_address = auth_address
if not full_auth_address.startswith('http'):
full_auth_address = 'https://' + full_auth_address
scheme = 'swift+https'
if auth_address.startswith('http://'):
scheme = 'swift+http'
full_auth_address = auth_address
elif auth_address.startswith('https://'):
full_auth_address = auth_address
else:
full_auth_address = 'https://' + auth_address # Defaults https
swift_conn = swift_client.Connection(
authurl=full_auth_address, user=user, key=key, snet=False)
@ -133,8 +243,13 @@ class SwiftBackend(glance.store.Backend):
create_container_if_missing(container, swift_conn, options)
obj_name = str(id)
location = format_swift_location(user, key, auth_address,
container, obj_name)
location = StoreLocation({'scheme': scheme,
'container': container,
'obj': obj_name,
'authurl': auth_address,
'user': user,
'key': key})
try:
obj_etag = swift_conn.put_object(container, obj_name, data)
@ -152,7 +267,7 @@ class SwiftBackend(glance.store.Backend):
# header keys are lowercased by Swift
if 'content-length' in resp_headers:
size = int(resp_headers['content-length'])
return (location, size, obj_etag)
return (location.get_uri(), size, obj_etag)
except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate("Swift already has an image at "
@ -162,89 +277,34 @@ class SwiftBackend(glance.store.Backend):
raise glance.store.BackendException(msg)
@classmethod
def delete(cls, parsed_uri):
def delete(cls, location):
"""
Deletes the swift object(s) at the parsed_uri location
Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
"""
from swift.common import client as swift_client
(user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri)
# TODO(sirp): snet=False for now, however, if the instance of
# swift we're talking to is within our same region, we should set
# snet=True
loc = location.store_location
swift_conn = swift_client.Connection(
authurl=authurl, user=user, key=key, snet=False)
authurl=loc.authurl, user=loc.user, key=loc.key, snet=False)
try:
swift_conn.delete_object(container, obj)
swift_conn.delete_object(loc.container, loc.obj)
except swift_client.ClientException, e:
if e.http_status == httplib.NOT_FOUND:
location = format_swift_location(user, key, authurl,
container, obj)
uri = location.get_store_uri()
raise exception.NotFound("Swift could not find image at "
"location %(location)s" % locals())
"uri %(uri)s" % locals())
else:
raise
def parse_swift_tokens(parsed_uri):
"""
Return the various tokens used by Swift.
:param parsed_uri: The pieces of a URI returned by urlparse
:retval A tuple of (user, key, auth_address, container, obj_name)
"""
path = parsed_uri.path.lstrip('//')
netloc = parsed_uri.netloc
try:
try:
creds, netloc = netloc.split('@')
path = '/'.join([netloc, path])
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
creds, path = path.split('@')
cred_parts = creds.split(':')
# User can be account:user, in which case cred_parts[0:2] will be
# the account and user. Combine them into a single username of
# account:user
if len(cred_parts) == 3:
user = ':'.join(cred_parts[0:2])
else:
user = cred_parts[0]
key = cred_parts[-1]
path_parts = path.split('/')
obj = path_parts.pop()
container = path_parts.pop()
except (ValueError, IndexError):
raise glance.store.BackendException(
"Expected four values to unpack in: swift:%s. "
"Should have received something like: %s."
% (parsed_uri.path, SwiftBackend.EXAMPLE_URL))
authurl = "https://%s" % '/'.join(path_parts)
return user, key, authurl, container, obj
def format_swift_location(user, key, auth_address, container, obj_name):
"""
Returns the swift URI in the format:
swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<OBJNAME>
:param user: The swift user to authenticate with
:param key: The auth key for the authenticating user
:param auth_address: The base URL for the authentication service
:param container: The name of the container
:param obj_name: The name of the object
"""
return "swift://%(user)s:%(key)s@%(auth_address)s/"\
"%(container)s/%(obj_name)s" % locals()
def create_container_if_missing(container, swift_conn, options):
"""
Creates a missing container in Swift if the

65
tests/functional/__init__.py
View File

@ -127,7 +127,7 @@ class ApiServer(Server):
Server object that starts/stops/manages the API server
"""
def __init__(self, test_dir, port, registry_port):
def __init__(self, test_dir, port, registry_port, delayed_delete=False):
super(ApiServer, self).__init__(test_dir, port)
self.server_name = 'api'
self.default_store = 'file'
@ -137,6 +137,7 @@ class ApiServer(Server):
"api.pid")
self.log_file = os.path.join(self.test_dir, "api.log")
self.registry_port = registry_port
self.delayed_delete = delayed_delete
self.conf_base = """[DEFAULT]
verbose = %(verbose)s
debug = %(debug)s
@ -147,9 +148,10 @@ bind_port = %(bind_port)s
registry_host = 0.0.0.0
registry_port = %(registry_port)s
log_file = %(log_file)s
delayed_delete = %(delayed_delete)s
[pipeline:glance-api]
pipeline = versionnegotiation apiv1app
pipeline = versionnegotiation context apiv1app
[pipeline:versions]
pipeline = versionsapp
@ -162,6 +164,9 @@ paste.app_factory = glance.api.v1:app_factory
[filter:versionnegotiation]
paste.filter_factory = glance.api.middleware.version_negotiation:filter_factory
[filter:context]
paste.filter_factory = glance.common.context:filter_factory
"""
@ -191,8 +196,44 @@ log_file = %(log_file)s
sql_connection = %(sql_connection)s
sql_idle_timeout = 3600
[app:glance-registry]
[pipeline:glance-registry]
pipeline = context registryapp
[app:registryapp]
paste.app_factory = glance.registry.server:app_factory
[filter:context]
paste.filter_factory = glance.common.context:filter_factory
"""
class ScrubberDaemon(Server):
"""
Server object that starts/stops/manages the Scrubber server
"""
def __init__(self, test_dir, sql_connection, daemon=False):
# NOTE(jkoelker): Set the port to 0 since we actually don't listen
super(ScrubberDaemon, self).__init__(test_dir, 0)
self.server_name = 'scrubber'
self.daemon = daemon
self.sql_connection = sql_connection
self.pid_file = os.path.join(self.test_dir, "scrubber.pid")
self.log_file = os.path.join(self.test_dir, "scrubber.log")
self.conf_base = """[DEFAULT]
verbose = %(verbose)s
debug = %(debug)s
log_file = %(log_file)s
scrub_time = 5
daemon = %(daemon)s
wakeup_time = 2
sql_connection = %(sql_connection)s
sql_idle_timeout = 3600
[app:glance-scrubber]
paste.app_factory = glance.store.scrubber:app_factory
"""
@ -217,8 +258,13 @@ class FunctionalTest(unittest.TestCase):
self.registry_server = RegistryServer(self.test_dir,
self.registry_port)
registry_db = self.registry_server.sql_connection
self.scrubber_daemon = ScrubberDaemon(self.test_dir,
sql_connection=registry_db)
self.pid_files = [self.api_server.pid_file,
self.registry_server.pid_file]
self.registry_server.pid_file,
self.scrubber_daemon.pid_file]
self.files_to_destroy = []
def tearDown(self):
@ -304,6 +350,13 @@ class FunctionalTest(unittest.TestCase):
"Got: %s" % err)
self.assertTrue("Starting glance-registry with" in out)
exitcode, out, err = self.scrubber_daemon.start(**kwargs)
self.assertEqual(0, exitcode,
"Failed to spin up the Scrubber daemon. "
"Got: %s" % err)
self.assertTrue("Starting glance-scrubber with" in out)
self.wait_for_servers()
def ping_server(self, port):
@ -361,6 +414,10 @@ class FunctionalTest(unittest.TestCase):
"Failed to spin down the Registry server. "
"Got: %s" % err)
exitcode, out, err = self.scrubber_daemon.stop()
self.assertEqual(0, exitcode,
"Failed to spin down the Scrubber daemon. "
"Got: %s" % err)
# If all went well, then just remove the test directory.
# We only want to check the logs and stuff if something
# went wrong...

2
tests/functional/test_curl_api.py
View File

@ -1227,3 +1227,5 @@ class TestCurlApi(functional.FunctionalTest):
self.assertEqual(images[0]['id'], 1)
self.assertEqual(images[1]['id'], 3)
self.assertEqual(images[2]['id'], 2)
self.stop_servers()

486
tests/functional/test_httplib2_api.py
View File

@ -21,6 +21,7 @@ import hashlib
import httplib2
import json
import os
import tempfile
from tests import functional
from tests.utils import execute
@ -590,3 +591,488 @@ class TestApiHttplib2(functional.FunctionalTest):
self.assertEqual(response['x-image-meta-is_public'], 'True')
self.stop_servers()
def test_traceback_not_consumed(self):
"""
A test that errors coming from the POST API do not
get consumed and print the actual error message, and
not something like &lt;traceback object at 0x1918d40&gt;
:see https://bugs.launchpad.net/glance/+bug/755912
"""
self.cleanup()
self.start_servers()
# POST /images with binary data, but not setting
# Content-Type to application/octet-stream, verify a
# 400 returned and that the error is readable.
with tempfile.NamedTemporaryFile() as test_data_file:
test_data_file.write("XXX")
test_data_file.flush()
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST',
body=test_data_file.name)
self.assertEqual(response.status, 400)
expected = "Content-Type must be application/octet-stream"
self.assertTrue(expected in content,
"Could not find '%s' in '%s'" % (expected, content))
self.stop_servers()
def test_filtered_images(self):
"""
Set up four test images and ensure each query param filter works
"""
self.cleanup()
self.start_servers()
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with three public images, and one private image
# with various attributes
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vdi',
'X-Image-Meta-Size': '19',
'X-Image-Meta-Is-Public': 'True',
'X-Image-Meta-Property-pants': 'are on'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['properties']['pants'], "are on")
self.assertEqual(data['image']['is_public'], True)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'My Image!',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vhd',
'X-Image-Meta-Size': '20',
'X-Image-Meta-Is-Public': 'True',
'X-Image-Meta-Property-pants': 'are on'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['properties']['pants'], "are on")
self.assertEqual(data['image']['is_public'], True)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'My Image!',
'X-Image-Meta-Status': 'saving',
'X-Image-Meta-Container-Format': 'ami',
'X-Image-Meta-Disk-Format': 'ami',
'X-Image-Meta-Size': '21',
'X-Image-Meta-Is-Public': 'True',
'X-Image-Meta-Property-pants': 'are off'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['properties']['pants'], "are off")
self.assertEqual(data['image']['is_public'], True)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'My Private Image',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ami',
'X-Image-Meta-Disk-Format': 'ami',
'X-Image-Meta-Size': '22',
'X-Image-Meta-Is-Public': 'False'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['is_public'], False)
# 2. GET /images
# Verify three public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
# 3. GET /images with name filter
# Verify correct images returned with name
params = "name=My%20Image!"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertEqual(image['name'], "My Image!")
# 4. GET /images with status filter
# Verify correct images returned with status
params = "status=queued"
path = "http://%s:%d/v1/images/detail?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
for image in data['images']:
self.assertEqual(image['status'], "queued")
params = "status=active"
path = "http://%s:%d/v1/images/detail?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 0)
# 5. GET /images with container_format filter
# Verify correct images returned with container_format
params = "container_format=ovf"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertEqual(image['container_format'], "ovf")
# 6. GET /images with disk_format filter
# Verify correct images returned with disk_format
params = "disk_format=vdi"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
for image in data['images']:
self.assertEqual(image['disk_format'], "vdi")
# 7. GET /images with size_max filter
# Verify correct images returned with size <= expected
params = "size_max=20"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertTrue(image['size'] <= 20)
# 8. GET /images with size_min filter
# Verify correct images returned with size >= expected
params = "size_min=20"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertTrue(image['size'] >= 20)
# 9. Get /images with is_public=None filter
# Verify correct images returned with property
# Bug lp:803656 Support is_public in filtering
params = "is_public=None"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 4)
# 10. Get /images with is_public=False filter
# Verify correct images returned with property
# Bug lp:803656 Support is_public in filtering
params = "is_public=False"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
for image in data['images']:
self.assertEqual(image['name'], "My Private Image")
# 11. Get /images with is_public=True filter
# Verify correct images returned with property
# Bug lp:803656 Support is_public in filtering
params = "is_public=True"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
for image in data['images']:
self.assertNotEqual(image['name'], "My Private Image")
# 12. GET /images with property filter
# Verify correct images returned with property
params = "property-pants=are%20on"
path = "http://%s:%d/v1/images/detail?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
for image in data['images']:
self.assertEqual(image['properties']['pants'], "are on")
# 13. GET /images with property filter and name filter
# Verify correct images returned with property and name
# Make sure you quote the url when using more than one param!
params = "name=My%20Image!&property-pants=are%20on"
path = "http://%s:%d/v1/images/detail?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
for image in data['images']:
self.assertEqual(image['properties']['pants'], "are on")
self.assertEqual(image['name'], "My Image!")
self.stop_servers()
def test_limited_images(self):
"""
Ensure marker and limit query params work
"""
self.cleanup()
self.start_servers()
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with three public images with various attributes
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image2',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image3',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
# 2. GET /images with limit of 2
# Verify only two images were returned
params = "limit=2"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
self.assertEqual(data['images'][0]['id'], 3)
self.assertEqual(data['images'][1]['id'], 2)
# 3. GET /images with marker
# Verify only two images were returned
params = "marker=3"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 2)
self.assertEqual(data['images'][0]['id'], 2)
self.assertEqual(data['images'][1]['id'], 1)
# 4. GET /images with marker and limit
# Verify only one image was returned with the correct id
params = "limit=1&marker=2"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
self.assertEqual(data['images'][0]['id'], 1)
# 5. GET /images/detail with marker and limit
# Verify only one image was returned with the correct id
params = "limit=1&marker=3"
path = "http://%s:%d/v1/images?%s" % (
"0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 1)
self.assertEqual(data['images'][0]['id'], 2)
self.stop_servers()
def test_ordered_images(self):
"""
Set up three test images and ensure each query param filter works
"""
self.cleanup()
self.start_servers()
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with three public images with various attributes
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vdi',
'X-Image-Meta-Size': '19',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'ASDF',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'bare',
'X-Image-Meta-Disk-Format': 'iso',
'X-Image-Meta-Size': '2',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'XYZ',
'X-Image-Meta-Status': 'saving',
'X-Image-Meta-Container-Format': 'ami',
'X-Image-Meta-Disk-Format': 'ami',
'X-Image-Meta-Size': '5',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
# 2. GET /images with no query params
# Verify three public images sorted by created_at desc
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
self.assertEqual(data['images'][0]['id'], 3)
self.assertEqual(data['images'][1]['id'], 2)
self.assertEqual(data['images'][2]['id'], 1)
# 3. GET /images sorted by name asc
params = 'sort_key=name&sort_dir=asc'
path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
self.assertEqual(data['images'][0]['id'], 2)
self.assertEqual(data['images'][1]['id'], 1)
self.assertEqual(data['images'][2]['id'], 3)
# 4. GET /images sorted by size desc
params = 'sort_key=size&sort_dir=desc'
path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
self.assertEqual(data['images'][0]['id'], 1)
self.assertEqual(data['images'][1]['id'], 3)
self.assertEqual(data['images'][2]['id'], 2)
self.stop_servers()
def test_duplicate_image_upload(self):
"""
Upload initial image, then attempt to upload duplicate image
"""
self.cleanup()
self.start_servers()
# 0. GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# 1. POST /images with public image named Image1
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vdi',
'X-Image-Meta-Size': '19',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
# 2. POST /images with public image named Image1, and ID: 1
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1 Update',
'X-Image-Meta-Status': 'active',
'X-Image-Meta-Container-Format': 'ovf',
'X-Image-Meta-Disk-Format': 'vdi',
'X-Image-Meta-Size': '19',
'X-Image-Meta-Id': '1',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 409)
expected = "An image with identifier 1 already exists"
self.assertTrue(expected in content,
"Could not find '%s' in '%s'" % (expected, content))
self.stop_servers()

111
tests/functional/test_scrubber.py Normal file
View File

@ -0,0 +1,111 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import time
import unittest
from sqlalchemy import create_engine
from tests import functional
from tests.utils import execute
from glance import client
TEST_IMAGE_DATA = '*' * 5 * 1024
TEST_IMAGE_META = {'name': 'test_image',
'is_public': False,
'disk_format': 'raw',
'container_format': 'ovf'}
class TestScrubber(functional.FunctionalTest):
"""Test that delayed_delete works and the scrubber deletes"""
def _get_client(self):
return client.Client("localhost", self.api_port)
@functional.runs_sql
def test_immediate_delete(self):
"""
test that images get deleted immediately by default
"""
self.cleanup()
self.start_servers()
client = self._get_client()
meta = client.add_image(TEST_IMAGE_META, TEST_IMAGE_DATA)
id = meta['id']
sql = "SELECT * FROM images WHERE status = 'pending_delete'"
recs = list(self.run_sql_cmd(sql))
self.assertFalse(recs)
client.delete_image(id)
recs = list(self.run_sql_cmd(sql))
self.assertFalse(recs)
sql = "SELECT * FROM images WHERE id = '%s'" % id
recs = list(self.run_sql_cmd(sql))
self.assertTrue(recs)
for rec in recs:
self.assertEqual(rec['status'], 'deleted')
self.stop_servers()
@functional.runs_sql
def test_delayed_delete(self):
"""
test that images don't get deleted immediatly and that the scrubber
scrubs them
"""
self.cleanup()
registry_db = self.registry_server.sql_connection
self.start_servers(delayed_delete=True, sql_connection=registry_db,
daemon=True)
client = self._get_client()
meta = client.add_image(TEST_IMAGE_META, TEST_IMAGE_DATA)
id = meta['id']
sql = "SELECT * FROM images WHERE status = 'pending_delete'"
recs = list(self.run_sql_cmd(sql))
self.assertFalse(recs)
client.delete_image(id)
recs = self.run_sql_cmd(sql)
self.assertTrue(recs)
sql = "SELECT * FROM images WHERE id = '%s'" % id
recs = list(self.run_sql_cmd(sql))
self.assertTrue(recs)
for rec in recs:
self.assertEqual(rec['status'], 'pending_delete')
# Wait 15 seconds for the scrubber to scrub
time.sleep(15)
recs = list(self.run_sql_cmd(sql))
self.assertTrue(recs)
for rec in recs:
self.assertEqual(rec['status'], 'deleted')
self.stop_servers()

38
tests/stubs.py
View File

@ -29,6 +29,7 @@ import stubout
import webob
import glance.common.client
from glance.common import context
from glance.common import exception
from glance.registry import server as rserver
from glance.api import v1 as server
@ -128,13 +129,9 @@ def stub_out_s3_backend(stubs):
DATA = 'I am a teapot, short and stout\n'
@classmethod
def get(cls, parsed_uri, expected_size, conn_class=None):
def get(cls, location, expected_size, conn_class=None):
S3Backend = glance.store.s3.S3Backend
# raise BackendException if URI is bad.
(user, key, authurl, container, obj) = \
S3Backend._parse_s3_tokens(parsed_uri)
def chunk_it():
for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE):
yield cls.DATA[i:i + cls.CHUNK_SIZE]
@ -176,7 +173,8 @@ def stub_out_registry_and_store_server(stubs):
"sqlite://")
options = {'sql_connection': sql_connection, 'verbose': VERBOSE,
'debug': DEBUG}
res = self.req.get_response(rserver.API(options))
api = context.ContextMiddleware(rserver.API(options), options)
res = self.req.get_response(api)
# httplib.Response has a read() method...fake it out
def fake_reader():
@ -227,7 +225,8 @@ def stub_out_registry_and_store_server(stubs):
'registry_port': '9191',
'default_store': 'file',
'filesystem_store_datadir': FAKE_FILESYSTEM_ROOTDIR}
res = self.req.get_response(server.API(options))
api = context.ContextMiddleware(server.API(options), options)
res = self.req.get_response(api)
# httplib.Response has a read() method...fake it out
def fake_reader():
@ -307,6 +306,7 @@ def stub_out_registry_db_image_api(stubs):
def __init__(self):
self.images = FakeDatastore.FIXTURES
self.deleted_images = []
self.next_id = 3
def image_create(self, _context, values):
@ -379,6 +379,8 @@ def stub_out_registry_db_image_api(stubs):
def image_destroy(self, _context, image_id):
image = self.image_get(_context, image_id)
self.images.remove(image)
image['deleted_at'] = datetime.datetime.utcnow()
self.deleted_images.append(image)
def image_get(self, _context, image_id):
@ -390,9 +392,16 @@ def stub_out_registry_db_image_api(stubs):
else:
return images[0]
def image_get_all_public(self, _context, filters=None, marker=None,
limit=1000, sort_key=None, sort_dir=None):
images = [f for f in self.images if f['is_public'] == True]
def image_get_all_pending_delete(self, _context, delete_time=None,
limit=None):
images = [f for f in self.deleted_images \
if f['status'] == 'pending_delete' and \
f['deleted_at'] <= delete_time]
return images
def image_get_all(self, _context, filters=None, marker=None,
limit=1000, sort_key=None, sort_dir=None):
images = self.images
if 'size_min' in filters:
size_min = int(filters.pop('size_min'))
@ -414,7 +423,8 @@ def stub_out_registry_db_image_api(stubs):
images = filter(_prop_filter(k, v), images)
for k, v in filters.items():
images = [f for f in images if f[k] == v]
if v is not None:
images = [f for f in images if f[k] == v]
# sorted func expects func that compares in descending order
def image_cmp(x, y):
@ -461,5 +471,7 @@ def stub_out_registry_db_image_api(stubs):
fake_datastore.image_destroy)
stubs.Set(glance.registry.db.api, 'image_get',
fake_datastore.image_get)
stubs.Set(glance.registry.db.api, 'image_get_all_public',
fake_datastore.image_get_all_public)
stubs.Set(glance.registry.db.api, 'image_get_all_pending_delete',
fake_datastore.image_get_all_pending_delete)
stubs.Set(glance.registry.db.api, 'image_get_all',
fake_datastore.image_get_all)

86
tests/unit/test_api.py
View File

@ -26,6 +26,7 @@ import stubout
import webob
from glance.api import v1 as server
from glance.common import context
from glance.registry import server as rserver
import glance.registry.db.api
from tests import stubs
@ -41,8 +42,9 @@ class TestRegistryAPI(unittest.TestCase):
stubs.stub_out_registry_and_store_server(self.stubs)
stubs.stub_out_registry_db_image_api(self.stubs)
stubs.stub_out_filesystem_backend()
self.api = rserver.API({'verbose': VERBOSE,
'debug': DEBUG})
options = {'verbose': VERBOSE,
'debug': DEBUG}
self.api = context.ContextMiddleware(rserver.API(options), options)
def tearDown(self):
"""Clear the test environment"""
@ -1077,6 +1079,84 @@ class TestRegistryAPI(unittest.TestCase):
for image in images:
self.assertEqual('v a', image['properties']['prop_123'])
def test_get_details_filter_public_none(self):
"""
Tests that the /images/detail registry API returns list of
all images if is_public none is passed
"""
extra_fixture = {'id': 3,
'status': 'active',
'is_public': False,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
glance.registry.db.api.image_create(None, extra_fixture)
req = webob.Request.blank('/images/detail?is_public=None')
res = req.get_response(self.api)
res_dict = json.loads(res.body)
self.assertEquals(res.status_int, 200)
images = res_dict['images']
self.assertEquals(len(images), 3)
def test_get_details_filter_public_false(self):
"""
Tests that the /images/detail registry API returns list of
private images if is_public false is passed
"""
extra_fixture = {'id': 3,
'status': 'active',
'is_public': False,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
glance.registry.db.api.image_create(None, extra_fixture)
req = webob.Request.blank('/images/detail?is_public=False')
res = req.get_response(self.api)
res_dict = json.loads(res.body)
self.assertEquals(res.status_int, 200)
images = res_dict['images']
self.assertEquals(len(images), 2)
for image in images:
self.assertEqual(False, image['is_public'])
def test_get_details_filter_public_true(self):
"""
Tests that the /images/detail registry API returns list of
public images if is_public true is passed (same as default)
"""
extra_fixture = {'id': 3,
'status': 'active',
'is_public': False,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
glance.registry.db.api.image_create(None, extra_fixture)
req = webob.Request.blank('/images/detail?is_public=True')
res = req.get_response(self.api)
res_dict = json.loads(res.body)
self.assertEquals(res.status_int, 200)
images = res_dict['images']
self.assertEquals(len(images), 1)
for image in images:
self.assertEqual(True, image['is_public'])
def test_get_details_sort_name_asc(self):
"""
Tests that the /images/details registry API returns list of
@ -1380,7 +1460,7 @@ class TestGlanceAPI(unittest.TestCase):
'sql_connection': sql_connection,
'default_store': 'file',
'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR}
self.api = server.API(options)
self.api = context.ContextMiddleware(server.API(options), options)
def tearDown(self):
"""Clear the test environment"""

148
tests/unit/test_context.py Normal file
View File

@ -0,0 +1,148 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010-2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from glance.common import context
class FakeImage(object):
"""
Fake image for providing the image attributes needed for
TestContext.
"""
def __init__(self, owner, is_public):
self.owner = owner
self.is_public = is_public
class TestContext(unittest.TestCase):
def do_visible(self, exp_res, img_owner, img_public, **kwargs):
"""
Perform a context test. Creates a (fake) image with the
specified owner and is_public attributes, then creates a
context with the given keyword arguments and expects exp_res
as the result of an is_image_visible() call on the context.
"""
img = FakeImage(img_owner, img_public)
ctx = context.RequestContext(**kwargs)
self.assertEqual(ctx.is_image_visible(img), exp_res)
def test_empty_public(self):
"""
Tests that an empty context (with is_admin set to True) can
access an image with is_public set to True.
"""
self.do_visible(True, None, True, is_admin=True)
def test_empty_public_owned(self):
"""
Tests that an empty context (with is_admin set to True) can
access an owned image with is_public set to True.
"""
self.do_visible(True, 'pattieblack', True, is_admin=True)
def test_empty_private(self):
"""
Tests that an empty context (with is_admin set to True) can
access an image with is_public set to False.
"""
self.do_visible(True, None, False, is_admin=True)
def test_empty_private_owned(self):
"""
Tests that an empty context (with is_admin set to True) can
access an owned image with is_public set to False.
"""
self.do_visible(True, 'pattieblack', False, is_admin=True)
def test_anon_public(self):
"""
Tests that an anonymous context (with is_admin set to False)
can access an image with is_public set to True.
"""
self.do_visible(True, None, True)
def test_anon_public_owned(self):
"""
Tests that an anonymous context (with is_admin set to False)
can access an owned image with is_public set to True.
"""
self.do_visible(True, 'pattieblack', True)
def test_anon_private(self):
"""
Tests that an anonymous context (with is_admin set to False)
can access an unowned image with is_public set to False.
"""
self.do_visible(True, None, False)
def test_anon_private_owned(self):
"""
Tests that an anonymous context (with is_admin set to False)
cannot access an owned image with is_public set to False.
"""
self.do_visible(False, 'pattieblack', False)
def test_auth_public(self):
"""
Tests that an authenticated context (with is_admin set to
False) can access an image with is_public set to True.
"""
self.do_visible(True, None, True, tenant='froggy')
def test_auth_public_unowned(self):
"""
Tests that an authenticated context (with is_admin set to
False) can access an image (which it does not own) with
is_public set to True.
"""
self.do_visible(True, 'pattieblack', True, tenant='froggy')
def test_auth_public_owned(self):
"""
Tests that an authenticated context (with is_admin set to
False) can access an image (which it does own) with is_public
set to True.
"""
self.do_visible(True, 'pattieblack', True, tenant='pattieblack')
def test_auth_private(self):
"""
Tests that an authenticated context (with is_admin set to
False) can access an image with is_public set to False.
"""
self.do_visible(True, None, False, tenant='froggy')
def test_auth_private_unowned(self):
"""
Tests that an authenticated context (with is_admin set to
False) cannot access an image (which it does not own) with
is_public set to False.
"""
self.do_visible(False, 'pattieblack', False, tenant='froggy')
def test_auth_private_owned(self):
"""
Tests that an authenticated context (with is_admin set to
False) can access an image (which it does own) with is_public
set to False.
"""
self.do_visible(True, 'pattieblack', False, tenant='pattieblack')

25
tests/unit/test_filesystem_store.py
View File

@ -20,11 +20,11 @@
import StringIO
import hashlib
import unittest
import urlparse
import stubout
from glance.common import exception
from glance.store.location import get_location_from_uri
from glance.store.filesystem import FilesystemBackend, ChunkedFile
from tests import stubs
@ -51,8 +51,8 @@ class TestFilesystemBackend(unittest.TestCase):
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2")
image_file = FilesystemBackend.get(url_pieces)
loc = get_location_from_uri("file:///tmp/glance-tests/2")
image_file = FilesystemBackend.get(loc)
expected_data = "chunk00000remainder"
expected_num_chunks = 2
@ -70,10 +70,10 @@ class TestFilesystemBackend(unittest.TestCase):
Test that trying to retrieve a file that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing")
loc = get_location_from_uri("file:///tmp/glance-tests/non-existing")
self.assertRaises(exception.NotFound,
FilesystemBackend.get,
url_pieces)
loc)
def test_add(self):
"""Test that we can add an image via the filesystem backend"""
@ -93,8 +93,8 @@ class TestFilesystemBackend(unittest.TestCase):
self.assertEquals(expected_file_size, size)
self.assertEquals(expected_checksum, checksum)
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/42")
new_image_file = FilesystemBackend.get(url_pieces)
loc = get_location_from_uri("file:///tmp/glance-tests/42")
new_image_file = FilesystemBackend.get(loc)
new_image_contents = ""
new_image_file_size = 0
@ -122,20 +122,19 @@ class TestFilesystemBackend(unittest.TestCase):
"""
Test we can delete an existing image in the filesystem store
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2")
FilesystemBackend.delete(url_pieces)
loc = get_location_from_uri("file:///tmp/glance-tests/2")
FilesystemBackend.delete(loc)
self.assertRaises(exception.NotFound,
FilesystemBackend.get,
url_pieces)
loc)
def test_delete_non_existing(self):
"""
Test that trying to delete a file that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing")
loc = get_location_from_uri("file:///tmp/glance-tests/non-existing")
self.assertRaises(exception.NotFound,
FilesystemBackend.delete,
url_pieces)
loc)

243
tests/unit/test_store_location.py Normal file
View File

@ -0,0 +1,243 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from glance.common import exception
import glance.store.location as location
import glance.store.http
import glance.store.filesystem
import glance.store.swift
import glance.store.s3
class TestStoreLocation(unittest.TestCase):
def test_get_location_from_uri_back_to_uri(self):
"""
Test that for various URIs, the correct Location
object can be contructed and then the original URI
returned via the get_store_uri() method.
"""
good_store_uris = [
'https://user:pass@example.com:80/images/some-id',
'http://images.oracle.com/123456',
'swift://account:user:pass@authurl.com/container/obj-id',
'swift+https://account:user:pass@authurl.com/container/obj-id',
's3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id',
's3://accesskey:secretwith/aslash@s3.amazonaws.com/bucket/key-id',
's3+http://accesskey:secret@s3.amazonaws.com/bucket/key-id',
's3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id',
'file:///var/lib/glance/images/1']
for uri in good_store_uris:
loc = location.get_location_from_uri(uri)
# The get_store_uri() method *should* return an identical URI
# to the URI that is passed to get_location_from_uri()
self.assertEqual(loc.get_store_uri(), uri)
def test_bad_store_scheme(self):
"""
Test that a URI with a non-existing scheme triggers exception
"""
bad_uri = 'unknown://user:pass@example.com:80/images/some-id'
self.assertRaises(exception.UnknownScheme,
location.get_location_from_uri,
bad_uri)
def test_filesystem_store_location(self):
"""
Test the specific StoreLocation for the Filesystem store
"""
uri = 'file:///var/lib/glance/images/1'
loc = glance.store.filesystem.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("file", loc.scheme)
self.assertEqual("/var/lib/glance/images/1", loc.path)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'fil://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 'file://'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
def test_http_store_location(self):
"""
Test the specific StoreLocation for the HTTP store
"""
uri = 'http://example.com/images/1'
loc = glance.store.http.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("http", loc.scheme)
self.assertEqual("example.com", loc.netloc)
self.assertEqual("/images/1", loc.path)
self.assertEqual(uri, loc.get_uri())
uri = 'https://example.com:8080/images/container/1'
loc.parse_uri(uri)
self.assertEqual("https", loc.scheme)
self.assertEqual("example.com:8080", loc.netloc)
self.assertEqual("/images/container/1", loc.path)
self.assertEqual(uri, loc.get_uri())
uri = 'https://user:password@example.com:8080/images/container/1'
loc.parse_uri(uri)
self.assertEqual("https", loc.scheme)
self.assertEqual("example.com:8080", loc.netloc)
self.assertEqual("user", loc.user)
self.assertEqual("password", loc.password)
self.assertEqual("/images/container/1", loc.path)
self.assertEqual(uri, loc.get_uri())
uri = 'https://user:@example.com:8080/images/1'
loc.parse_uri(uri)
self.assertEqual("https", loc.scheme)
self.assertEqual("example.com:8080", loc.netloc)
self.assertEqual("user", loc.user)
self.assertEqual("", loc.password)
self.assertEqual("/images/1", loc.path)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'htt://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 'http://'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
bad_uri = 'http://user@example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
def test_swift_store_location(self):
"""
Test the specific StoreLocation for the Swift store
"""
uri = 'swift://example.com/images/1'
loc = glance.store.swift.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("swift", loc.scheme)
self.assertEqual("example.com", loc.authurl)
self.assertEqual("images", loc.container)
self.assertEqual("1", loc.obj)
self.assertEqual(None, loc.user)
self.assertEqual(uri, loc.get_uri())
uri = 'swift+https://user:pass@authurl.com/images/1'
loc.parse_uri(uri)
self.assertEqual("swift+https", loc.scheme)
self.assertEqual("authurl.com", loc.authurl)
self.assertEqual("images", loc.container)
self.assertEqual("1", loc.obj)
self.assertEqual("user", loc.user)
self.assertEqual("pass", loc.key)
self.assertEqual(uri, loc.get_uri())
uri = 'swift+https://user:pass@authurl.com/v1/container/12345'
loc.parse_uri(uri)
self.assertEqual("swift+https", loc.scheme)
self.assertEqual("authurl.com/v1", loc.authurl)
self.assertEqual("container", loc.container)
self.assertEqual("12345", loc.obj)
self.assertEqual("user", loc.user)
self.assertEqual("pass", loc.key)
self.assertEqual(uri, loc.get_uri())
uri = 'swift://account:user:pass@authurl.com/v1/container/12345'
loc.parse_uri(uri)
self.assertEqual("swift", loc.scheme)
self.assertEqual("authurl.com/v1", loc.authurl)
self.assertEqual("container", loc.container)
self.assertEqual("12345", loc.obj)
self.assertEqual("account:user", loc.user)
self.assertEqual("pass", loc.key)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'swif://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 'swift://'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
bad_uri = 'swift://user@example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
def test_s3_store_location(self):
"""
Test the specific StoreLocation for the S3 store
"""
uri = 's3://example.com/images/1'
loc = glance.store.s3.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("s3", loc.scheme)
self.assertEqual("example.com", loc.s3serviceurl)
self.assertEqual("images", loc.bucket)
self.assertEqual("1", loc.key)
self.assertEqual(None, loc.accesskey)
self.assertEqual(uri, loc.get_uri())
uri = 's3+https://accesskey:pass@s3serviceurl.com/images/1'
loc.parse_uri(uri)
self.assertEqual("s3+https", loc.scheme)
self.assertEqual("s3serviceurl.com", loc.s3serviceurl)
self.assertEqual("images", loc.bucket)
self.assertEqual("1", loc.key)
self.assertEqual("accesskey", loc.accesskey)
self.assertEqual("pass", loc.secretkey)
self.assertEqual(uri, loc.get_uri())
uri = 's3+https://accesskey:pass@s3serviceurl.com/v1/bucket/12345'
loc.parse_uri(uri)
self.assertEqual("s3+https", loc.scheme)
self.assertEqual("s3serviceurl.com/v1", loc.s3serviceurl)
self.assertEqual("bucket", loc.bucket)
self.assertEqual("12345", loc.key)
self.assertEqual("accesskey", loc.accesskey)
self.assertEqual("pass", loc.secretkey)
self.assertEqual(uri, loc.get_uri())
uri = 's3://accesskey:pass/withslash@s3serviceurl.com/v1/bucket/12345'
loc.parse_uri(uri)
self.assertEqual("s3", loc.scheme)
self.assertEqual("s3serviceurl.com/v1", loc.s3serviceurl)
self.assertEqual("bucket", loc.bucket)
self.assertEqual("12345", loc.key)
self.assertEqual("accesskey", loc.accesskey)
self.assertEqual("pass/withslash", loc.secretkey)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'swif://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 's3://'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
bad_uri = 's3://accesskey@example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)

1
tests/unit/test_stores.py
View File

@ -19,7 +19,6 @@ from StringIO import StringIO
import stubout
import unittest
import urlparse
from glance.store.s3 import S3Backend
from glance.store import Backend, BackendException, get_from_backend

152
tests/unit/test_swift_store.py
View File

@ -29,9 +29,8 @@ import swift.common.client
from glance.common import exception
from glance.store import BackendException
from glance.store.swift import (SwiftBackend,
format_swift_location,
parse_swift_tokens)
from glance.store.swift import SwiftBackend
from glance.store.location import get_location_from_uri
FIVE_KB = (5 * 1024)
SWIFT_OPTIONS = {'verbose': True,
@ -146,6 +145,18 @@ def stub_out_swift_common_client(stubs):
'http_connection', fake_http_connection)
def format_swift_location(user, key, authurl, container, obj):
"""
Helper method that returns a Swift store URI given
the component pieces.
"""
scheme = 'swift+https'
if authurl.startswith('http://'):
scheme = 'swift+http'
return "%s://%s:%s@%s/%s/%s" % (scheme, user, key, authurl,
container, obj)
class TestSwiftBackend(unittest.TestCase):
def setUp(self):
@ -157,46 +168,27 @@ class TestSwiftBackend(unittest.TestCase):
"""Clear the test environment"""
self.stubs.UnsetAll()
def test_parse_swift_tokens(self):
"""
Test that the parse_swift_tokens function returns
user, key, authurl, container, and objname properly
"""
uri = "swift://user:key@localhost/v1.0/container/objname"
url_pieces = urlparse.urlparse(uri)
user, key, authurl, container, objname =\
parse_swift_tokens(url_pieces)
self.assertEqual("user", user)
self.assertEqual("key", key)
self.assertEqual("https://localhost/v1.0", authurl)
self.assertEqual("container", container)
self.assertEqual("objname", objname)
uri = "swift://user:key@localhost:9090/v1.0/container/objname"
url_pieces = urlparse.urlparse(uri)
user, key, authurl, container, objname =\
parse_swift_tokens(url_pieces)
self.assertEqual("user", user)
self.assertEqual("key", key)
self.assertEqual("https://localhost:9090/v1.0", authurl)
self.assertEqual("container", container)
self.assertEqual("objname", objname)
uri = "swift://account:user:key@localhost:9090/v1.0/container/objname"
url_pieces = urlparse.urlparse(uri)
user, key, authurl, container, objname =\
parse_swift_tokens(url_pieces)
self.assertEqual("account:user", user)
self.assertEqual("key", key)
self.assertEqual("https://localhost:9090/v1.0", authurl)
self.assertEqual("container", container)
self.assertEqual("objname", objname)
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
image_swift = SwiftBackend.get(url_pieces)
loc = get_location_from_uri("swift://user:key@auth_address/glance/2")
image_swift = SwiftBackend.get(loc)
expected_data = "*" * FIVE_KB
data = ""
for chunk in image_swift:
data += chunk
self.assertEqual(expected_data, data)
def test_get_with_http_auth(self):
"""
Test a retrieval from Swift with an HTTP authurl. This is
specified either via a Location header with swift+http:// or using
http:// in the swift_store_auth_address config value
"""
loc = get_location_from_uri("swift+http://user:key@auth_address/"
"glance/2")
image_swift = SwiftBackend.get(loc)
expected_data = "*" * FIVE_KB
data = ""
@ -210,11 +202,10 @@ class TestSwiftBackend(unittest.TestCase):
Test retrieval of an image with wrong expected_size param
raises an exception
"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
loc = get_location_from_uri("swift://user:key@auth_address/glance/2")
self.assertRaises(BackendException,
SwiftBackend.get,
url_pieces,
loc,
{'expected_size': 42})
def test_get_non_existing(self):
@ -222,11 +213,10 @@ class TestSwiftBackend(unittest.TestCase):
Test that trying to retrieve a swift that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/noexist")
loc = get_location_from_uri("swift://user:key@authurl/glance/noexist")
self.assertRaises(exception.NotFound,
SwiftBackend.get,
url_pieces)
loc)
def test_add(self):
"""Test that we can add an image via the swift backend"""
@ -249,14 +239,62 @@ class TestSwiftBackend(unittest.TestCase):
self.assertEquals(expected_swift_size, size)
self.assertEquals(expected_checksum, checksum)
url_pieces = urlparse.urlparse(expected_location)
new_image_swift = SwiftBackend.get(url_pieces)
loc = get_location_from_uri(expected_location)
new_image_swift = SwiftBackend.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
self.assertEquals(expected_swift_contents, new_image_contents)
self.assertEquals(expected_swift_size, new_image_swift_size)
def test_add_auth_url_variations(self):
"""
Test that we can add an image via the swift backend with
a variety of different auth_address values
"""
variations = ['http://localhost:80',
'http://localhost',
'http://localhost/v1',
'http://localhost/v1/',
'https://localhost',
'https://localhost:8080',
'https://localhost/v1',
'https://localhost/v1/',
'localhost',
'localhost:8080/v1']
i = 42
for variation in variations:
expected_image_id = i
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = \
hashlib.md5(expected_swift_contents).hexdigest()
new_options = SWIFT_OPTIONS.copy()
new_options['swift_store_auth_address'] = variation
expected_location = format_swift_location(
new_options['swift_store_user'],
new_options['swift_store_key'],
new_options['swift_store_auth_address'],
new_options['swift_store_container'],
expected_image_id)
image_swift = StringIO.StringIO(expected_swift_contents)
location, size, checksum = SwiftBackend.add(i, image_swift,
new_options)
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_swift = SwiftBackend.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
self.assertEquals(expected_swift_contents, new_image_contents)
self.assertEquals(expected_swift_size, new_image_swift_size)
i = i + 1
def test_add_no_container_no_create(self):
"""
Tests that adding an image with a non-existing container
@ -306,8 +344,8 @@ class TestSwiftBackend(unittest.TestCase):
self.assertEquals(expected_swift_size, size)
self.assertEquals(expected_checksum, checksum)
url_pieces = urlparse.urlparse(expected_location)
new_image_swift = SwiftBackend.get(url_pieces)
loc = get_location_from_uri(expected_location)
new_image_swift = SwiftBackend.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
@ -356,22 +394,20 @@ class TestSwiftBackend(unittest.TestCase):
"""
Test we can delete an existing image in the swift store
"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
loc = get_location_from_uri("swift://user:key@authurl/glance/2")
SwiftBackend.delete(url_pieces)
SwiftBackend.delete(loc)
self.assertRaises(exception.NotFound,
SwiftBackend.get,
url_pieces)
loc)
def test_delete_non_existing(self):
"""
Test that trying to delete a swift that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/noexist")
loc = get_location_from_uri("swift://user:key@authurl/glance/noexist")
self.assertRaises(exception.NotFound,
SwiftBackend.delete,
url_pieces)
loc)