openstack/glance
Code Issues Proposed changes

Merge trunk and resolve conflicts

Browse Source
This commit is contained in:
jaypipes@gmail.com 2011-03-08 19:07:44 -05:00
commit 0c25ad6ca1
14 changed files with 718 additions and 509 deletions

2
doc/source/installing.rst

@ -73,7 +73,7 @@ Debian/Ubuntu
1. Install Bazaar and build dependencies::
$> sudo apt-get install bzr python-eventlet python-routes python-greenlet
$> sudo apt-get install bzr python-eventlet python-routes python-greenlet swift
$> sudo apt-get install python-argparse python-sqlalchemy python-wsgiref python-pastedeploy
.. note::

31
etc/glance.conf.sample

@ -8,10 +8,6 @@ debug = False
[app:glance-api]
paste.app_factory = glance.server:app_factory
# Directory that the Filesystem backend store
# writes image data to
filesystem_store_datadir=/var/lib/glance/images/
# Which backend store should Glance use by default is not specified
# in a request to add a new image to Glance? Default: 'file'
# Available choices are 'file', 'swift', and 's3'
@ -29,6 +25,33 @@ registry_host = 0.0.0.0
# Port the registry server is listening on
registry_port = 9191
# ============ Filesystem Store Options ========================
# Directory that the Filesystem backend store
# writes image data to
filesystem_store_datadir=/var/lib/glance/images/
# ============ Swift Store Options =============================
# Address where the Swift authentication service lives
# The auth address should be in the form:
# <DOMAIN>[:<PORT>]/<VERSION>/<ACCOUNT>
swift_store_auth_address = 127.0.0.1:8080/v1.0/glance-account
# User to authenticate against the Swift authentication service
swift_store_user = jdoe
# Auth key for the user authenticating against the
# Swift authentication service
swift_store_key = a86850deb2742ec3cb41518e26aa2d89
# Container within the account that the account should use
# for storing images in Swift
swift_store_container = glance
# Do we create the container if it does not exist?
swift_store_create_container_on_put = False
[app:glance-registry]
paste.app_factory = glance.registry.server:app_factory

7
glance/registry/db/migration.py

@ -29,6 +29,9 @@ except ImportError:
from glance.common import exception
logger = logging.getLogger('glance.registry.db.migration')
def db_version(options):
"""Return the database's current migration number
@ -56,7 +59,7 @@ def upgrade(options, version=None):
repo_path = _find_migrate_repo()
sql_connection = options['sql_connection']
version_str = version or 'latest'
logging.info("Upgrading %(sql_connection)s to version %(version_str)s" %
logger.info("Upgrading %(sql_connection)s to version %(version_str)s" %
locals())
return versioning_api.upgrade(sql_connection, repo_path, version)
@ -71,7 +74,7 @@ def downgrade(options, version):
db_version(options) # Ensure db is under migration control
repo_path = _find_migrate_repo()
sql_connection = options['sql_connection']
logging.info("Downgrading %(sql_connection)s to version %(version)s" %
logger.info("Downgrading %(sql_connection)s to version %(version)s" %
locals())
return versioning_api.downgrade(sql_connection, repo_path, version)

3
glance/server.py

@ -156,7 +156,8 @@ class Controller(wsgi.Controller):
def image_iterator():
chunks = get_from_backend(image['location'],
expected_size=image['size'])
expected_size=image['size'],
options=self.options)
for chunk in chunks:
yield chunk

13
glance/store/filesystem.py

@ -19,12 +19,15 @@
A simple filesystem-backed store
"""
import logging
import os
import urlparse
from glance.common import exception
import glance.store
logger = logging.getLogger('glance.store.filesystem')
class ChunkedFile(object):
@ -60,8 +63,7 @@ class ChunkedFile(object):
class FilesystemBackend(glance.store.Backend):
@classmethod
def get(cls, parsed_uri, opener=lambda p: open(p, "rb"),
expected_size=None):
def get(cls, parsed_uri, expected_size=None, options=None):
""" Filesystem-based backend
file:///path/to/file.tar.gz.0
@ -71,6 +73,8 @@ class FilesystemBackend(glance.store.Backend):
if not os.path.exists(filepath):
raise exception.NotFound("Image file %s not found" % filepath)
else:
logger.debug("Found image at %s. Returning in ChunkedFile.",
filepath)
return ChunkedFile(filepath)
@classmethod
@ -87,6 +91,7 @@ class FilesystemBackend(glance.store.Backend):
fn = parsed_uri.path
if os.path.exists(fn):
try:
logger.debug("Deleting image at %s", fn)
os.unlink(fn)
except OSError:
raise exception.NotAuthorized("You cannot delete file %s" % fn)
@ -112,6 +117,8 @@ class FilesystemBackend(glance.store.Backend):
datadir = options['filesystem_store_datadir']
if not os.path.exists(datadir):
logger.info("Directory to write image files does not exist "
"(%s). Creating.", datadir)
os.makedirs(datadir)
filepath = os.path.join(datadir, str(id))
@ -129,6 +136,8 @@ class FilesystemBackend(glance.store.Backend):
bytes_written += len(buf)
f.write(buf)
logger.debug("Wrote %(bytes_written)d bytes to %(filepath)s"
% locals())
return ('file://%s' % filepath, bytes_written)
@classmethod

217
glance/store/swift.py

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 OpenStack, LLC
# Copyright 2010-2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -15,40 +15,59 @@
# License for the specific language governing permissions and limitations
# under the License.
"""Storage backend for SWIFT"""
from __future__ import absolute_import
import httplib
import logging
from swift.common.client import Connection, ClientException
from glance.common import config
from glance.common import exception
import glance.store
DEFAULT_SWIFT_CONTAINER = 'glance'
logger = logging.getLogger('glance.store.swift')
class SwiftBackend(glance.store.Backend):
"""
An implementation of the swift backend adapter.
"""
EXAMPLE_URL = "swift://user:password@auth_url/container/file.gz.0"
EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"
CHUNKSIZE = 65536
@classmethod
def get(cls, parsed_uri, expected_size, conn_class=None):
def get(cls, parsed_uri, expected_size=None, options=None):
"""
Takes a parsed_uri in the format of:
swift://user:password@auth_url/container/file.gz.0, connects to the
swift instance at auth_url and downloads the file. Returns the
generator resp_body provided by get_object.
"""
(user, key, authurl, container, obj) = \
cls._parse_swift_tokens(parsed_uri)
(user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri)
# TODO(sirp): snet=False for now, however, if the instance of
# swift we're talking to is within our same region, we should set
# snet=True
connection_class = get_connection_class(conn_class)
swift_conn = conn_class(
swift_conn = Connection(
authurl=authurl, user=user, key=key, snet=False)
try:
(resp_headers, resp_body) = swift_conn.get_object(
container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE)
except ClientException, e:
if e.http_status == httplib.NOT_FOUND:
location = format_swift_location(user, key, authurl,
container, obj)
raise exception.NotFound("Swift could not find image at "
"location %(location)s" % locals())
if expected_size:
obj_size = int(resp_headers['content-length'])
if obj_size != expected_size:
raise glance.store.BackendException(
@ -58,34 +77,130 @@ class SwiftBackend(glance.store.Backend):
return resp_body
@classmethod
def delete(cls, parsed_uri, conn_class=None):
def add(cls, id, data, options):
"""
Stores image data to Swift and returns a location that the image was
written to.
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:param id: The opaque image identifier
:param data: The image data to write, as a file-like object
:param options: Conf mapping
:retval Tuple with (location, size)
The location that was written,
and the size in bytes of the data written
"""
container = options.get('swift_store_container',
DEFAULT_SWIFT_CONTAINER)
auth_address = options.get('swift_store_auth_address')
user = options.get('swift_store_user')
key = options.get('swift_store_key')
# TODO(jaypipes): This needs to be checked every time
# because of the decision to make glance.store.Backend's
# interface all @classmethods. This is inefficient. Backend
# should be a stateful object with options parsed once in
# a constructor.
if not auth_address:
logger.error(msg)
msg = ("Could not find swift_store_auth_address in configuration "
"options.")
raise glance.store.BackendException(msg)
else:
full_auth_address = auth_address
if not full_auth_address.startswith('http'):
full_auth_address = 'https://' + full_auth_address
if not user:
logger.error(msg)
msg = ("Could not find swift_store_user in configuration "
"options.")
raise glance.store.BackendException(msg)
if not key:
logger.error(msg)
msg = ("Could not find swift_store_key in configuration "
"options.")
raise glance.store.BackendException(msg)
swift_conn = Connection(authurl=full_auth_address, user=user,
key=key, snet=False)
logger.debug("Adding image object to Swift using "
"(auth_address=%(auth_address)s, user=%(user)s, "
"key=%(key)s)" % locals())
create_container_if_missing(container, swift_conn, options)
obj_name = str(id)
location = format_swift_location(user, key, auth_address,
container, obj_name)
try:
obj_etag = swift_conn.put_object(container, obj_name, data)
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
# We do a HEAD on the newly-added image to determine the size
# of the image. A bit slow, but better than taking the word
# of the user adding the image with size attribute in the metadata
resp_headers = swift_conn.head_object(container, obj_name)
size = 0
# header keys are lowercased by Swift
if 'content-length' in resp_headers:
size = int(resp_headers['content-length'])
return (location, size)
except ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate("Swift already has an image at "
"location %(location)s" % locals())
msg = ("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s" % locals())
raise glance.store.BackendException(msg)
@classmethod
def delete(cls, parsed_uri):
"""
Deletes the swift object(s) at the parsed_uri location
"""
(user, key, authurl, container, obj) = \
cls._parse_swift_tokens(parsed_uri)
(user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri)
# TODO(sirp): snet=False for now, however, if the instance of
# swift we're talking to is within our same region, we should set
# snet=True
connection_class = get_connection_class(conn_class)
swift_conn = conn_class(
swift_conn = Connection(
authurl=authurl, user=user, key=key, snet=False)
(resp_headers, resp_body) = swift_conn.delete_object(
container=container, obj=obj)
try:
swift_conn.delete_object(container, obj)
except ClientException, e:
if e.http_status == httplib.NOT_FOUND:
location = format_swift_location(user, key, authurl,
container, obj)
raise exception.NotFound("Swift could not find image at "
"location %(location)s" % locals())
else:
raise
# TODO(jaypipes): What to return here? After reading the docs
# at swift.common.client, I'm not sure what to check for...
@classmethod
def _parse_swift_tokens(cls, parsed_uri):
def parse_swift_tokens(parsed_uri):
"""
Parsing the swift uri is three phases:
1) urlparse to split the tokens
2) use RE to split on @ and /
3) reassemble authurl
Return the various tokens used by Swift.
:param parsed_uri: The pieces of a URI returned by urlparse
:retval A tuple of (user, key, auth_address, container, obj_name)
"""
path = parsed_uri.path.lstrip('//')
netloc = parsed_uri.netloc
@ -107,15 +222,57 @@ class SwiftBackend(glance.store.Backend):
raise glance.store.BackendException(
"Expected four values to unpack in: swift:%s. "
"Should have received something like: %s."
% (parsed_uri.path, cls.EXAMPLE_URL))
% (parsed_uri.path, SwiftBackend.EXAMPLE_URL))
authurl = "https://%s" % '/'.join(path_parts)
return user, key, authurl, container, obj
def get_connection_class(conn_class):
if not conn_class:
import swift.common.client
conn_class = swift.common.client.Connection
return conn_class
def format_swift_location(user, key, auth_address, container, obj_name):
"""
Returns the swift URI in the format:
swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<OBJNAME>
:param user: The swift user to authenticate with
:param key: The auth key for the authenticating user
:param auth_address: The base URL for the authentication service
:param container: The name of the container
:param obj_name: The name of the object
"""
return "swift://%(user)s:%(key)s@%(auth_address)s/"\
"%(container)s/%(obj_name)s" % locals()
def create_container_if_missing(container, swift_conn, options):
"""
Creates a missing container in Swift if the
``swift_store_create_container_on_put`` option is set.
:param container: Name of container to create
:param swift_conn: Connection to Swift
:param options: Option mapping
"""
try:
swift_conn.head_container(container)
except ClientException, e:
if e.http_status == httplib.NOT_FOUND:
add_container = config.get_option(options,
'swift_store_create_container_on_put',
type='bool', default=False)
if add_container:
try:
swift_conn.put_container(container)
except ClientException, e:
msg = ("Failed to add container to Swift.\n"
"Got error from Swift: %(e)s" % locals())
raise glance.store.BackendException(msg)
else:
msg = ("The container %(container)s does not exist in "
"Swift. Please set the "
"swift_store_create_container_on_put option"
"to add container to Swift automatically."
% locals())
raise glance.store.BackendException(msg)
else:
raise

10
glance/utils.py

@ -32,10 +32,14 @@ def image_meta_to_http_headers(image_meta):
for k, v in image_meta.items():
if k == 'properties':
for pk, pv in v.items():
if pv is None:
pv = ''
headers["x-image-meta-property-%s"
% pk.lower()] = unicode(pv)
else:
headers["x-image-meta-%s" % k.lower()] = unicode(v)
if v is None:
v = ''
return headers
@ -77,10 +81,10 @@ def get_image_meta_from_headers(response):
key = str(key.lower())
if key.startswith('x-image-meta-property-'):
field_name = key[len('x-image-meta-property-'):].replace('-', '_')
properties[field_name] = value
properties[field_name] = value or None
elif key.startswith('x-image-meta-'):
field_name = key[len('x-image-meta-'):].replace('-', '_')
result[field_name] = value
result[field_name] = value or None
result['properties'] = properties
if 'id' in result:
result['id'] = int(result['id'])
@ -88,6 +92,8 @@ def get_image_meta_from_headers(response):
result['size'] = int(result['size'])
if 'is_public' in result:
result['is_public'] = (result['is_public'] == 'True')
if 'deleted' in result:
result['deleted'] = (result['deleted'] == 'True')
return result

76
tests/stubs.py

@ -33,7 +33,6 @@ from glance import server
import glance.store
import glance.store.filesystem
import glance.store.http
import glance.store.swift
import glance.registry.db.api
@ -110,7 +109,7 @@ def stub_out_filesystem_backend():
def stub_out_s3_backend(stubs):
""" Stubs out the S3 Backend with fake data and calls.
The stubbed swift backend provides back an iterator over
The stubbed s3 backend provides back an iterator over
the data ""
:param stubs: Set of stubout stubs
@ -140,78 +139,9 @@ def stub_out_s3_backend(stubs):
yield cls.DATA[i:i + cls.CHUNK_SIZE]
return chunk_it()
fake_swift_backend = FakeS3Backend()
fake_s3_backend = FakeS3Backend()
stubs.Set(glance.store.s3.S3Backend, 'get',
fake_swift_backend.get)
def stub_out_swift_backend(stubs):
"""Stubs out the Swift Glance backend with fake data
and calls.
The stubbed swift backend provides back an iterator over
the data "I am a teapot, short and stout\n"
:param stubs: Set of stubout stubs
"""
class FakeSwiftAuth(object):
pass
class FakeSwiftConnection(object):
pass
class FakeSwiftBackend(object):
CHUNK_SIZE = 2
DATA = 'I am a teapot, short and stout\n'
@classmethod
def get(cls, parsed_uri, expected_size, conn_class=None):
SwiftBackend = glance.store.swift.SwiftBackend
# raise BackendException if URI is bad.
(user, key, authurl, container, obj) = \
SwiftBackend._parse_swift_tokens(parsed_uri)
def chunk_it():
for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE):
yield cls.DATA[i:i + cls.CHUNK_SIZE]
return chunk_it()
fake_swift_backend = FakeSwiftBackend()
stubs.Set(glance.store.swift.SwiftBackend, 'get',
fake_swift_backend.get)
def stub_out_registry(stubs):
"""Stubs out the Registry registry with fake data returns.
The stubbed Registry always returns the following fixture::
{'files': [
{'location': 'file:///chunk0', 'size': 12345},
{'location': 'file:///chunk1', 'size': 1235}
]}
:param stubs: Set of stubout stubs
"""
class FakeRegistry(object):
DATA = \
{'files': [
{'location': 'file:///chunk0', 'size': 12345},
{'location': 'file:///chunk1', 'size': 1235}]}
@classmethod
def lookup(cls, _parsed_uri):
return cls.DATA
fake_registry_registry = FakeRegistry()
stubs.Set(glance.store.registries.Registry, 'lookup',
fake_registry_registry.lookup)
fake_s3_backend.get)
def stub_out_registry_and_store_server(stubs):

294
tests/unit/swiftfakehttp.py

@ -1,294 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
fakehttp/socket implementation
- TrackerSocket: an object which masquerades as a socket and responds to
requests in a manner consistent with a *very* stupid CloudFS tracker.
- CustomHTTPConnection: an object which subclasses httplib.HTTPConnection
in order to replace it's socket with a TrackerSocket instance.
The unittests each have setup methods which create freerange connection
instances that have had their HTTPConnection instances replaced by
intances of CustomHTTPConnection.
"""
from httplib import HTTPConnection as connbase
import StringIO
class FakeSocket(object):
def __init__(self):
self._rbuffer = StringIO.StringIO()
self._wbuffer = StringIO.StringIO()
def close(self):
pass
def send(self, data, flags=0):
self._rbuffer.write(data)
sendall = send
def recv(self, len=1024, flags=0):
return self._wbuffer(len)
def connect(self):
pass
def makefile(self, mode, flags):
return self._wbuffer
class TrackerSocket(FakeSocket):
def write(self, data):
self._wbuffer.write(data)
def read(self, length=-1):
return self._rbuffer.read(length)
def _create_GET_account_content(self, path, args):
if 'format' in args and args['format'] == 'json':
containers = []
containers.append('[\n')
containers.append('{"name":"container1","count":2,"bytes":78},\n')
containers.append('{"name":"container2","count":1,"bytes":39},\n')
containers.append('{"name":"container3","count":3,"bytes":117}\n')
containers.append(']\n')
elif 'format' in args and args['format'] == 'xml':
containers = []
containers.append('<?xml version="1.0" encoding="UTF-8"?>\n')
containers.append('<account name="FakeAccount">\n')
containers.append('<container><name>container1</name>'
'<count>2</count>'
'<bytes>78</bytes></container>\n')
containers.append('<container><name>container2</name>'
'<count>1</count>'
'<bytes>39</bytes></container>\n')
containers.append('<container><name>container3</name>'
'<count>3</count>'
'<bytes>117</bytes></container>\n')
containers.append('</account>\n')
else:
containers = ['container%s\n' % i for i in range(1, 4)]
return ''.join(containers)
def _create_GET_container_content(self, path, args):
left = 0
right = 9
if 'offset' in args:
left = int(args['offset'])
if 'limit' in args:
right = left + int(args['limit'])
if 'format' in args and args['format'] == 'json':
objects = []
objects.append('{"name":"object1",'
'"hash":"4281c348eaf83e70ddce0e07221c3d28",'
'"bytes":14,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object2",'
'"hash":"b039efe731ad111bc1b0ef221c3849d0",'
'"bytes":64,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object3",'
'"hash":"4281c348eaf83e70ddce0e07221c3d28",'
'"bytes":14,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object4",'
'"hash":"b039efe731ad111bc1b0ef221c3849d0",'
'"bytes":64,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object5",'
'"hash":"4281c348eaf83e70ddce0e07221c3d28",'
'"bytes":14,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object6",'
'"hash":"b039efe731ad111bc1b0ef221c3849d0",'
'"bytes":64,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object7",'
'"hash":"4281c348eaf83e70ddce0e07221c3d28",'
'"bytes":14,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
objects.append('{"name":"object8",'
'"hash":"b039efe731ad111bc1b0ef221c3849d0",'
'"bytes":64,'
'"content_type":"application\/octet-stream",'
'"last_modified":"2007-03-04 20:32:17"}')
output = '[\n%s\n]\n' % (',\n'.join(objects[left:right]))
elif 'format' in args and args['format'] == 'xml':
objects = []
objects.append('<object><name>object1</name>'
'<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
'<bytes>14</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object2</name>'
'<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
'<bytes>64</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object3</name>'
'<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
'<bytes>14</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object4</name>'
'<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
'<bytes>64</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object5</name>'
'<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
'<bytes>14</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object6</name>'
'<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
'<bytes>64</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object7</name>'
'<hash>4281c348eaf83e70ddce0e07221c3d28</hash>'
'<bytes>14</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects.append('<object><name>object8</name>'
'<hash>b039efe731ad111bc1b0ef221c3849d0</hash>'
'<bytes>64</bytes>'
'<content_type>application/octet-stream</content_type>'
'<last_modified>2007-03-04 20:32:17</last_modified>'
'</object>\n')
objects = objects[left:right]
objects.insert(0, '<?xml version="1.0" encoding="UTF-8"?>\n')
objects.insert(1, '<container name="test_container_1"\n')
objects.append('</container>\n')
output = ''.join(objects)
else:
objects = ['object%s\n' % i for i in range(1, 9)]
objects = objects[left:right]
output = ''.join(objects)
# prefix/path don't make much sense given our test data
if 'prefix' in args or 'path' in args:
pass
return output
def render_GET(self, path, args):
# Special path that returns 404 Not Found
if (len(path) == 4) and (path[3] == 'bogus'):
self.write('HTTP/1.1 404 Not Found\n')
self.write('Content-Type: text/plain\n')
self.write('Content-Length: 0\n')
self.write('Connection: close\n\n')
return
self.write('HTTP/1.1 200 Ok\n')
self.write('Content-Type: text/plain\n')
if len(path) == 2:
content = self._create_GET_account_content(path, args)
elif len(path) == 3:
content = self._create_GET_container_content(path, args)
# Object
elif len(path) == 4:
content = 'I am a teapot, short and stout\n'
self.write('Content-Length: %d\n' % len(content))
self.write('Connection: close\n\n')
self.write(content)
def render_HEAD(self, path, args):
# Account
if len(path) == 2:
self.write('HTTP/1.1 204 No Content\n')
self.write('Content-Type: text/plain\n')
self.write('Connection: close\n')
self.write('X-Account-Container-Count: 3\n')
self.write('X-Account-Bytes-Used: 234\n\n')
else:
self.write('HTTP/1.1 200 Ok\n')
self.write('Content-Type: text/plain\n')
self.write('ETag: d5c7f3babf6c602a8da902fb301a9f27\n')
self.write('Content-Length: 21\n')
self.write('Connection: close\n\n')
def render_POST(self, path, args):
self.write('HTTP/1.1 202 Ok\n')
self.write('Connection: close\n\n')
def render_PUT(self, path, args):
self.write('HTTP/1.1 200 Ok\n')
self.write('Content-Type: text/plain\n')
self.write('Connection: close\n\n')
render_DELETE = render_PUT
def render(self, method, uri):
if '?' in uri:
parts = uri.split('?')
query = parts[1].strip('&').split('&')
args = dict([tuple(i.split('=', 1)) for i in query])
path = parts[0].strip('/').split('/')
else:
args = {}
path = uri.strip('/').split('/')
if hasattr(self, 'render_%s' % method):
getattr(self, 'render_%s' % method)(path, args)
else:
self.write('HTTP/1.1 406 Not Acceptable\n')
self.write('Content-Type: text/plain\n')
self.write('Connection: close\n')
def makefile(self, mode, flags):
self._rbuffer.seek(0)
lines = self.read().splitlines()
(method, uri, version) = lines[0].split()
self.render(method, uri)
self._wbuffer.seek(0)
return self._wbuffer
class CustomHTTPConnection(connbase):
def connect(self):
self.sock = TrackerSocket()
if __name__ == '__main__':
conn = CustomHTTPConnection('localhost', 8000)
conn.request('HEAD', '/v1/account/container/object')
response = conn.getresponse()
print "Status:", response.status, response.reason
for (key, value) in response.getheaders():
print "%s: %s" % (key, value)
print response.read()

135
tests/unit/test_filesystem_store.py Normal file

@ -0,0 +1,135 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests the filesystem backend store"""
import StringIO
import unittest
import urlparse
import stubout
from glance.common import exception
from glance.store.filesystem import FilesystemBackend, ChunkedFile
from tests import stubs
class TestFilesystemBackend(unittest.TestCase):
def setUp(self):
"""Establish a clean test environment"""
self.stubs = stubout.StubOutForTesting()
stubs.stub_out_filesystem_backend()
self.orig_chunksize = ChunkedFile.CHUNKSIZE
ChunkedFile.CHUNKSIZE = 10
def tearDown(self):
"""Clear the test environment"""
stubs.clean_out_fake_filesystem_backend()
self.stubs.UnsetAll()
ChunkedFile.CHUNKSIZE = self.orig_chunksize
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2")
image_file = FilesystemBackend.get(url_pieces)
expected_data = "chunk00000remainder"
expected_num_chunks = 2
data = ""
num_chunks = 0
for chunk in image_file:
num_chunks += 1
data += chunk
self.assertEqual(expected_data, data)
self.assertEqual(expected_num_chunks, num_chunks)
def test_get_non_existing(self):
"""
Test that trying to retrieve a file that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing")
self.assertRaises(exception.NotFound,
FilesystemBackend.get,
url_pieces)
def test_add(self):
"""Test that we can add an image via the filesystem backend"""
ChunkedFile.CHUNKSIZE = 1024
expected_image_id = 42
expected_file_size = 1024 * 5 # 5K
expected_file_contents = "*" * expected_file_size
expected_location = "file://%s/%s" % (stubs.FAKE_FILESYSTEM_ROOTDIR,
expected_image_id)
image_file = StringIO.StringIO(expected_file_contents)
options = {'verbose': True,
'debug': True,
'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR}
location, size = FilesystemBackend.add(42, image_file, options)
self.assertEquals(expected_location, location)
self.assertEquals(expected_file_size, size)
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/42")
new_image_file = FilesystemBackend.get(url_pieces)
new_image_contents = ""
new_image_file_size = 0
for chunk in new_image_file:
new_image_file_size += len(chunk)
new_image_contents += chunk
self.assertEquals(expected_file_contents, new_image_contents)
self.assertEquals(expected_file_size, new_image_file_size)
def test_add_already_existing(self):
"""
Tests that adding an image with an existing identifier
raises an appropriate exception
"""
image_file = StringIO.StringIO("nevergonnamakeit")
options = {'verbose': True,
'debug': True,
'filesystem_store_datadir': stubs.FAKE_FILESYSTEM_ROOTDIR}
self.assertRaises(exception.Duplicate,
FilesystemBackend.add,
2, image_file, options)
def test_delete(self):
"""
Test we can delete an existing image in the filesystem store
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/2")
FilesystemBackend.delete(url_pieces)
self.assertRaises(exception.NotFound,
FilesystemBackend.get,
url_pieces)
def test_delete_non_existing(self):
"""
Test that trying to delete a file that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse("file:///tmp/glance-tests/non-existing")
self.assertRaises(exception.NotFound,
FilesystemBackend.delete,
url_pieces)

2
tests/unit/test_misc.py

@ -93,7 +93,9 @@ class TestMiscellaneous(unittest.TestCase):
"""
fixture = {'name': 'fake public image',
'is_public': True,
'deleted': False,
'type': 'kernel',
'name': None,
'size': 19,
'location': "file:///tmp/glance-tests/2",
'properties': {'distro': 'Ubuntu 10.04 LTS'}}

64
tests/unit/test_stores.py

@ -22,7 +22,6 @@ import unittest
import urlparse
from glance.store.s3 import S3Backend
from glance.store.swift import SwiftBackend
from glance.store import Backend, BackendException, get_from_backend
from tests import stubs
@ -40,27 +39,6 @@ class TestBackend(unittest.TestCase):
self.stubs.UnsetAll()
class TestFilesystemBackend(TestBackend):
def setUp(self):
"""Establish a clean test environment"""
stubs.stub_out_filesystem_backend()
def tearDown(self):
"""Clear the test environment"""
stubs.clean_out_fake_filesystem_backend()
def test_get(self):
fetcher = get_from_backend("file:///tmp/glance-tests/2",
expected_size=19)
data = ""
for chunk in fetcher:
data += chunk
self.assertEqual(data, "chunk00000remainder")
class TestHTTPBackend(TestBackend):
def setUp(self):
@ -104,45 +82,3 @@ class TestS3Backend(TestBackend):
chunks = [c for c in fetcher]
self.assertEqual(chunks, expected_returns)
class TestSwiftBackend(TestBackend):
def setUp(self):
super(TestSwiftBackend, self).setUp()
stubs.stub_out_swift_backend(self.stubs)
def test_get(self):
swift_uri = "swift://user:pass@localhost/container1/file.tar.gz"
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
fetcher = get_from_backend(swift_uri,
expected_size=21,
conn_class=SwiftBackend)
chunks = [c for c in fetcher]
self.assertEqual(chunks, expected_returns)
def test_get_bad_uri(self):
swift_url = "swift://localhost/container1/file.tar.gz"
self.assertRaises(BackendException, get_from_backend,
swift_url, expected_size=21)
def test_url_parsing(self):
swift_uri = "swift://user:pass@localhost/v1.0/container1/file.tar.gz"
parsed_uri = urlparse.urlparse(swift_uri)
(user, key, authurl, container, obj) = \
SwiftBackend._parse_swift_tokens(parsed_uri)
self.assertEqual(user, 'user')
self.assertEqual(key, 'pass')
self.assertEqual(authurl, 'https://localhost/v1.0')
self.assertEqual(container, 'container1')
self.assertEqual(obj, 'file.tar.gz')

300
tests/unit/test_swift_store.py Normal file

@ -0,0 +1,300 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this swift except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests the Swift backend store"""
import StringIO
import hashlib
import httplib
import sys
import unittest
import urlparse
import stubout
import swift.common.client
from glance.common import exception
from glance.store import BackendException
from glance.store.swift import SwiftBackend, format_swift_location
FIVE_KB = (5 * 1024)
SWIFT_OPTIONS = {'verbose': True,
'debug': True,
'swift_store_user': 'user',
'swift_store_key': 'key',
'swift_store_auth_address': 'localhost:8080',
'swift_store_container': 'glance'}
# We stub out as little as possible to ensure that the code paths
# between glance.store.swift and swift.common.client are tested
# thoroughly
def stub_out_swift_common_client(stubs):
fixture_containers = ['glance']
fixture_headers = {'glance/2':
{'content-length': FIVE_KB,
'etag': 'c2e5db72bd7fd153f53ede5da5a06de3'}}
fixture_objects = {'glance/2':
StringIO.StringIO("*" * FIVE_KB)}
def fake_head_container(url, token, container, **kwargs):
if container not in fixture_containers:
msg = "No container %s found" % container
raise swift.common.client.ClientException(msg,
http_status=httplib.NOT_FOUND)
def fake_put_container(url, token, container, **kwargs):
fixture_containers.append(container)
def fake_put_object(url, token, container, name, contents, **kwargs):
# PUT returns the ETag header for the newly-added object
fixture_key = "%s/%s" % (container, name)
if not fixture_key in fixture_headers.keys():
if hasattr(contents, 'read'):
fixture_object = StringIO.StringIO()
chunk = contents.read(SwiftBackend.CHUNKSIZE)
while chunk:
fixture_object.write(chunk)
chunk = contents.read(SwiftBackend.CHUNKSIZE)
else:
fixture_object = StringIO.StringIO(contents)
fixture_objects[fixture_key] = fixture_object
fixture_headers[fixture_key] = {
'content-length': fixture_object.len,
'etag': hashlib.md5(fixture_object.read()).hexdigest()}
return fixture_headers[fixture_key]['etag']
else:
msg = ("Object PUT failed - Object with key %s already exists"
% fixture_key)
raise swift.common.client.ClientException(msg,
http_status=httplib.CONFLICT)
def fake_get_object(url, token, container, name, **kwargs):
# GET returns the tuple (list of headers, file object)
try:
fixture_key = "%s/%s" % (container, name)
return fixture_headers[fixture_key], fixture_objects[fixture_key]
except KeyError:
msg = "Object GET failed"
raise swift.common.client.ClientException(msg,
http_status=httplib.NOT_FOUND)
def fake_head_object(url, token, container, name, **kwargs):
# HEAD returns the list of headers for an object
try:
fixture_key = "%s/%s" % (container, name)
return fixture_headers[fixture_key]
except KeyError:
msg = "Object HEAD failed - Object does not exist"
raise swift.common.client.ClientException(msg,
http_status=httplib.NOT_FOUND)
def fake_delete_object(url, token, container, name, **kwargs):
# DELETE returns nothing
fixture_key = "%s/%s" % (container, name)
if fixture_key not in fixture_headers.keys():
msg = "Object DELETE failed - Object does not exist"
raise swift.common.client.ClientException(msg,
http_status=httplib.NOT_FOUND)
else:
del fixture_headers[fixture_key]
del fixture_objects[fixture_key]
def fake_http_connection(*args, **kwargs):
return None
def fake_get_auth(*args, **kwargs):
return None, None
stubs.Set(swift.common.client,
'head_container', fake_head_container)
stubs.Set(swift.common.client,
'put_container', fake_put_container)
stubs.Set(swift.common.client,
'put_object', fake_put_object)
stubs.Set(swift.common.client,
'delete_object', fake_delete_object)
stubs.Set(swift.common.client,
'head_object', fake_head_object)
stubs.Set(swift.common.client,
'get_object', fake_get_object)
stubs.Set(swift.common.client,
'get_auth', fake_get_auth)
stubs.Set(swift.common.client,
'http_connection', fake_http_connection)
class TestSwiftBackend(unittest.TestCase):
def setUp(self):
"""Establish a clean test environment"""
self.stubs = stubout.StubOutForTesting()
stub_out_swift_common_client(self.stubs)
def tearDown(self):
"""Clear the test environment"""
self.stubs.UnsetAll()
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
image_swift = SwiftBackend.get(url_pieces)
expected_data = "*" * FIVE_KB
data = ""
for chunk in image_swift:
data += chunk
self.assertEqual(expected_data, data)
def test_get_mismatched_expected_size(self):
"""
Test retrieval of an image with wrong expected_size param
raises an exception
"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
self.assertRaises(BackendException,
SwiftBackend.get,
url_pieces,
{'expected_size': 42})
def test_get_non_existing(self):
"""
Test that trying to retrieve a swift that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/noexist")
self.assertRaises(exception.NotFound,
SwiftBackend.get,
url_pieces)
def test_add(self):
"""Test that we can add an image via the swift backend"""
expected_image_id = 42
expected_swift_size = 1024 * 5 # 5K
expected_swift_contents = "*" * expected_swift_size
expected_location = format_swift_location(
SWIFT_OPTIONS['swift_store_user'],
SWIFT_OPTIONS['swift_store_key'],
SWIFT_OPTIONS['swift_store_auth_address'],
SWIFT_OPTIONS['swift_store_container'],
expected_image_id)
image_swift = StringIO.StringIO(expected_swift_contents)
location, size = SwiftBackend.add(42, image_swift, SWIFT_OPTIONS)
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
url_pieces = urlparse.urlparse(expected_location)
new_image_swift = SwiftBackend.get(url_pieces)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
self.assertEquals(expected_swift_contents, new_image_contents)
self.assertEquals(expected_swift_size, new_image_swift_size)
def test_add_no_container_no_create(self):
"""
Tests that adding an image with a non-existing container
raises an appropriate exception
"""
options = SWIFT_OPTIONS.copy()
options['swift_store_create_container_on_put'] = 'False'
options['swift_store_container'] = 'noexist'
image_swift = StringIO.StringIO("nevergonnamakeit")
# We check the exception text to ensure the container
# missing text is found in it, otherwise, we would have
# simply used self.assertRaises here
exception_caught = False
try:
SwiftBackend.add(3, image_swift, options)
except BackendException, e:
exception_caught = True
self.assertTrue("container noexist does not exist "
"in Swift" in str(e))
self.assertTrue(exception_caught)
def test_add_no_container_and_create(self):
"""
Tests that adding an image with a non-existing container
creates the container automatically if flag is set
"""
options = SWIFT_OPTIONS.copy()
options['swift_store_create_container_on_put'] = 'True'
options['swift_store_container'] = 'noexist'
expected_image_id = 42
expected_swift_size = 1024 * 5 # 5K
expected_swift_contents = "*" * expected_swift_size
expected_location = format_swift_location(
options['swift_store_user'],
options['swift_store_key'],
options['swift_store_auth_address'],
options['swift_store_container'],
expected_image_id)
image_swift = StringIO.StringIO(expected_swift_contents)
location, size = SwiftBackend.add(42, image_swift, options)
self.assertEquals(expected_location, location)
self.assertEquals(expected_swift_size, size)
url_pieces = urlparse.urlparse(expected_location)
new_image_swift = SwiftBackend.get(url_pieces)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
self.assertEquals(expected_swift_contents, new_image_contents)
self.assertEquals(expected_swift_size, new_image_swift_size)
def test_add_already_existing(self):
"""
Tests that adding an image with an existing identifier
raises an appropriate exception
"""
image_swift = StringIO.StringIO("nevergonnamakeit")
self.assertRaises(exception.Duplicate,
SwiftBackend.add,
2, image_swift, SWIFT_OPTIONS)
def test_delete(self):
"""
Test we can delete an existing image in the swift store
"""
url_pieces = urlparse.urlparse(
"swift://user:key@auth_address/glance/2")
SwiftBackend.delete(url_pieces)
self.assertRaises(exception.NotFound,
SwiftBackend.get,
url_pieces)
def test_delete_non_existing(self):
"""
Test that trying to delete a swift that doesn't exist
raises an error
"""
url_pieces = urlparse.urlparse("swift://user:key@auth_address/noexist")
self.assertRaises(exception.NotFound,
SwiftBackend.delete,
url_pieces)

1
tools/pip-requires

@ -12,5 +12,6 @@ nose
sphinx
argparse
mox==0.5.0
swift
-f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz
sqlalchemy-migrate>=0.6