openstack/glance
Code Issues Proposed changes

Merge latest trunk

Browse Source
This commit is contained in:
jaypipes@gmail.com 2011-07-28 19:01:40 -04:00
commit 38cc809bb8
8 changed files with 104 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/glance-api.conf
View File

@ -76,6 +76,8 @@ s3_store_create_bucket_on_put = False
# ============ Image Cache Options ========================
image_cache_enabled = False
# Directory that the Image Cache writes data to
# Make sure this is also set in glance-pruner.conf
image_cache_datadir = /var/lib/glance/image-cache/

24
glance/api/v1/images.py
View File

@ -214,26 +214,30 @@ class Controller(api.BaseController):
if cache.enabled:
if cache.hit(id):
# hit
logger.debug("image cache HIT, retrieving image '%s'"
" from cache", id)
logger.debug("image '%s' is a cache HIT", id)
image_iterator = get_from_cache(image, cache)
else:
# miss
logger.debug("image cache MISS, retrieving image '%s'"
" from store and tee'ing into cache", id)
logger.debug("image '%s' is a cache MISS", id)
# We only want to tee-into the cache if we're not currently
# prefetching an image
image_id = image['id']
if cache.is_image_currently_prefetching(image_id):
# Make sure we're not already prefetching or caching the image
# that just generated the miss
if cache.is_image_currently_prefetching(id):
logger.debug("image '%s' is already being prefetched,"
" not tee'ing into the cache", id)
image_iterator = get_from_store(image)
elif cache.is_image_currently_being_written(id):
logger.debug("image '%s' is already being cached,"
" not tee'ing into the cache", id)
image_iterator = get_from_store(image)
else:
# NOTE(sirp): If we're about to download and cache an
# image which is currently in the prefetch queue, just
# delete the queue items since we're caching it anyway
if cache.is_image_queued_for_prefetch(image_id):
cache.delete_queued_prefetch_image(image_id)
if cache.is_image_queued_for_prefetch(id):
cache.delete_queued_prefetch_image(id)
logger.debug("tee'ing image '%s' into cache", id)
image_iterator = get_from_store_tee_into_cache(
image, cache)
else:

17
glance/store/__init__.py
View File

@ -59,17 +59,16 @@ def get_backend_class(backend):
:param backend: Name of backend to create
"""
# NOTE(sirp): avoiding circular import
from glance.store.http import HTTPBackend
from glance.store.s3 import S3Backend
from glance.store.swift import SwiftBackend
from glance.store.filesystem import FilesystemBackend
import glance.store.http
import glance.store.s3
import glance.store.swift
import glance.store.filesystem
BACKENDS = {
"file": FilesystemBackend,
"http": HTTPBackend,
"https": HTTPBackend,
"swift": SwiftBackend,
"s3": S3Backend}
"filesystem": glance.store.filesystem.FilesystemBackend,
"http": glance.store.http.HTTPBackend,
"swift": glance.store.swift.SwiftBackend,
"s3": glance.store.s3.S3Backend}
try:
return BACKENDS[backend]

5
glance/store/filesystem.py
View File

@ -30,7 +30,8 @@ import glance.store.location
logger = logging.getLogger('glance.store.filesystem')
glance.store.location.add_scheme_map({'file': 'filesystem'})
glance.store.location.add_scheme_map({'file': 'filesystem',
'filesystem': 'filesystem'})
class StoreLocation(glance.store.location.StoreLocation):
@ -51,7 +52,7 @@ class StoreLocation(glance.store.location.StoreLocation):
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme == 'file'
assert pieces.scheme in ('file', 'filesystem')
self.scheme = pieces.scheme
path = (pieces.netloc + pieces.path).strip()
if path == '':

12
glance/store/s3.py
View File

@ -84,6 +84,18 @@ class StoreLocation(glance.store.location.StoreLocation):
which is entirely retarded, and breaks urlparse miserably.
This function works around that issue.
"""
# Make sure that URIs that contain multiple schemes, such as:
# swift://user:pass@http://authurl.com/v1/container/obj
# are immediately rejected.
if uri.count('://') != 1:
reason = ("URI Cannot contain more than one occurrence of a "
"scheme. If you have specified a "
"URI like s3://user:pass@https://s3.amazonaws.com/"
"bucket/key, you need to change it to use the "
"s3+https:// scheme, like so: "
"s3+https://user:pass@s3.amazonaws.com/bucket/key")
raise exception.BadStoreUri(uri, reason)
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('s3', 's3+http', 's3+https')
self.scheme = pieces.scheme

34
glance/store/swift.py
View File

@ -44,10 +44,13 @@ class StoreLocation(glance.store.location.StoreLocation):
the following:
swift://user:pass@authurl.com/container/obj-id
swift://account:user:pass@authurl.com/container/obj-id
swift+http://user:pass@authurl.com/container/obj-id
swift+https://user:pass@authurl.com/container/obj-id
The swift+https:// URIs indicate there is an HTTPS authentication URL
The swift+http:// URIs indicate there is an HTTP authentication URL.
The default for Swift is an HTTPS authentication URL, so swift:// and
swift+https:// are the same...
"""
def process_specs(self):
@ -85,6 +88,18 @@ class StoreLocation(glance.store.location.StoreLocation):
swift://account:user:pass@authurl.com/container/obj
"""
# Make sure that URIs that contain multiple schemes, such as:
# swift://user:pass@http://authurl.com/v1/container/obj
# are immediately rejected.
if uri.count('://') != 1:
reason = ("URI Cannot contain more than one occurrence of a "
"scheme. If you have specified a "
"URI like swift://user:pass@http://authurl.com/v1/"
"container/obj, you need to change it to use the "
"swift+http:// scheme, like so: "
"swift+http://user:pass@authurl.com/v1/container/obj")
raise exception.BadStoreUri(uri, reason)
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https')
self.scheme = pieces.scheme
@ -127,9 +142,10 @@ class StoreLocation(glance.store.location.StoreLocation):
try:
self.obj = path_parts.pop()
self.container = path_parts.pop()
self.authurl = netloc
if len(path_parts) > 0:
self.authurl = netloc + '/' + '/'.join(path_parts).strip('/')
if not netloc.startswith('http'):
# push hostname back into the remaining to build full authurl
path_parts.insert(0, netloc)
self.authurl = '/'.join(path_parts)
except IndexError:
reason = "Badly formed Swift URI"
raise exception.BadStoreUri(uri, reason)
@ -143,16 +159,10 @@ class StoreLocation(glance.store.location.StoreLocation):
HTTPS is assumed, unless 'swift+http' is specified.
"""
if self.scheme.startswith('swift+http'):
auth_scheme = 'http://'
elif self.scheme.startswith('swift+https'):
auth_scheme = 'https://'
elif self.scheme.startswith('swift'):
if self.scheme in ('swift+https', 'swift'):
auth_scheme = 'https://'
else:
logger.warn("Unrecognized scheme '%s', defaulting auth url"
" to https", self.scheme)
auth_scheme = 'https://'
auth_scheme = 'http://'
full_url = ''.join([auth_scheme, self.authurl])
return full_url

43
tests/unit/test_store_location.py
View File

@ -138,6 +138,7 @@ class TestStoreLocation(unittest.TestCase):
self.assertEqual("swift", loc.scheme)
self.assertEqual("example.com", loc.authurl)
self.assertEqual("https://example.com", loc.swift_auth_url)
self.assertEqual("images", loc.container)
self.assertEqual("1", loc.obj)
self.assertEqual(None, loc.user)
@ -148,6 +149,7 @@ class TestStoreLocation(unittest.TestCase):
self.assertEqual("swift+https", loc.scheme)
self.assertEqual("authurl.com", loc.authurl)
self.assertEqual("https://authurl.com", loc.swift_auth_url)
self.assertEqual("images", loc.container)
self.assertEqual("1", loc.obj)
self.assertEqual("user", loc.user)
@ -159,17 +161,19 @@ class TestStoreLocation(unittest.TestCase):
self.assertEqual("swift+https", loc.scheme)
self.assertEqual("authurl.com/v1", loc.authurl)
self.assertEqual("https://authurl.com/v1", loc.swift_auth_url)
self.assertEqual("container", loc.container)
self.assertEqual("12345", loc.obj)
self.assertEqual("user", loc.user)
self.assertEqual("pass", loc.key)
self.assertEqual(uri, loc.get_uri())
uri = 'swift://account:user:pass@authurl.com/v1/container/12345'
uri = 'swift+http://account:user:pass@authurl.com/v1/container/12345'
loc.parse_uri(uri)
self.assertEqual("swift", loc.scheme)
self.assertEqual("swift+http", loc.scheme)
self.assertEqual("authurl.com/v1", loc.authurl)
self.assertEqual("http://authurl.com/v1", loc.swift_auth_url)
self.assertEqual("container", loc.container)
self.assertEqual("12345", loc.obj)
self.assertEqual("account:user", loc.user)
@ -185,6 +189,9 @@ class TestStoreLocation(unittest.TestCase):
bad_uri = 'swift://user@example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
bad_uri = 'swift://user:pass@http://example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
def test_s3_store_location(self):
"""
Test the specific StoreLocation for the S3 store
@ -233,7 +240,7 @@ class TestStoreLocation(unittest.TestCase):
self.assertEqual("pass/withslash", loc.secretkey)
self.assertEqual(uri, loc.get_uri())
bad_uri = 'swif://'
bad_uri = 's://'
self.assertRaises(Exception, loc.parse_uri, bad_uri)
bad_uri = 's3://'
@ -241,3 +248,33 @@ class TestStoreLocation(unittest.TestCase):
bad_uri = 's3://accesskey@example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
bad_uri = 's3://user:pass@http://example.com:8080/images/1'
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
def test_get_backend_class(self):
"""
Test that the backend returned by glance.store.get_backend_class
is correct or raises an appropriate error.
"""
good_results = {
'swift': glance.store.swift.SwiftBackend,
'swift+http': glance.store.swift.SwiftBackend,
'swift+https': glance.store.swift.SwiftBackend,
's3': glance.store.s3.S3Backend,
's3+http': glance.store.s3.S3Backend,
's3+https': glance.store.s3.S3Backend,
'file': glance.store.filesystem.FilesystemBackend,
'filesystem': glance.store.filesystem.FilesystemBackend,
'http': glance.store.http.HTTPBackend,
'https': glance.store.http.HTTPBackend}
for scheme, store in good_results.items():
self.assertEqual(glance.store.get_backend_class(scheme), store)
bad_results = ['fil', 'swift+h', 'unknown']
for store in bad_results:
self.assertRaises(glance.store.UnsupportedBackend,
glance.store.get_backend_class,
store)

4
tests/unit/test_swift_store.py
View File

@ -124,7 +124,9 @@ def stub_out_swift_common_client(stubs):
def fake_http_connection(*args, **kwargs):
return None
def fake_get_auth(*args, **kwargs):
def fake_get_auth(url, *args, **kwargs):
if 'http' in url and '://' not in url:
raise ValueError('Invalid url %s' % url)
return None, None
stubs.Set(swift.common.client,