openstack
/
glance
Code Issues Proposed changes

Fixing bug 794582 - Now able to stream http(s) images 

Change-Id: Ic04dfe32acc21ff068ef0964541c47eee41fbe3b
This commit is contained in:
Brian Waldon 2011-09-15 17:30:08 -04:00
parent 6c60376f9d
commit ef19685880
10 changed files with 56 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
glance/api/v1/images.py
View File

@ -200,13 +200,15 @@ class Controller(api.BaseController):
"""
image = self.get_active_image_meta_or_404(req, id)
def get_from_store(image):
def get_from_store(image_meta):
"""Called if caching disabled"""
try:
image = get_from_backend(image['location'])
location = image_meta['location']
image_data, image_size = get_from_backend(location)
image_meta["size"] = image_size or image_meta["size"]
except exception.NotFound, e:
raise HTTPNotFound(explanation="%s" % e)
return image
return image_data
def get_from_cache(image, cache):
"""Called if cache hit"""

4
glance/store/base.py
View File

@ -55,8 +55,8 @@ class Store(object):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()

6
glance/store/filesystem.py
View File

@ -126,8 +126,8 @@ class Store(glance.store.base.Store):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
@ -140,7 +140,7 @@ class Store(glance.store.base.Store):
else:
msg = _("Found image at %s. Returning in ChunkedFile.") % filepath
logger.debug(msg)
return ChunkedFile(filepath)
return (ChunkedFile(filepath), None)
def delete(self, location):
"""

15
glance/store/http.py
View File

@ -85,14 +85,14 @@ class StoreLocation(glance.store.location.StoreLocation):
self.path = path
def http_response_iterator(conn, size):
def http_response_iterator(conn, response, size):
"""
Return an iterator for a file-like object.
:param conn: HTTP(S) Connection
:param response: httplib.HTTPResponse object
:param size: Chunk size to iterate with
"""
response = conn.getresponse()
chunk = response.read(size)
while chunk:
yield chunk
@ -107,20 +107,23 @@ class Store(glance.store.base.Store):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises `glance.exception.NotFound` if image does not exist
"""
loc = location.store_location
conn_class = self._get_conn_class(loc)
conn = conn_class(loc.netloc)
conn.request("GET", loc.path, "", {})
resp = conn.getresponse()
return http_response_iterator(conn, self.CHUNKSIZE)
content_length = resp.getheader('content-length', 0)
iterator = http_response_iterator(conn, resp, self.CHUNKSIZE)
return (iterator, content_length)
def _get_conn_class(self, loc):
"""

6
glance/store/s3.py
View File

@ -223,8 +223,8 @@ class Store(glance.store.base.Store):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
@ -253,7 +253,7 @@ class Store(glance.store.base.Store):
# raise glance.store.BackendException(msg)
key.BufferSize = self.CHUNKSIZE
return ChunkedFile(key)
return (ChunkedFile(key), None)
def add(self, image_id, image_file, image_size):
"""

6
glance/store/swift.py
View File

@ -230,8 +230,8 @@ class Store(glance.store.base.Store):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
@ -260,7 +260,7 @@ class Store(glance.store.base.Store):
# "Expected %s byte file, Swift has %s bytes" %
# (expected_size, obj_size))
return resp_body
return (resp_body, None)
def _make_swift_connection(self, auth_url, user, key):
"""

4
glance/tests/unit/test_filesystem_store.py
View File

@ -53,7 +53,7 @@ class TestStore(unittest.TestCase):
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
loc = get_location_from_uri("file:///tmp/glance-tests/2")
image_file = self.store.get(loc)
(image_file, image_size) = self.store.get(loc)
expected_data = "chunk00000remainder"
expected_num_chunks = 2
@ -95,7 +95,7 @@ class TestStore(unittest.TestCase):
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri("file:///tmp/glance-tests/42")
new_image_file = self.store.get(loc)
(new_image_file, new_image_size) = self.store.get(loc)
new_image_contents = ""
new_image_file_size = 0

20
glance/tests/unit/test_http_store.py
View File

@ -37,15 +37,25 @@ def stub_out_http_backend(stubs):
:param stubs: Set of stubout stubs
"""
class FakeHTTPConnection(object):
class FakeHTTPResponse(object):
DATA = 'I am a teapot, short and stout\n'
HEADERS = {'content-length': 31}
def __init__(self, *args, **kwargs):
self.data = StringIO.StringIO(self.DATA)
self.read = self.data.read
def getheader(self, name, default=None):
return self.HEADERS.get(name.lower(), default)
class FakeHTTPConnection(object):
def __init__(self, *args, **kwargs):
pass
def getresponse(self):
return StringIO.StringIO(self.DATA)
return FakeHTTPResponse()
def request(self, *_args, **_kwargs):
pass
@ -72,7 +82,8 @@ class TestHttpStore(unittest.TestCase):
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
loc = get_location_from_uri(uri)
image_file = self.store.get(loc)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)
chunks = [c for c in image_file]
self.assertEqual(chunks, expected_returns)
@ -82,7 +93,8 @@ class TestHttpStore(unittest.TestCase):
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
loc = get_location_from_uri(uri)
image_file = self.store.get(loc)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)
chunks = [c for c in image_file]
self.assertEqual(chunks, expected_returns)

8
glance/tests/unit/test_s3_store.py
View File

@ -169,7 +169,9 @@ class TestStore(unittest.TestCase):
"""Test a "normal" retrieval of an image in chunks"""
loc = get_location_from_uri(
"s3://user:key@auth_address/glance/2")
image_s3 = self.store.get(loc)
(image_s3, image_size) = self.store.get(loc)
self.assertEqual(image_size, None)
expected_data = "*" * FIVE_KB
data = ""
@ -217,7 +219,7 @@ class TestStore(unittest.TestCase):
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_s3 = self.store.get(loc)
(new_image_s3, new_image_size) = self.store.get(loc)
new_image_contents = StringIO.StringIO()
for chunk in new_image_s3:
new_image_contents.write(chunk)
@ -267,7 +269,7 @@ class TestStore(unittest.TestCase):
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_s3 = self.store.get(loc)
(new_image_s3, new_image_size) = self.store.get(loc)
new_image_contents = new_image_s3.getvalue()
new_image_s3_size = new_image_s3.len

14
glance/tests/unit/test_swift_store.py
View File

@ -197,7 +197,8 @@ class TestStore(unittest.TestCase):
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
loc = get_location_from_uri("swift://user:key@auth_address/glance/2")
image_swift = self.store.get(loc)
(image_swift, image_size) = self.store.get(loc)
self.assertEqual(image_size, None)
expected_data = "*" * FIVE_KB
data = ""
@ -214,7 +215,8 @@ class TestStore(unittest.TestCase):
"""
loc = get_location_from_uri("swift+http://user:key@auth_address/"
"glance/2")
image_swift = self.store.get(loc)
(image_swift, image_size) = self.store.get(loc)
self.assertEqual(image_size, None)
expected_data = "*" * FIVE_KB
data = ""
@ -255,7 +257,7 @@ class TestStore(unittest.TestCase):
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
@ -303,7 +305,7 @@ class TestStore(unittest.TestCase):
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
@ -363,7 +365,7 @@ class TestStore(unittest.TestCase):
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len
@ -408,7 +410,7 @@ class TestStore(unittest.TestCase):
self.assertEquals(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len