openstack/glance
Code Issues Proposed changes

Provide stores access to the request context.

Browse Source 
Adds context to the constructor for the Store class. This can be used
to provide Store's access to the RequestContext.

Includes the following changes:

* Updates common store helper methods to create new instance of
  Store objects for each request.

* Updates Glance scrubber to create a real context from credentials.

The motivation for this change is that we will need access to the
service catalog and token for the Swift multi-tenant backend.

Partially implements blueprint: swift-tenant-specific-storage

Change-Id: I3def2b7c4085a36dd6c0961e762783ebaf7ffa74
This commit is contained in:
Dan Prince 2012-06-29 07:50:10 -04:00
parent 1b45a115dd
commit 53e210a0b3
10 changed files with 66 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
glance/api/v1/images.py
View File

@ -34,6 +34,7 @@ from webob.exc import (HTTPError,
from glance.api import common
from glance.api import policy
import glance.api.v1
from glance import context
from glance.api.v1 import controller
from glance.api.v1 import filters
from glance.common import exception
@ -251,9 +252,9 @@ class Controller(controller.BaseController):
return Controller._validate_source(source, req)
@staticmethod
def _get_from_store(where):
def _get_from_store(context, where):
try:
image_data, image_size = get_from_backend(where)
image_data, image_size = get_from_backend(context, where)
except exception.NotFound, e:
raise HTTPNotFound(explanation="%s" % e)
image_size = int(image_size) if image_size else None
@ -275,7 +276,8 @@ class Controller(controller.BaseController):
if image_meta.get('size') == 0:
image_iterator = iter([])
else:
image_iterator, size = self._get_from_store(image_meta['location'])
image_iterator, size = self._get_from_store(req.context,
image_meta['location'])
image_iterator = utils.cooperative_iter(image_iterator)
image_meta['size'] = size or image_meta['size']
@ -310,7 +312,8 @@ class Controller(controller.BaseController):
self.get_store_or_400(req, store)
# retrieve the image size from remote store (if not provided)
image_meta['size'] = self._get_size(image_meta, location)
image_meta['size'] = self._get_size(req.context, image_meta,
location)
else:
# Ensure that the size attribute is set to zero for directly
# uploadable images (if not provided). The size will be set
@ -357,7 +360,8 @@ class Controller(controller.BaseController):
copy_from = self._copy_from(req)
if copy_from:
image_data, image_size = self._get_from_store(copy_from)
image_data, image_size = self._get_from_store(req.context,
copy_from)
image_meta['size'] = image_size or image_meta['size']
else:
try:
@ -555,9 +559,10 @@ class Controller(controller.BaseController):
location = self._upload(req, image_meta)
return self._activate(req, image_id, location)
def _get_size(self, image_meta, location):
def _get_size(self, context, image_meta, location):
# retrieve the image size from remote store (if not provided)
return image_meta.get('size', 0) or get_size_from_backend(location)
return image_meta.get('size', 0) or get_size_from_backend(context,
location)
def _handle_source(self, req, image_id, image_meta, image_data):
if image_data or self._copy_from(req):
@ -675,7 +680,8 @@ class Controller(controller.BaseController):
try:
if location:
image_meta['size'] = self._get_size(image_meta, location)
image_meta['size'] = self._get_size(req.context, image_meta,
location)
image_meta = registry.update_image_metadata(req.context,
id,
@ -781,7 +787,7 @@ class Controller(controller.BaseController):
:raises HTTPNotFound if store does not exist
"""
try:
return get_store_from_scheme(scheme)
return get_store_from_scheme(request.context, scheme)
except exception.UnknownScheme:
msg = _("Store for scheme %s not found")
LOG.error(msg % scheme)
@ -797,7 +803,7 @@ class Controller(controller.BaseController):
:param scheme: The backend store scheme
"""
try:
get_store_from_scheme(scheme)
get_store_from_scheme(context.RequestContext(), scheme)
except exception.UnknownScheme:
msg = _("Store for scheme %s not found")
LOG.error(msg % scheme)

8
glance/api/v2/image_data.py
View File

@ -40,7 +40,7 @@ class ImageDataController(object):
self._get_image(req.context, image_id)
try:
location, size, checksum = self.store_api.add_to_backend(
'file', image_id, data, size)
req.context, 'file', image_id, data, size)
except exception.Duplicate:
raise webob.exc.HTTPConflict()
@ -48,10 +48,12 @@ class ImageDataController(object):
self.db_api.image_update(req.context, image_id, values)
def download(self, req, image_id):
image = self._get_image(req.context, image_id)
ctx = req.context
image = self._get_image(ctx, image_id)
location = image['location']
if location:
image_data, image_size = self.store_api.get_from_backend(location)
image_data, image_size = self.store_api.get_from_backend(ctx,
location)
return {'data': image_data, 'size': image_size}
else:
raise webob.exc.HTTPNotFound(_("No image data could be found"))

2
glance/image_cache/prefetcher.py
View File

@ -60,7 +60,7 @@ class Prefetcher(base.CacheApp):
LOG.warn(_("No metadata found for image '%s'"), image_id)
return False
image_data, image_size = get_from_backend(image_meta['location'])
image_data, image_size = get_from_backend(ctx, image_meta['location'])
LOG.debug(_("Caching image '%s'"), image_id)
self.cache.cache_image_iter(image_id, image_data)
return True

36
glance/store/__init__.py
View File

@ -46,9 +46,6 @@ store_opts = [
CONF = cfg.CONF
CONF.register_opts(store_opts)
# Set of store objects, constructed in create_stores()
STORES = {}
class ImageAddResult(object):
@ -161,6 +158,7 @@ def create_stores():
from the given config. Duplicates are not re-registered.
"""
store_count = 0
store_classes = set()
for store_entry in CONF.known_stores:
store_entry = store_entry.strip()
if not store_entry:
@ -173,10 +171,10 @@ def create_stores():
'No schemes associated with it.'
% store_cls)
else:
if store_cls not in STORES:
if store_cls not in store_classes:
LOG.debug("Registering store %s with schemes %s",
store_cls, schemes)
STORES[store_cls] = store_instance
store_classes.add(store_cls)
scheme_map = {}
for scheme in schemes:
loc_cls = store_instance.get_store_location_class()
@ -191,7 +189,7 @@ def create_stores():
return store_count
def get_store_from_scheme(scheme):
def get_store_from_scheme(context, scheme):
"""
Given a scheme, return the appropriate store object
for handling that scheme.
@ -199,10 +197,11 @@ def get_store_from_scheme(scheme):
if scheme not in location.SCHEME_TO_CLS_MAP:
raise exception.UnknownScheme(scheme=scheme)
scheme_info = location.SCHEME_TO_CLS_MAP[scheme]
return STORES[scheme_info['store_class']]
store = scheme_info['store_class'](context)
return store
def get_store_from_uri(uri):
def get_store_from_uri(context, uri):
"""
Given a URI, return the store object that would handle
operations on the URI.
@ -210,30 +209,31 @@ def get_store_from_uri(uri):
:param uri: URI to analyze
"""
scheme = uri[0:uri.find('/') - 1]
return get_store_from_scheme(scheme)
store = get_store_from_scheme(context, scheme)
return store
def get_from_backend(uri, **kwargs):
def get_from_backend(context, uri, **kwargs):
"""Yields chunks of data from backend specified by uri"""
store = get_store_from_uri(uri)
store = get_store_from_uri(context, uri)
loc = location.get_location_from_uri(uri)
return store.get(loc)
def get_size_from_backend(uri):
def get_size_from_backend(context, uri):
"""Retrieves image size from backend specified by uri"""
store = get_store_from_uri(uri)
store = get_store_from_uri(context, uri)
loc = location.get_location_from_uri(uri)
return store.get_size(loc)
def delete_from_backend(uri, **kwargs):
def delete_from_backend(context, uri, **kwargs):
"""Removes chunks of data from backend specified by uri"""
store = get_store_from_uri(uri)
store = get_store_from_uri(context, uri)
loc = location.get_location_from_uri(uri)
try:
@ -262,7 +262,7 @@ def schedule_delete_from_backend(uri, context, image_id, **kwargs):
registry.update_image_metadata(context, image_id,
{'status': 'deleted'})
try:
return delete_from_backend(uri, **kwargs)
return delete_from_backend(context, uri, **kwargs)
except (UnsupportedBackend,
exception.StoreDeleteNotSupported,
exception.NotFound):
@ -293,6 +293,6 @@ def schedule_delete_from_backend(uri, context, image_id, **kwargs):
{'status': 'pending_delete'})
def add_to_backend(scheme, image_id, data, size):
store = get_store_from_scheme(scheme)
def add_to_backend(context, scheme, image_id, data, size):
store = get_store_from_scheme(context, scheme)
return store.add(image_id, data, size)

3
glance/store/base.py
View File

@ -28,11 +28,12 @@ class Store(object):
CHUNKSIZE = (16 * 1024 * 1024) # 16M
def __init__(self):
def __init__(self, context=None):
"""
Initialize the Store
"""
self.store_location_class = None
self.context = context
self.configure()
try:

17
glance/store/scrubber.py
View File

@ -20,11 +20,11 @@ import eventlet
import os
import time
from glance import context
from glance.common import utils
from glance.openstack.common import cfg
import glance.openstack.common.log as logging
from glance import registry
from glance.registry import client
from glance import store
import glance.store.filesystem
import glance.store.http
@ -74,6 +74,9 @@ class Scrubber(object):
self.datadir = CONF.scrubber_datadir
self.cleanup = CONF.cleanup_scrubber
self.cleanup_time = CONF.cleanup_scrubber_time
# configs for registry API store auth
self.admin_user = CONF.admin_user
self.admin_tenant = CONF.admin_tenant_name
host, port = CONF.registry_host, CONF.registry_port
@ -82,7 +85,10 @@ class Scrubber(object):
'cleanup_time': self.cleanup_time,
'registry_host': host, 'registry_port': port})
self.registry = client.RegistryClient(host, port)
registry.configure_registry_client()
registry.configure_registry_admin_creds()
ctx = context.RequestContext()
self.registry = registry.get_registry_client(ctx)
utils.safe_mkdirs(self.datadir)
@ -124,7 +130,12 @@ class Scrubber(object):
file_path = os.path.join(self.datadir, str(id))
try:
LOG.debug(_("Deleting %(uri)s") % {'uri': uri})
store.delete_from_backend(uri)
# Here we create a request context with credentials to support
# delayed delete when using multi-tenant backend storage
ctx = context.RequestContext(auth_tok=self.registry.auth_tok,
user=self.admin_user,
tenant=self.admin_tenant)
store.delete_from_backend(ctx, uri)
except store.UnsupportedBackend:
msg = _("Failed to delete image from store (%(uri)s).")
LOG.error(msg % {'uri': uri})

2
glance/tests/unit/base.py
View File

@ -36,14 +36,12 @@ class StoreClearingUnitTest(test_utils.BaseTestCase):
def setUp(self):
super(StoreClearingUnitTest, self).setUp()
# Ensure stores + locations cleared
store.STORES = {}
location.SCHEME_TO_CLS_MAP = {}
store.create_stores()
def tearDown(self):
super(StoreClearingUnitTest, self).tearDown()
# Ensure stores + locations cleared
store.STORES = {}
location.SCHEME_TO_CLS_MAP = {}

3
glance/tests/unit/test_http_store.py
View File

@ -125,9 +125,10 @@ class TestHttpStore(base.StoreClearingUnitTest):
def test_http_delete_raise_error(self):
uri = "https://netloc/path/to/file.tar.gz"
loc = get_location_from_uri(uri)
ctx = context.RequestContext()
self.assertRaises(NotImplementedError, self.store.delete, loc)
self.assertRaises(exception.StoreDeleteNotSupported,
delete_from_backend, uri)
delete_from_backend, ctx, uri)
def test_http_schedule_delete_swallows_error(self):
uri = "https://netloc/path/to/file.tar.gz"

5
glance/tests/unit/test_store_location.py
View File

@ -15,6 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from glance import context
from glance.common import exception
import glance.store
import glance.store.filesystem
@ -275,8 +276,9 @@ class TestStoreLocation(base.StoreClearingUnitTest):
'http': glance.store.http.Store,
'https': glance.store.http.Store}
ctx = context.RequestContext()
for scheme, store in good_results.items():
store_obj = glance.store.get_store_from_scheme(scheme)
store_obj = glance.store.get_store_from_scheme(ctx, scheme)
self.assertEqual(store_obj.__class__, store)
bad_results = ['fil', 'swift+h', 'unknown']
@ -284,4 +286,5 @@ class TestStoreLocation(base.StoreClearingUnitTest):
for store in bad_results:
self.assertRaises(exception.UnknownScheme,
glance.store.get_store_from_scheme,
ctx,
store)

8
glance/tests/unit/utils.py
View File

@ -91,7 +91,7 @@ class FakeStoreAPI(object):
def create_stores(self):
pass
def get_from_backend(self, location):
def get_from_backend(self, context, location):
try:
#NOTE(bcwaldon): This fake API is store-agnostic, so we only
# care about location being some unique string
@ -99,10 +99,10 @@ class FakeStoreAPI(object):
except KeyError:
raise exception.NotFound()
def get_size_from_backend(self, location):
return self.get_from_backend(location)[1]
def get_size_from_backend(self, context, location):
return self.get_from_backend(context, location)[1]
def add_to_backend(self, scheme, image_id, data, size):
def add_to_backend(self, context, scheme, image_id, data, size):
if image_id in self.data:
raise exception.Duplicate()
self.data[image_id] = (data, size or len(data))