From 53e210a0b32d1fad4678e978995433692cfb09cc Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Fri, 29 Jun 2012 07:50:10 -0400 Subject: [PATCH] Provide stores access to the request context. Adds context to the constructor for the Store class. This can be used to provide Store's access to the RequestContext. Includes the following changes: * Updates common store helper methods to create new instance of Store objects for each request. * Updates Glance scrubber to create a real context from credentials. The motivation for this change is that we will need access to the service catalog and token for the Swift multi-tenant backend. Partially implements blueprint: swift-tenant-specific-storage Change-Id: I3def2b7c4085a36dd6c0961e762783ebaf7ffa74 --- glance/api/v1/images.py | 26 ++++++++++------- glance/api/v2/image_data.py | 8 ++++-- glance/image_cache/prefetcher.py | 2 +- glance/store/__init__.py | 36 ++++++++++++------------ glance/store/base.py | 3 +- glance/store/scrubber.py | 17 +++++++++-- glance/tests/unit/base.py | 2 -- glance/tests/unit/test_http_store.py | 3 +- glance/tests/unit/test_store_location.py | 5 +++- glance/tests/unit/utils.py | 8 +++--- 10 files changed, 66 insertions(+), 44 deletions(-) diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index 0ffcb2b05c..8697e0238f 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -34,6 +34,7 @@ from webob.exc import (HTTPError, from glance.api import common from glance.api import policy import glance.api.v1 +from glance import context from glance.api.v1 import controller from glance.api.v1 import filters from glance.common import exception @@ -251,9 +252,9 @@ class Controller(controller.BaseController): return Controller._validate_source(source, req) @staticmethod - def _get_from_store(where): + def _get_from_store(context, where): try: - image_data, image_size = get_from_backend(where) + image_data, image_size = get_from_backend(context, where) except exception.NotFound, e: raise HTTPNotFound(explanation="%s" % e) image_size = int(image_size) if image_size else None @@ -275,7 +276,8 @@ class Controller(controller.BaseController): if image_meta.get('size') == 0: image_iterator = iter([]) else: - image_iterator, size = self._get_from_store(image_meta['location']) + image_iterator, size = self._get_from_store(req.context, + image_meta['location']) image_iterator = utils.cooperative_iter(image_iterator) image_meta['size'] = size or image_meta['size'] @@ -310,7 +312,8 @@ class Controller(controller.BaseController): self.get_store_or_400(req, store) # retrieve the image size from remote store (if not provided) - image_meta['size'] = self._get_size(image_meta, location) + image_meta['size'] = self._get_size(req.context, image_meta, + location) else: # Ensure that the size attribute is set to zero for directly # uploadable images (if not provided). The size will be set @@ -357,7 +360,8 @@ class Controller(controller.BaseController): copy_from = self._copy_from(req) if copy_from: - image_data, image_size = self._get_from_store(copy_from) + image_data, image_size = self._get_from_store(req.context, + copy_from) image_meta['size'] = image_size or image_meta['size'] else: try: @@ -555,9 +559,10 @@ class Controller(controller.BaseController): location = self._upload(req, image_meta) return self._activate(req, image_id, location) - def _get_size(self, image_meta, location): + def _get_size(self, context, image_meta, location): # retrieve the image size from remote store (if not provided) - return image_meta.get('size', 0) or get_size_from_backend(location) + return image_meta.get('size', 0) or get_size_from_backend(context, + location) def _handle_source(self, req, image_id, image_meta, image_data): if image_data or self._copy_from(req): @@ -675,7 +680,8 @@ class Controller(controller.BaseController): try: if location: - image_meta['size'] = self._get_size(image_meta, location) + image_meta['size'] = self._get_size(req.context, image_meta, + location) image_meta = registry.update_image_metadata(req.context, id, @@ -781,7 +787,7 @@ class Controller(controller.BaseController): :raises HTTPNotFound if store does not exist """ try: - return get_store_from_scheme(scheme) + return get_store_from_scheme(request.context, scheme) except exception.UnknownScheme: msg = _("Store for scheme %s not found") LOG.error(msg % scheme) @@ -797,7 +803,7 @@ class Controller(controller.BaseController): :param scheme: The backend store scheme """ try: - get_store_from_scheme(scheme) + get_store_from_scheme(context.RequestContext(), scheme) except exception.UnknownScheme: msg = _("Store for scheme %s not found") LOG.error(msg % scheme) diff --git a/glance/api/v2/image_data.py b/glance/api/v2/image_data.py index 5e3f79847f..459c4d525a 100644 --- a/glance/api/v2/image_data.py +++ b/glance/api/v2/image_data.py @@ -40,7 +40,7 @@ class ImageDataController(object): self._get_image(req.context, image_id) try: location, size, checksum = self.store_api.add_to_backend( - 'file', image_id, data, size) + req.context, 'file', image_id, data, size) except exception.Duplicate: raise webob.exc.HTTPConflict() @@ -48,10 +48,12 @@ class ImageDataController(object): self.db_api.image_update(req.context, image_id, values) def download(self, req, image_id): - image = self._get_image(req.context, image_id) + ctx = req.context + image = self._get_image(ctx, image_id) location = image['location'] if location: - image_data, image_size = self.store_api.get_from_backend(location) + image_data, image_size = self.store_api.get_from_backend(ctx, + location) return {'data': image_data, 'size': image_size} else: raise webob.exc.HTTPNotFound(_("No image data could be found")) diff --git a/glance/image_cache/prefetcher.py b/glance/image_cache/prefetcher.py index 7f15781eb3..dd9081e9c3 100644 --- a/glance/image_cache/prefetcher.py +++ b/glance/image_cache/prefetcher.py @@ -60,7 +60,7 @@ class Prefetcher(base.CacheApp): LOG.warn(_("No metadata found for image '%s'"), image_id) return False - image_data, image_size = get_from_backend(image_meta['location']) + image_data, image_size = get_from_backend(ctx, image_meta['location']) LOG.debug(_("Caching image '%s'"), image_id) self.cache.cache_image_iter(image_id, image_data) return True diff --git a/glance/store/__init__.py b/glance/store/__init__.py index 369f37e403..2c3a046869 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -46,9 +46,6 @@ store_opts = [ CONF = cfg.CONF CONF.register_opts(store_opts) -# Set of store objects, constructed in create_stores() -STORES = {} - class ImageAddResult(object): @@ -161,6 +158,7 @@ def create_stores(): from the given config. Duplicates are not re-registered. """ store_count = 0 + store_classes = set() for store_entry in CONF.known_stores: store_entry = store_entry.strip() if not store_entry: @@ -173,10 +171,10 @@ def create_stores(): 'No schemes associated with it.' % store_cls) else: - if store_cls not in STORES: + if store_cls not in store_classes: LOG.debug("Registering store %s with schemes %s", store_cls, schemes) - STORES[store_cls] = store_instance + store_classes.add(store_cls) scheme_map = {} for scheme in schemes: loc_cls = store_instance.get_store_location_class() @@ -191,7 +189,7 @@ def create_stores(): return store_count -def get_store_from_scheme(scheme): +def get_store_from_scheme(context, scheme): """ Given a scheme, return the appropriate store object for handling that scheme. @@ -199,10 +197,11 @@ def get_store_from_scheme(scheme): if scheme not in location.SCHEME_TO_CLS_MAP: raise exception.UnknownScheme(scheme=scheme) scheme_info = location.SCHEME_TO_CLS_MAP[scheme] - return STORES[scheme_info['store_class']] + store = scheme_info['store_class'](context) + return store -def get_store_from_uri(uri): +def get_store_from_uri(context, uri): """ Given a URI, return the store object that would handle operations on the URI. @@ -210,30 +209,31 @@ def get_store_from_uri(uri): :param uri: URI to analyze """ scheme = uri[0:uri.find('/') - 1] - return get_store_from_scheme(scheme) + store = get_store_from_scheme(context, scheme) + return store -def get_from_backend(uri, **kwargs): +def get_from_backend(context, uri, **kwargs): """Yields chunks of data from backend specified by uri""" - store = get_store_from_uri(uri) + store = get_store_from_uri(context, uri) loc = location.get_location_from_uri(uri) return store.get(loc) -def get_size_from_backend(uri): +def get_size_from_backend(context, uri): """Retrieves image size from backend specified by uri""" - store = get_store_from_uri(uri) + store = get_store_from_uri(context, uri) loc = location.get_location_from_uri(uri) return store.get_size(loc) -def delete_from_backend(uri, **kwargs): +def delete_from_backend(context, uri, **kwargs): """Removes chunks of data from backend specified by uri""" - store = get_store_from_uri(uri) + store = get_store_from_uri(context, uri) loc = location.get_location_from_uri(uri) try: @@ -262,7 +262,7 @@ def schedule_delete_from_backend(uri, context, image_id, **kwargs): registry.update_image_metadata(context, image_id, {'status': 'deleted'}) try: - return delete_from_backend(uri, **kwargs) + return delete_from_backend(context, uri, **kwargs) except (UnsupportedBackend, exception.StoreDeleteNotSupported, exception.NotFound): @@ -293,6 +293,6 @@ def schedule_delete_from_backend(uri, context, image_id, **kwargs): {'status': 'pending_delete'}) -def add_to_backend(scheme, image_id, data, size): - store = get_store_from_scheme(scheme) +def add_to_backend(context, scheme, image_id, data, size): + store = get_store_from_scheme(context, scheme) return store.add(image_id, data, size) diff --git a/glance/store/base.py b/glance/store/base.py index 0d8c76bb4f..2594cdf53b 100644 --- a/glance/store/base.py +++ b/glance/store/base.py @@ -28,11 +28,12 @@ class Store(object): CHUNKSIZE = (16 * 1024 * 1024) # 16M - def __init__(self): + def __init__(self, context=None): """ Initialize the Store """ self.store_location_class = None + self.context = context self.configure() try: diff --git a/glance/store/scrubber.py b/glance/store/scrubber.py index 1265f46493..d6ff5f9cec 100644 --- a/glance/store/scrubber.py +++ b/glance/store/scrubber.py @@ -20,11 +20,11 @@ import eventlet import os import time +from glance import context from glance.common import utils from glance.openstack.common import cfg import glance.openstack.common.log as logging from glance import registry -from glance.registry import client from glance import store import glance.store.filesystem import glance.store.http @@ -74,6 +74,9 @@ class Scrubber(object): self.datadir = CONF.scrubber_datadir self.cleanup = CONF.cleanup_scrubber self.cleanup_time = CONF.cleanup_scrubber_time + # configs for registry API store auth + self.admin_user = CONF.admin_user + self.admin_tenant = CONF.admin_tenant_name host, port = CONF.registry_host, CONF.registry_port @@ -82,7 +85,10 @@ class Scrubber(object): 'cleanup_time': self.cleanup_time, 'registry_host': host, 'registry_port': port}) - self.registry = client.RegistryClient(host, port) + registry.configure_registry_client() + registry.configure_registry_admin_creds() + ctx = context.RequestContext() + self.registry = registry.get_registry_client(ctx) utils.safe_mkdirs(self.datadir) @@ -124,7 +130,12 @@ class Scrubber(object): file_path = os.path.join(self.datadir, str(id)) try: LOG.debug(_("Deleting %(uri)s") % {'uri': uri}) - store.delete_from_backend(uri) + # Here we create a request context with credentials to support + # delayed delete when using multi-tenant backend storage + ctx = context.RequestContext(auth_tok=self.registry.auth_tok, + user=self.admin_user, + tenant=self.admin_tenant) + store.delete_from_backend(ctx, uri) except store.UnsupportedBackend: msg = _("Failed to delete image from store (%(uri)s).") LOG.error(msg % {'uri': uri}) diff --git a/glance/tests/unit/base.py b/glance/tests/unit/base.py index 5db08fff19..6cafab353c 100644 --- a/glance/tests/unit/base.py +++ b/glance/tests/unit/base.py @@ -36,14 +36,12 @@ class StoreClearingUnitTest(test_utils.BaseTestCase): def setUp(self): super(StoreClearingUnitTest, self).setUp() # Ensure stores + locations cleared - store.STORES = {} location.SCHEME_TO_CLS_MAP = {} store.create_stores() def tearDown(self): super(StoreClearingUnitTest, self).tearDown() # Ensure stores + locations cleared - store.STORES = {} location.SCHEME_TO_CLS_MAP = {} diff --git a/glance/tests/unit/test_http_store.py b/glance/tests/unit/test_http_store.py index ea93cc8130..84a71564c8 100644 --- a/glance/tests/unit/test_http_store.py +++ b/glance/tests/unit/test_http_store.py @@ -125,9 +125,10 @@ class TestHttpStore(base.StoreClearingUnitTest): def test_http_delete_raise_error(self): uri = "https://netloc/path/to/file.tar.gz" loc = get_location_from_uri(uri) + ctx = context.RequestContext() self.assertRaises(NotImplementedError, self.store.delete, loc) self.assertRaises(exception.StoreDeleteNotSupported, - delete_from_backend, uri) + delete_from_backend, ctx, uri) def test_http_schedule_delete_swallows_error(self): uri = "https://netloc/path/to/file.tar.gz" diff --git a/glance/tests/unit/test_store_location.py b/glance/tests/unit/test_store_location.py index 4e78930420..2fd61e8154 100644 --- a/glance/tests/unit/test_store_location.py +++ b/glance/tests/unit/test_store_location.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +from glance import context from glance.common import exception import glance.store import glance.store.filesystem @@ -275,8 +276,9 @@ class TestStoreLocation(base.StoreClearingUnitTest): 'http': glance.store.http.Store, 'https': glance.store.http.Store} + ctx = context.RequestContext() for scheme, store in good_results.items(): - store_obj = glance.store.get_store_from_scheme(scheme) + store_obj = glance.store.get_store_from_scheme(ctx, scheme) self.assertEqual(store_obj.__class__, store) bad_results = ['fil', 'swift+h', 'unknown'] @@ -284,4 +286,5 @@ class TestStoreLocation(base.StoreClearingUnitTest): for store in bad_results: self.assertRaises(exception.UnknownScheme, glance.store.get_store_from_scheme, + ctx, store) diff --git a/glance/tests/unit/utils.py b/glance/tests/unit/utils.py index 502a4b6f80..e40e34a7f7 100644 --- a/glance/tests/unit/utils.py +++ b/glance/tests/unit/utils.py @@ -91,7 +91,7 @@ class FakeStoreAPI(object): def create_stores(self): pass - def get_from_backend(self, location): + def get_from_backend(self, context, location): try: #NOTE(bcwaldon): This fake API is store-agnostic, so we only # care about location being some unique string @@ -99,10 +99,10 @@ class FakeStoreAPI(object): except KeyError: raise exception.NotFound() - def get_size_from_backend(self, location): - return self.get_from_backend(location)[1] + def get_size_from_backend(self, context, location): + return self.get_from_backend(context, location)[1] - def add_to_backend(self, scheme, image_id, data, size): + def add_to_backend(self, context, scheme, image_id, data, size): if image_id in self.data: raise exception.Duplicate() self.data[image_id] = (data, size or len(data))