Provide stores access to the request context.

Adds context to the constructor for the Store class. This can be used
to provide Store's access to the RequestContext.

Includes the following changes:

* Updates common store helper methods to create new instance of
  Store objects for each request.

* Updates Glance scrubber to create a real context from credentials.

The motivation for this change is that we will need access to the
service catalog and token for the Swift multi-tenant backend.

Partially implements blueprint: swift-tenant-specific-storage

Change-Id: I3def2b7c4085a36dd6c0961e762783ebaf7ffa74

glance/api/v1/images.py

@@ -34,6 +34,7 @@ from webob.exc import (HTTPError,
 from glance.api import common
 from glance.api import policy
 import glance.api.v1
+from glance import context
 from glance.api.v1 import controller
 from glance.api.v1 import filters
 from glance.common import exception
@@ -251,9 +252,9 @@ class Controller(controller.BaseController):
         return Controller._validate_source(source, req)
 
     @staticmethod
-    def _get_from_store(where):
+    def _get_from_store(context, where):
         try:
-            image_data, image_size = get_from_backend(where)
+            image_data, image_size = get_from_backend(context, where)
         except exception.NotFound, e:
             raise HTTPNotFound(explanation="%s" % e)
         image_size = int(image_size) if image_size else None
@@ -275,7 +276,8 @@ class Controller(controller.BaseController):
         if image_meta.get('size') == 0:
             image_iterator = iter([])
         else:
-            image_iterator, size = self._get_from_store(image_meta['location'])
+            image_iterator, size = self._get_from_store(req.context,
+                                                        image_meta['location'])
             image_iterator = utils.cooperative_iter(image_iterator)
             image_meta['size'] = size or image_meta['size']
 
@@ -310,7 +312,8 @@ class Controller(controller.BaseController):
             self.get_store_or_400(req, store)
 
             # retrieve the image size from remote store (if not provided)
-            image_meta['size'] = self._get_size(image_meta, location)
+            image_meta['size'] = self._get_size(req.context, image_meta,
+                                                location)
         else:
             # Ensure that the size attribute is set to zero for directly
             # uploadable images (if not provided). The size will be set
@@ -357,7 +360,8 @@ class Controller(controller.BaseController):
 
         copy_from = self._copy_from(req)
         if copy_from:
-            image_data, image_size = self._get_from_store(copy_from)
+            image_data, image_size = self._get_from_store(req.context,
+                                                          copy_from)
             image_meta['size'] = image_size or image_meta['size']
         else:
             try:
@@ -555,9 +559,10 @@ class Controller(controller.BaseController):
         location = self._upload(req, image_meta)
         return self._activate(req, image_id, location)
 
-    def _get_size(self, image_meta, location):
+    def _get_size(self, context, image_meta, location):
         # retrieve the image size from remote store (if not provided)
-        return image_meta.get('size', 0) or get_size_from_backend(location)
+        return image_meta.get('size', 0) or get_size_from_backend(context,
+                                                                  location)
 
     def _handle_source(self, req, image_id, image_meta, image_data):
         if image_data or self._copy_from(req):
@@ -675,7 +680,8 @@ class Controller(controller.BaseController):
 
         try:
             if location:
-                image_meta['size'] = self._get_size(image_meta, location)
+                image_meta['size'] = self._get_size(req.context, image_meta,
+                                                    location)
 
             image_meta = registry.update_image_metadata(req.context,
                                                         id,
@@ -781,7 +787,7 @@ class Controller(controller.BaseController):
         :raises HTTPNotFound if store does not exist
         """
         try:
-            return get_store_from_scheme(scheme)
+            return get_store_from_scheme(request.context, scheme)
         except exception.UnknownScheme:
             msg = _("Store for scheme %s not found")
             LOG.error(msg % scheme)
@@ -797,7 +803,7 @@ class Controller(controller.BaseController):
         :param scheme: The backend store scheme
         """
         try:
-            get_store_from_scheme(scheme)
+            get_store_from_scheme(context.RequestContext(), scheme)
         except exception.UnknownScheme:
             msg = _("Store for scheme %s not found")
             LOG.error(msg % scheme)

glance/api/v2/image_data.py

@@ -40,7 +40,7 @@ class ImageDataController(object):
         self._get_image(req.context, image_id)
         try:
             location, size, checksum = self.store_api.add_to_backend(
-                    'file', image_id, data, size)
+                    req.context, 'file', image_id, data, size)
         except exception.Duplicate:
             raise webob.exc.HTTPConflict()
 
@@ -48,10 +48,12 @@ class ImageDataController(object):
         self.db_api.image_update(req.context, image_id, values)
 
     def download(self, req, image_id):
-        image = self._get_image(req.context, image_id)
+        ctx = req.context
+        image = self._get_image(ctx, image_id)
         location = image['location']
         if location:
-            image_data, image_size = self.store_api.get_from_backend(location)
+            image_data, image_size = self.store_api.get_from_backend(ctx,
+                                                                     location)
             return {'data': image_data, 'size': image_size}
         else:
             raise webob.exc.HTTPNotFound(_("No image data could be found"))

glance/image_cache/prefetcher.py

@@ -60,7 +60,7 @@ class Prefetcher(base.CacheApp):
             LOG.warn(_("No metadata found for image '%s'"), image_id)
             return False
 
-        image_data, image_size = get_from_backend(image_meta['location'])
+        image_data, image_size = get_from_backend(ctx, image_meta['location'])
         LOG.debug(_("Caching image '%s'"), image_id)
         self.cache.cache_image_iter(image_id, image_data)
         return True

glance/store/__init__.py

@@ -46,9 +46,6 @@ store_opts = [
 CONF = cfg.CONF
 CONF.register_opts(store_opts)
 
-# Set of store objects, constructed in create_stores()
-STORES = {}
-
 
 class ImageAddResult(object):
 
@@ -161,6 +158,7 @@ def create_stores():
     from the given config. Duplicates are not re-registered.
     """
     store_count = 0
+    store_classes = set()
     for store_entry in CONF.known_stores:
         store_entry = store_entry.strip()
         if not store_entry:
@@ -173,10 +171,10 @@ def create_stores():
                                    'No schemes associated with it.'
                                    % store_cls)
         else:
-            if store_cls not in STORES:
+            if store_cls not in store_classes:
                 LOG.debug("Registering store %s with schemes %s",
                           store_cls, schemes)
-                STORES[store_cls] = store_instance
+                store_classes.add(store_cls)
                 scheme_map = {}
                 for scheme in schemes:
                     loc_cls = store_instance.get_store_location_class()
@@ -191,7 +189,7 @@ def create_stores():
     return store_count
 
 
-def get_store_from_scheme(scheme):
+def get_store_from_scheme(context, scheme):
     """
     Given a scheme, return the appropriate store object
     for handling that scheme.
@@ -199,10 +197,11 @@ def get_store_from_scheme(scheme):
     if scheme not in location.SCHEME_TO_CLS_MAP:
         raise exception.UnknownScheme(scheme=scheme)
     scheme_info = location.SCHEME_TO_CLS_MAP[scheme]
-    return STORES[scheme_info['store_class']]
+    store = scheme_info['store_class'](context)
+    return store
 
 
-def get_store_from_uri(uri):
+def get_store_from_uri(context, uri):
     """
     Given a URI, return the store object that would handle
     operations on the URI.
@@ -210,30 +209,31 @@ def get_store_from_uri(uri):
     :param uri: URI to analyze
     """
     scheme = uri[0:uri.find('/') - 1]
-    return get_store_from_scheme(scheme)
+    store = get_store_from_scheme(context, scheme)
+    return store
 
 
-def get_from_backend(uri, **kwargs):
+def get_from_backend(context, uri, **kwargs):
     """Yields chunks of data from backend specified by uri"""
 
-    store = get_store_from_uri(uri)
+    store = get_store_from_uri(context, uri)
     loc = location.get_location_from_uri(uri)
 
     return store.get(loc)
 
 
-def get_size_from_backend(uri):
+def get_size_from_backend(context, uri):
     """Retrieves image size from backend specified by uri"""
 
-    store = get_store_from_uri(uri)
+    store = get_store_from_uri(context, uri)
     loc = location.get_location_from_uri(uri)
 
     return store.get_size(loc)
 
 
-def delete_from_backend(uri, **kwargs):
+def delete_from_backend(context, uri, **kwargs):
     """Removes chunks of data from backend specified by uri"""
-    store = get_store_from_uri(uri)
+    store = get_store_from_uri(context, uri)
     loc = location.get_location_from_uri(uri)
 
     try:
@@ -262,7 +262,7 @@ def schedule_delete_from_backend(uri, context, image_id, **kwargs):
         registry.update_image_metadata(context, image_id,
                                        {'status': 'deleted'})
         try:
-            return delete_from_backend(uri, **kwargs)
+            return delete_from_backend(context, uri, **kwargs)
         except (UnsupportedBackend,
                 exception.StoreDeleteNotSupported,
                 exception.NotFound):
@@ -293,6 +293,6 @@ def schedule_delete_from_backend(uri, context, image_id, **kwargs):
                                    {'status': 'pending_delete'})
 
 
-def add_to_backend(scheme, image_id, data, size):
+def add_to_backend(context, scheme, image_id, data, size):
-    store = get_store_from_scheme(scheme)
+    store = get_store_from_scheme(context, scheme)
     return store.add(image_id, data, size)

glance/store/base.py

@@ -28,11 +28,12 @@ class Store(object):
 
     CHUNKSIZE = (16 * 1024 * 1024)  # 16M
 
-    def __init__(self):
+    def __init__(self, context=None):
         """
         Initialize the Store
         """
         self.store_location_class = None
+        self.context = context
         self.configure()
 
         try:

glance/store/scrubber.py

@@ -20,11 +20,11 @@ import eventlet
 import os
 import time
 
+from glance import context
 from glance.common import utils
 from glance.openstack.common import cfg
 import glance.openstack.common.log as logging
 from glance import registry
-from glance.registry import client
 from glance import store
 import glance.store.filesystem
 import glance.store.http
@@ -74,6 +74,9 @@ class Scrubber(object):
         self.datadir = CONF.scrubber_datadir
         self.cleanup = CONF.cleanup_scrubber
         self.cleanup_time = CONF.cleanup_scrubber_time
+        # configs for registry API store auth
+        self.admin_user = CONF.admin_user
+        self.admin_tenant = CONF.admin_tenant_name
 
         host, port = CONF.registry_host, CONF.registry_port
 
@@ -82,7 +85,10 @@ class Scrubber(object):
                   'cleanup_time': self.cleanup_time,
                   'registry_host': host, 'registry_port': port})
 
-        self.registry = client.RegistryClient(host, port)
+        registry.configure_registry_client()
+        registry.configure_registry_admin_creds()
+        ctx = context.RequestContext()
+        self.registry = registry.get_registry_client(ctx)
 
         utils.safe_mkdirs(self.datadir)
 
@@ -124,7 +130,12 @@ class Scrubber(object):
         file_path = os.path.join(self.datadir, str(id))
         try:
             LOG.debug(_("Deleting %(uri)s") % {'uri': uri})
-            store.delete_from_backend(uri)
+            # Here we create a request context with credentials to support
+            # delayed delete when using multi-tenant backend storage
+            ctx = context.RequestContext(auth_tok=self.registry.auth_tok,
+                                         user=self.admin_user,
+                                         tenant=self.admin_tenant)
+            store.delete_from_backend(ctx, uri)
         except store.UnsupportedBackend:
             msg = _("Failed to delete image from store (%(uri)s).")
             LOG.error(msg % {'uri': uri})

glance/tests/unit/base.py

@@ -36,14 +36,12 @@ class StoreClearingUnitTest(test_utils.BaseTestCase):
     def setUp(self):
         super(StoreClearingUnitTest, self).setUp()
         # Ensure stores + locations cleared
-        store.STORES = {}
         location.SCHEME_TO_CLS_MAP = {}
         store.create_stores()
 
     def tearDown(self):
         super(StoreClearingUnitTest, self).tearDown()
         # Ensure stores + locations cleared
-        store.STORES = {}
         location.SCHEME_TO_CLS_MAP = {}
 
 

glance/tests/unit/test_http_store.py

@@ -125,9 +125,10 @@ class TestHttpStore(base.StoreClearingUnitTest):
     def test_http_delete_raise_error(self):
         uri = "https://netloc/path/to/file.tar.gz"
         loc = get_location_from_uri(uri)
+        ctx = context.RequestContext()
         self.assertRaises(NotImplementedError, self.store.delete, loc)
         self.assertRaises(exception.StoreDeleteNotSupported,
-                          delete_from_backend, uri)
+                          delete_from_backend, ctx, uri)
 
     def test_http_schedule_delete_swallows_error(self):
         uri = "https://netloc/path/to/file.tar.gz"

glance/tests/unit/test_store_location.py

@@ -15,6 +15,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from glance import context
 from glance.common import exception
 import glance.store
 import glance.store.filesystem
@@ -275,8 +276,9 @@ class TestStoreLocation(base.StoreClearingUnitTest):
             'http': glance.store.http.Store,
             'https': glance.store.http.Store}
 
+        ctx = context.RequestContext()
         for scheme, store in good_results.items():
-            store_obj = glance.store.get_store_from_scheme(scheme)
+            store_obj = glance.store.get_store_from_scheme(ctx, scheme)
             self.assertEqual(store_obj.__class__, store)
 
         bad_results = ['fil', 'swift+h', 'unknown']
@@ -284,4 +286,5 @@ class TestStoreLocation(base.StoreClearingUnitTest):
         for store in bad_results:
             self.assertRaises(exception.UnknownScheme,
                               glance.store.get_store_from_scheme,
+                              ctx,
                               store)

glance/tests/unit/utils.py

@@ -91,7 +91,7 @@ class FakeStoreAPI(object):
     def create_stores(self):
         pass
 
-    def get_from_backend(self, location):
+    def get_from_backend(self, context, location):
         try:
             #NOTE(bcwaldon): This fake API is store-agnostic, so we only
             # care about location being some unique string
@@ -99,10 +99,10 @@ class FakeStoreAPI(object):
         except KeyError:
             raise exception.NotFound()
 
-    def get_size_from_backend(self, location):
+    def get_size_from_backend(self, context, location):
-        return self.get_from_backend(location)[1]
+        return self.get_from_backend(context, location)[1]
 
-    def add_to_backend(self, scheme, image_id, data, size):
+    def add_to_backend(self, context, scheme, image_id, data, size):
         if image_id in self.data:
             raise exception.Duplicate()
         self.data[image_id] = (data, size or len(data))