diff --git a/glance/api/authorization.py b/glance/api/authorization.py index 791c537187..ff26c73e60 100644 --- a/glance/api/authorization.py +++ b/glance/api/authorization.py @@ -35,7 +35,7 @@ def lazy_update_store_info(func): def wrapped(context, image, image_repo, **kwargs): if CONF.enabled_backends: store_utils.update_store_in_locations( - image, image_repo) + context, image, image_repo) return func(context, image, image_repo, **kwargs) diff --git a/glance/common/store_utils.py b/glance/common/store_utils.py index ebc132b7f0..ac5ccbee7a 100644 --- a/glance/common/store_utils.py +++ b/glance/common/store_utils.py @@ -21,7 +21,7 @@ from oslo_utils import encodeutils import six.moves.urllib.parse as urlparse import glance.db as db_api -from glance.i18n import _LE +from glance.i18n import _LE, _LW from glance import scrubber LOG = logging.getLogger(__name__) @@ -178,11 +178,15 @@ def _get_store_id_from_uri(uri): return -def update_store_in_locations(image, image_repo): +def update_store_in_locations(context, image, image_repo): + store_updated = False for loc in image.locations: if (not loc['metadata'].get( 'store') or loc['metadata'].get( 'store') not in CONF.enabled_backends): + if loc['url'].startswith("cinder://"): + _update_cinder_location_and_store_id(context, loc) + store_id = _get_store_id_from_uri(loc['url']) if store_id: if 'store' in loc['metadata']: @@ -195,8 +199,43 @@ def update_store_in_locations(image, image_repo): 'new': store_id, 'id': image.image_id}) + store_updated = True loc['metadata']['store'] = store_id - image_repo.save(image) + + if store_updated: + image_repo.save(image) + + +def _update_cinder_location_and_store_id(context, loc): + """Update store location of legacy images + + While upgrading from single cinder store to multiple stores, + the images having a store configured with a volume type matching + the image-volume's type will be migrated/associated to that store + and their location url will be updated respectively to the new format + i.e. cinder://store-id/volume-id + If there is no store configured for the image, the location url will + not be updated. + """ + uri = loc['url'] + volume_id = loc['url'].split("/")[-1] + scheme = urlparse.urlparse(uri).scheme + location_map = store_api.location.SCHEME_TO_CLS_BACKEND_MAP + if scheme not in location_map: + LOG.warning(_LW("Unknown scheme '%(scheme)s' found in uri '%(uri)s'"), + {'scheme': scheme, 'uri': uri}) + return + + for store in location_map[scheme]: + store_instance = location_map[scheme][store]['store'] + if store_instance.is_image_associated_with_store(context, volume_id): + url_prefix = store_instance.url_prefix + loc['url'] = "%s/%s" % (url_prefix, volume_id) + loc['metadata']['store'] = "%s" % store + return + + LOG.warning(_LW("Not able to update location url '%s' of legacy image " + "due to unknown issues."), uri) def get_updated_store_location(locations): diff --git a/glance/tests/unit/base.py b/glance/tests/unit/base.py index 4b069e2299..d859dbea47 100644 --- a/glance/tests/unit/base.py +++ b/glance/tests/unit/base.py @@ -71,7 +71,8 @@ class MultiStoreClearingUnitTest(test_utils.BaseTestCase): :returns: the number of how many store drivers been loaded. """ self.config(enabled_backends={'fast': 'file', 'cheap': 'file', - 'readonly_store': 'http'}) + 'readonly_store': 'http', + 'fast-cinder': 'cinder'}) store.register_store_opts(CONF) self.config(default_backend='fast', diff --git a/glance/tests/unit/common/test_utils.py b/glance/tests/unit/common/test_utils.py index 9e2cae56f3..d320ba9995 100644 --- a/glance/tests/unit/common/test_utils.py +++ b/glance/tests/unit/common/test_utils.py @@ -14,10 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -import glance_store as store import tempfile from unittest import mock +import glance_store as store +from glance_store._drivers import cinder from oslo_config import cfg from oslo_log import log as logging import six @@ -26,6 +27,7 @@ import webob from glance.common import exception from glance.common import store_utils from glance.common import utils +from glance.tests.unit import base from glance.tests import utils as test_utils @@ -41,6 +43,7 @@ class TestStoreUtils(test_utils.BaseTestCase): image = mock.Mock() image_repo = mock.Mock() image_repo.save = mock.Mock() + context = mock.Mock() locations = [{ 'url': 'rbd://aaaaaaaa/images/id', 'metadata': metadata @@ -49,7 +52,7 @@ class TestStoreUtils(test_utils.BaseTestCase): with mock.patch.object( store_utils, '_get_store_id_from_uri') as mock_get_store_id: mock_get_store_id.return_value = store_id - store_utils.update_store_in_locations(image, image_repo) + store_utils.update_store_in_locations(context, image, image_repo) self.assertEqual(image.locations[0]['metadata'].get( 'store'), expected) self.assertEqual(store_id_call_count, mock_get_store_id.call_count) @@ -92,6 +95,64 @@ class TestStoreUtils(test_utils.BaseTestCase): save_call_count=0) +class TestCinderStoreUtils(base.MultiStoreClearingUnitTest): + """Test glance.common.store_utils module for cinder multistore""" + + @mock.patch.object(cinder.Store, 'is_image_associated_with_store') + @mock.patch.object(cinder.Store, 'url_prefix', + new_callable=mock.PropertyMock) + def _test_update_cinder_store_in_location(self, mock_url_prefix, + mock_associate_store, + is_valid=True): + volume_id = 'db457a25-8f16-4b2c-a644-eae8d17fe224' + store_id = 'fast-cinder' + expected = 'fast-cinder' + image = mock.Mock() + image_repo = mock.Mock() + image_repo.save = mock.Mock() + context = mock.Mock() + mock_associate_store.return_value = is_valid + locations = [{ + 'url': 'cinder://%s' % volume_id, + 'metadata': {} + }] + mock_url_prefix.return_value = 'cinder://%s' % store_id + image.locations = locations + store_utils.update_store_in_locations(context, image, image_repo) + + if is_valid: + # This is the case where we found an image that has an + # old-style URL which does not include the store name, + # but for which we know the corresponding store that + # refers to the volume type that backs it. We expect that + # the URL should be updated to point to the store/volume from + # just a naked pointer to the volume, as was the old + # format i.e. this is the case when store is valid and location + # url, metadata are updated and image_repo.save is called + expected_url = mock_url_prefix.return_value + '/' + volume_id + self.assertEqual(expected_url, image.locations[0].get('url')) + self.assertEqual(expected, image.locations[0]['metadata'].get( + 'store')) + self.assertEqual(1, image_repo.save.call_count) + else: + # Here, we've got an image backed by a volume which does + # not have a corresponding store specifying the volume_type. + # Expect that we leave these alone and do not touch the + # location URL since we cannot update it with a valid store i.e. + # this is the case when store is invalid and location url, + # metadata are not updated and image_repo.save is not called + self.assertEqual(locations[0]['url'], + image.locations[0].get('url')) + self.assertEqual({}, image.locations[0]['metadata']) + self.assertEqual(0, image_repo.save.call_count) + + def test_update_cinder_store_location_valid_type(self): + self._test_update_cinder_store_in_location() + + def test_update_cinder_store_location_invalid_type(self): + self._test_update_cinder_store_in_location(is_valid=False) + + class TestUtils(test_utils.BaseTestCase): """Test routines in glance.utils""" diff --git a/glance/tests/unit/v2/test_discovery_stores.py b/glance/tests/unit/v2/test_discovery_stores.py index 51974bf3af..0f1fad05d2 100644 --- a/glance/tests/unit/v2/test_discovery_stores.py +++ b/glance/tests/unit/v2/test_discovery_stores.py @@ -39,7 +39,7 @@ class TestInfoControllers(base.MultiStoreClearingUnitTest): req) def test_get_stores(self): - available_stores = ['cheap', 'fast', 'readonly_store'] + available_stores = ['cheap', 'fast', 'readonly_store', 'fast-cinder'] req = unit_test_utils.get_fake_request() output = self.controller.get_stores(req) self.assertIn('stores', output) @@ -48,7 +48,7 @@ class TestInfoControllers(base.MultiStoreClearingUnitTest): self.assertIn(stores['id'], available_stores) def test_get_stores_read_only_store(self): - available_stores = ['cheap', 'fast', 'readonly_store'] + available_stores = ['cheap', 'fast', 'readonly_store', 'fast-cinder'] req = unit_test_utils.get_fake_request() output = self.controller.get_stores(req) self.assertIn('stores', output) diff --git a/lower-constraints.txt b/lower-constraints.txt index d259a8e0ab..6bf90afdbc 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -29,7 +29,7 @@ fasteners==0.14.1 fixtures==3.0.0 future==0.16.0 futurist==1.2.0 -glance-store==1.0.0 +glance-store==2.3.0 greenlet==0.4.13 httplib2==0.9.1 idna==2.6 diff --git a/releasenotes/notes/support-cinder-multiple-stores-eb4e6d912d549ee9.yaml b/releasenotes/notes/support-cinder-multiple-stores-eb4e6d912d549ee9.yaml new file mode 100644 index 0000000000..5e7ca6e61d --- /dev/null +++ b/releasenotes/notes/support-cinder-multiple-stores-eb4e6d912d549ee9.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Added support for cinder multiple stores. +upgrade: + - | + During upgrade from single cinder store to multiple cinder stores, legacy + images location url will be updated to the new format with respect to the + volume type configured in the stores. + Legacy location url: cinder:// + New location url: cinder:/// + diff --git a/requirements.txt b/requirements.txt index 6f28374728..374e8e4270 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,7 @@ retrying!=1.3.0,>=1.2.3 # Apache-2.0 osprofiler>=1.4.0 # Apache-2.0 # Glance Store -glance-store>=1.0.0 # Apache-2.0 +glance-store>=2.3.0 # Apache-2.0 debtcollector>=1.2.0 # Apache-2.0