Merge "Remove location_strategy functionality"

This commit is contained in:
Zuul 2024-07-31 18:50:00 +00:00 committed by Gerrit Code Review
commit 536a34b5d7
11 changed files with 13 additions and 540 deletions

View File

@ -304,8 +304,8 @@ location is where the image data is stored in backend storage. This
image location is shown under the image property ``direct_url``.
When multiple image locations exist for an image, the best location
is displayed based on the location strategy indicated by the
configuration option ``location_strategy``.
is displayed based on the store weightage assigned for each store
indicated by the configuration option ``weight``.
NOTES:
* Revealing image locations can present a GRAVE SECURITY RISK as
@ -322,7 +322,7 @@ Possible values:
Related options:
* show_multiple_locations
* location_strategy
* weight
""")),
# NOTE(flaper87): The policy.yaml file should be updated and the location
@ -354,9 +354,9 @@ Show all image locations when returning an image.
This configuration option indicates whether to show all the image
locations when returning image details to the user. When multiple
image locations exist for an image, the locations are ordered based
on the location strategy indicated by the configuration opt
``location_strategy``. The image locations are shown under the
image property ``locations``.
on the store weightage assigned for each store indicated by the
configuration option ``weight``. The image locations are shown
under the image property ``locations``.
NOTES:
* Revealing image locations can present a GRAVE SECURITY RISK as
@ -375,7 +375,7 @@ Possible values:
Related options:
* show_image_direct_url
* location_strategy
* weight
""")),
cfg.BoolOpt('do_secure_hash', default=True,

View File

@ -1,141 +0,0 @@
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
from oslo_config import cfg
from oslo_log import log as logging
import stevedore
from glance.i18n import _, _LE
location_strategy_opts = [
cfg.StrOpt('location_strategy',
deprecated_for_removal=True,
deprecated_since="Caracal",
deprecated_reason=_("""
In Bobcat, a new weighing mechanism has been introduced, which makes the
location strategy obsolete. This option is scheduled to be removed during the
Dalmatian development cycle.
"""),
default='location_order',
choices=('location_order', 'store_type'),
help=_("""
Strategy to determine the preference order of image locations.
This configuration option indicates the strategy to determine
the order in which an image's locations must be accessed to
serve the image's data. Glance then retrieves the image data
from the first responsive active location it finds in this list.
This option takes one of two possible values ``location_order``
and ``store_type``. The default value is ``location_order``,
which suggests that image data be served by using locations in
the order they are stored in Glance. The ``store_type`` value
sets the image location preference based on the order in which
the storage backends are listed as a comma separated list for
the configuration option ``store_type_preference``.
Possible values:
* location_order
* store_type
Related options:
* store_type_preference
""")),
]
CONF = cfg.CONF
CONF.register_opts(location_strategy_opts)
LOG = logging.getLogger(__name__)
def _load_strategies():
"""Load all strategy modules."""
modules = {}
namespace = "glance.common.image_location_strategy.modules"
ex = stevedore.extension.ExtensionManager(namespace)
for module_name in ex.names():
try:
mgr = stevedore.driver.DriverManager(
namespace=namespace,
name=module_name,
invoke_on_load=False)
# Obtain module name
strategy_name = str(mgr.driver.get_strategy_name())
if strategy_name in modules:
msg = (_('%(strategy)s is registered as a module twice. '
'%(module)s is not being used.') %
{'strategy': strategy_name, 'module': module_name})
LOG.warning(msg)
else:
# Initialize strategy module
mgr.driver.init()
modules[strategy_name] = mgr.driver
except Exception as e:
LOG.error(_LE("Failed to load location strategy module "
"%(module)s: %(e)s"), {'module': module_name,
'e': e})
return modules
_available_strategies = _load_strategies()
# TODO(kadachi): Not used but don't remove this until glance_store
# development/migration stage.
def verify_location_strategy(conf=None, strategies=_available_strategies):
"""Validate user configured 'location_strategy' option value."""
if not conf:
conf = CONF.location_strategy
if conf not in strategies:
msg = (_('Invalid location_strategy option: %(name)s. '
'The valid strategy option(s) is(are): %(strategies)s') %
{'name': conf, 'strategies': ", ".join(strategies.keys())})
LOG.error(msg)
raise RuntimeError(msg)
def get_ordered_locations(locations, **kwargs):
"""
Order image location list by configured strategy.
:param locations: The original image location list.
:param kwargs: Strategy-specific arguments for under layer strategy module.
:returns: The image location list with strategy-specific order.
"""
if not locations:
return []
strategy_module = _available_strategies[CONF.location_strategy]
return strategy_module.get_ordered_locations(copy.deepcopy(locations),
**kwargs)
def choose_best_location(locations, **kwargs):
"""
Choose best location from image location list by configured strategy.
:param locations: The original image location list.
:param kwargs: Strategy-specific arguments for under layer strategy module.
:returns: The best location from image location list.
"""
locations = get_ordered_locations(locations, **kwargs)
if locations:
return locations[0]
else:
return None

View File

@ -1,36 +0,0 @@
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Image location order based location strategy module"""
def get_strategy_name():
"""Return strategy module name."""
return 'location_order'
def init():
"""Initialize strategy module."""
pass
def get_ordered_locations(locations, **kwargs):
"""
Order image location list.
:param locations: The original image location list.
:returns: The image location list with original natural order.
"""
return locations

View File

@ -1,144 +0,0 @@
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Storage preference based location strategy module"""
import urllib.parse as urlparse
from oslo_config import cfg
from glance.i18n import _
store_type_opts = [
cfg.ListOpt('store_type_preference',
default=[],
deprecated_for_removal=True,
deprecated_since="Caracal",
deprecated_reason=_("""
In Bobcat, a new weighing mechanism has been introduced, which makes the
location strategy obsolete. This option is scheduled to be removed during the
Dalmatian development cycle.
"""),
help=_("""
Preference order of storage backends.
Provide a comma separated list of store names in the order in
which images should be retrieved from storage backends.
These store names must be registered with the ``stores``
configuration option.
NOTE: The ``store_type_preference`` configuration option is applied
only if ``store_type`` is chosen as a value for the
``location_strategy`` configuration option. An empty list will not
change the location order.
Possible values:
* Empty list
* Comma separated list of registered store names. Legal values are:
* file
* http
* rbd
* swift
* cinder
* vmware
Related options:
* location_strategy
* stores
"""))
]
CONF = cfg.CONF
CONF.register_opts(store_type_opts, group='store_type_location_strategy')
_STORE_TO_SCHEME_MAP = {}
def get_strategy_name():
"""Return strategy module name."""
return 'store_type'
def init():
"""Initialize strategy module."""
# NOTE(zhiyan): We have a plan to do a reusable glance client library for
# all clients like Nova and Cinder in near period, it would be able to
# contains common code to provide uniform image service interface for them,
# just like Brick in Cinder, this code can be moved to there and shared
# between Glance and client both side. So this implementation as far as
# possible to prevent make relationships with Glance(server)-specific code,
# for example: using functions within store module to validate
# 'store_type_preference' option.
mapping = {'file': ['file', 'filesystem'],
'http': ['http', 'https'],
'rbd': ['rbd'],
'swift': ['swift', 'swift+https', 'swift+http'],
'cinder': ['cinder'],
'vmware': ['vsphere']}
_STORE_TO_SCHEME_MAP.clear()
_STORE_TO_SCHEME_MAP.update(mapping)
def get_ordered_locations(locations, uri_key='url', **kwargs):
"""
Order image location list.
:param locations: The original image location list.
:param uri_key: The key name for location URI in image location dictionary.
:returns: The image location list with preferred store type order.
"""
def _foreach_store_type_preference():
store_types = CONF.store_type_location_strategy.store_type_preference
for preferred_store in store_types:
preferred_store = str(preferred_store).strip()
if not preferred_store:
continue
yield preferred_store
if not locations:
return locations
preferences = {}
others = []
for preferred_store in _foreach_store_type_preference():
preferences[preferred_store] = []
for location in locations:
uri = location.get(uri_key)
if not uri:
continue
pieces = urlparse.urlparse(uri.strip())
store_name = None
for store, schemes in _STORE_TO_SCHEME_MAP.items():
if pieces.scheme.strip() in schemes:
store_name = store
break
if store_name in preferences:
preferences[store_name].append(location)
else:
others.append(location)
ret = []
# NOTE(zhiyan): While configuration again since py26 does not support
# ordereddict container.
for preferred_store in _foreach_store_type_preference():
ret.extend(preferences[preferred_store])
ret.extend(others)
return ret

View File

@ -42,7 +42,6 @@ from oslo_utils import strutils
from webob import exc
from glance.common import exception
from glance.common import location_strategy
from glance.common import timeutils
from glance.common import wsgi
from glance.i18n import _, _LE, _LW
@ -717,7 +716,7 @@ def get_stores_from_request(req, body):
def sort_image_locations(locations):
if not CONF.enabled_backends:
return location_strategy.get_ordered_locations(locations)
return locations
def get_store_weight(location):
store_id = location['metadata'].get('store')

View File

@ -33,8 +33,6 @@ import glance.async_.flows.convert
from glance.async_.flows.plugins import plugin_opts
import glance.async_.taskflow_executor
import glance.common.config
import glance.common.location_strategy
import glance.common.location_strategy.store_type
import glance.common.property_utils
import glance.common.wsgi
import glance.image_cache
@ -48,7 +46,6 @@ _api_opts = [
glance.api.middleware.context.context_opts,
glance.api.versions.versions_opts,
glance.common.config.common_opts,
glance.common.location_strategy.location_strategy_opts,
glance.common.property_utils.property_opts,
glance.common.wsgi.bind_opts,
glance.common.wsgi.eventlet_opts,
@ -64,8 +61,6 @@ _api_opts = [
('taskflow_executor', list(itertools.chain(
glance.async_.taskflow_executor.taskflow_executor_opts,
glance.async_.flows.convert.convert_task_opts))),
('store_type_location_strategy',
glance.common.location_strategy.store_type.store_type_opts),
profiler.list_opts()[0],
('paste_deploy', glance.common.config.paste_deploy_opts),
('wsgi', glance.common.config.wsgi_opts),

View File

@ -418,9 +418,6 @@ class ApiServer(Server):
self.user_storage_quota = '0'
self.lock_path = self.test_dir
self.location_strategy = 'location_order'
self.store_type_location_strategy_preference = ""
self.node_staging_uri = 'file://%s' % os.path.join(
self.test_dir, 'staging')
@ -449,7 +446,6 @@ image_member_quota=%(image_member_quota)s
image_property_quota=%(image_property_quota)s
image_tag_quota=%(image_tag_quota)s
image_location_quota=%(image_location_quota)s
location_strategy=%(location_strategy)s
node_staging_uri=%(node_staging_uri)s
[database]
connection = %(sql_connection)s
@ -459,8 +455,6 @@ policy_default_rule = %(policy_default_rule)s
enforce_new_defaults=%(enforce_new_defaults)s
[paste_deploy]
flavor = %(deployment_flavor)s
[store_type_location_strategy]
store_type_preference = %(store_type_location_strategy_preference)s
[glance_store]
filesystem_store_datadir=%(image_dir)s
default_store = %(default_store)s
@ -609,9 +603,6 @@ class ApiServerForMultipleBackend(Server):
self.user_storage_quota = '0'
self.lock_path = self.test_dir
self.location_strategy = 'location_order'
self.store_type_location_strategy_preference = ""
self.conf_base = """[DEFAULT]
debug = %(debug)s
default_log_levels = eventlet.wsgi.server=DEBUG,stevedore.extension=INFO
@ -637,7 +628,6 @@ image_member_quota=%(image_member_quota)s
image_property_quota=%(image_property_quota)s
image_tag_quota=%(image_tag_quota)s
image_location_quota=%(image_location_quota)s
location_strategy=%(location_strategy)s
enabled_backends=file1:file,file2:file,file3:file
[database]
connection = %(sql_connection)s
@ -647,8 +637,6 @@ policy_default_rule = %(policy_default_rule)s
enforce_new_defaults=%(enforce_new_defaults)s
[paste_deploy]
flavor = %(deployment_flavor)s
[store_type_location_strategy]
store_type_preference = %(store_type_location_strategy_preference)s
[glance_store]
default_backend = %(default_backend)s
[file1]

View File

@ -4362,9 +4362,7 @@ class TestImageLocationSelectionStrategy(functional.FunctionalTest):
self.api_server.show_image_direct_url = True
self.api_server.show_multiple_locations = True
self.image_location_quota = 10
self.api_server.location_strategy = 'location_order'
preference = "http, swift, filesystem"
self.api_server.store_type_location_strategy_preference = preference
self.start_servers(**self.__dict__.copy())
# Create an image

View File

@ -1,187 +0,0 @@
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import stevedore
from glance.common import location_strategy
from glance.common.location_strategy import location_order
from glance.common.location_strategy import store_type
from glance.tests.unit import base
class TestLocationStrategy(base.IsolatedUnitTest):
"""Test routines in glance.common.location_strategy"""
def _set_original_strategies(self, original_strategies):
for name in location_strategy._available_strategies.keys():
if name not in original_strategies:
del location_strategy._available_strategies[name]
def setUp(self):
super(TestLocationStrategy, self).setUp()
original_strategies = ['location_order', 'store_type']
self.addCleanup(self._set_original_strategies, original_strategies)
def test_load_strategy_modules(self):
modules = location_strategy._load_strategies()
# By default we have two built-in strategy modules.
self.assertEqual(2, len(modules))
self.assertEqual(set(['location_order', 'store_type']),
set(modules.keys()))
self.assertEqual(location_strategy._available_strategies, modules)
def test_load_strategy_module_with_deduplicating(self):
modules = ['module1', 'module2']
def _fake_stevedore_extension_manager(*args, **kwargs):
def ret():
return None
ret.names = lambda: modules
return ret
def _fake_stevedore_driver_manager(*args, **kwargs):
def ret():
return None
ret.driver = lambda: None
ret.driver.__name__ = kwargs['name']
# Module 1 and 2 has a same strategy name
ret.driver.get_strategy_name = lambda: 'module_name'
ret.driver.init = lambda: None
return ret
self.stub = self.mock_object(stevedore.extension, "ExtensionManager",
_fake_stevedore_extension_manager)
self.stub = self.mock_object(stevedore.driver, "DriverManager",
_fake_stevedore_driver_manager)
loaded_modules = location_strategy._load_strategies()
self.assertEqual(1, len(loaded_modules))
self.assertIn('module_name', loaded_modules)
# Skipped module #2, duplicated one.
self.assertEqual('module1', loaded_modules['module_name'].__name__)
def test_load_strategy_module_with_init_exception(self):
modules = ['module_init_exception', 'module_good']
def _fake_stevedore_extension_manager(*args, **kwargs):
def ret():
return None
ret.names = lambda: modules
return ret
def _fake_stevedore_driver_manager(*args, **kwargs):
if kwargs['name'] == 'module_init_exception':
raise Exception('strategy module failed to initialize.')
else:
def ret():
return None
ret.driver = lambda: None
ret.driver.__name__ = kwargs['name']
ret.driver.get_strategy_name = lambda: kwargs['name']
ret.driver.init = lambda: None
return ret
self.stub = self.mock_object(stevedore.extension, "ExtensionManager",
_fake_stevedore_extension_manager)
self.stub = self.mock_object(stevedore.driver, "DriverManager",
_fake_stevedore_driver_manager)
loaded_modules = location_strategy._load_strategies()
self.assertEqual(1, len(loaded_modules))
self.assertIn('module_good', loaded_modules)
# Skipped module #1, initialize failed one.
self.assertEqual('module_good', loaded_modules['module_good'].__name__)
def test_verify_valid_location_strategy(self):
for strategy_name in ['location_order', 'store_type']:
self.config(location_strategy=strategy_name)
location_strategy.verify_location_strategy()
def test_get_ordered_locations_with_none_or_empty_locations(self):
self.assertEqual([], location_strategy.get_ordered_locations(None))
self.assertEqual([], location_strategy.get_ordered_locations([]))
def test_get_ordered_locations(self):
self.config(location_strategy='location_order')
original_locs = [{'url': 'loc1'}, {'url': 'loc2'}]
ordered_locs = location_strategy.get_ordered_locations(original_locs)
# Original location list should remain unchanged
self.assertNotEqual(id(original_locs), id(ordered_locs))
self.assertEqual(original_locs, ordered_locs)
def test_choose_best_location_with_none_or_empty_locations(self):
self.assertIsNone(location_strategy.choose_best_location(None))
self.assertIsNone(location_strategy.choose_best_location([]))
def test_choose_best_location(self):
self.config(location_strategy='location_order')
original_locs = [{'url': 'loc1'}, {'url': 'loc2'}]
best_loc = location_strategy.choose_best_location(original_locs)
# Deep copy protect original location.
self.assertNotEqual(id(original_locs), id(best_loc))
self.assertEqual(original_locs[0], best_loc)
class TestLocationOrderStrategyModule(base.IsolatedUnitTest):
"""Test routines in glance.common.location_strategy.location_order"""
def test_get_ordered_locations(self):
original_locs = [{'url': 'loc1'}, {'url': 'loc2'}]
ordered_locs = location_order.get_ordered_locations(original_locs)
# The result will ordered by original natural order.
self.assertEqual(original_locs, ordered_locs)
class TestStoreTypeStrategyModule(base.IsolatedUnitTest):
"""Test routines in glance.common.location_strategy.store_type"""
def test_get_ordered_locations(self):
self.config(store_type_preference=[' rbd', ' file',
'swift ', ' http ', 'vmware'],
group='store_type_location_strategy')
locs = [{'url': 'file://image0', 'metadata': {'idx': 3}},
{'url': 'rbd://image1', 'metadata': {'idx': 0}},
{'url': 'file://image3', 'metadata': {'idx': 4}},
{'url': 'swift://image4', 'metadata': {'idx': 6}},
{'url': 'cinder://image5', 'metadata': {'idx': 9}},
{'url': 'file://image6', 'metadata': {'idx': 5}},
{'url': 'rbd://image7', 'metadata': {'idx': 1}},
{'url': 'vsphere://image8', 'metadata': {'idx': 8}}]
ordered_locs = store_type.get_ordered_locations(copy.deepcopy(locs))
locs.sort(key=lambda loc: loc['metadata']['idx'])
# The result will ordered by preferred store type order.
self.assertEqual(locs, ordered_locs)
def test_get_ordered_locations_with_invalid_store_name(self):
self.config(store_type_preference=[' rbd', 'invalid',
'swift ', ' http '],
group='store_type_location_strategy')
locs = [{'url': 'file://image0', 'metadata': {'idx': 4}},
{'url': 'rbd://image1', 'metadata': {'idx': 0}},
{'url': 'file://image3', 'metadata': {'idx': 5}},
{'url': 'swift://image4', 'metadata': {'idx': 3}},
{'url': 'cinder://image5', 'metadata': {'idx': 6}},
{'url': 'file://image6', 'metadata': {'idx': 7}},
{'url': 'rbd://image7', 'metadata': {'idx': 1}}]
ordered_locs = store_type.get_ordered_locations(copy.deepcopy(locs))
locs.sort(key=lambda loc: loc['metadata']['idx'])
# The result will ordered by preferred store type order.
self.assertEqual(locs, ordered_locs)

View File

@ -0,0 +1,4 @@
---
upgrade:
- The ``location_strategy`` functionality which was deprecated in
Bobcat(2023.2), has been removed in this release.

View File

@ -47,9 +47,6 @@ console_scripts =
glance-status = glance.cmd.status:main
wsgi_scripts =
glance-wsgi-api = glance.common.wsgi_app:init_app
glance.common.image_location_strategy.modules =
location_order_strategy = glance.common.location_strategy.location_order
store_type_strategy = glance.common.location_strategy.store_type
oslo.config.opts =
glance.api = glance.opts:list_api_opts
glance.scrubber = glance.opts:list_scrubber_opts