Allow wildcard in index --type argument
Allow a wildcard character * at the end of the --type argument to searchlight-manage index sync --type arguments. For instance, OS::Cinder::* will match all resources with doc types starting OS::Cinder::. The substitution is done very early early in the process so as to largely leave the code alone. Patch also adds functional tests for the manage command (both with and without this functionality). Change-Id: Ia7f1efc0a08f085ce081a800c385246c81663bc4 Implements: blueprint allow-reindexing-entire-service
This commit is contained in:
parent
f7d1a48428
commit
5cda277269
|
@ -113,6 +113,14 @@ up-to-date information from the Glance API. Data for other plugins will be
|
|||
bulk-copied from a preexisting index into the new one using the scroll_ and
|
||||
bulk_ features of Elasticsearch.
|
||||
|
||||
You can use the wildcard character * at the *end* of the ``type`` argument.
|
||||
For instance, the following will match all cinder plugins::
|
||||
|
||||
$ searchlight-manage index sync --type OS::Cinder::*
|
||||
|
||||
Wildcard characters are only allowed at the end of the argument; they will not
|
||||
be matched anywhere else.
|
||||
|
||||
To index all resources in the 'searchlight' resource group::
|
||||
|
||||
$ searchlight-manage index sync --index searchlight
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
prelude: >
|
||||
Allow wildcard character as an argument to
|
||||
searchlight-manage index sync --type
|
||||
features:
|
||||
- The --type argument to searchlight-manage index sync
|
||||
accepts a wildcard as the final character.
|
|
@ -165,11 +165,18 @@ class IndexCommands(object):
|
|||
"than 0."))
|
||||
sys.exit(3)
|
||||
|
||||
# Grab the list of plugins registered as entry points through stevedore
|
||||
search_plugins = utils.get_search_plugins()
|
||||
|
||||
# Verify all indices and types have registered plugins.
|
||||
# index and _type are lists because of nargs='*'
|
||||
group = group.split(',') if group else []
|
||||
_type = _type.split(',') if _type else []
|
||||
|
||||
_type = utils.expand_type_matches(
|
||||
_type, six.viewkeys(search_plugins))
|
||||
LOG.debug("After expansion, 'type' argument: %s", ", ".join(_type))
|
||||
|
||||
group_set = set(group)
|
||||
type_set = set(_type)
|
||||
|
||||
|
@ -198,7 +205,7 @@ class IndexCommands(object):
|
|||
|
||||
# First Pass: Document Types.
|
||||
if _type:
|
||||
for res_type, ext in six.iteritems(utils.get_search_plugins()):
|
||||
for res_type, ext in six.iteritems(search_plugins):
|
||||
plugin_obj = ext.obj
|
||||
type_set.discard(plugin_obj.get_document_type())
|
||||
if plugin_obj.get_document_type() in _type:
|
||||
|
@ -211,7 +218,7 @@ class IndexCommands(object):
|
|||
resource_groups = []
|
||||
plugin_objs = {}
|
||||
plugins_list = []
|
||||
for res_type, ext in six.iteritems(utils.get_search_plugins()):
|
||||
for res_type, ext in six.iteritems(search_plugins):
|
||||
plugin_obj = ext.obj
|
||||
group_set.discard(plugin_obj.resource_group_name)
|
||||
if (not group) or (plugin_obj.resource_group_name in group):
|
||||
|
@ -336,7 +343,7 @@ class IndexCommands(object):
|
|||
# from the plugins and add a mapping for them as needed to the newly
|
||||
# created indices.
|
||||
doc_type_info = []
|
||||
for res_type, ext in six.iteritems(utils.get_search_plugins()):
|
||||
for res_type, ext in six.iteritems(search_plugins):
|
||||
doc_type_info.append((ext.obj.get_document_type(),
|
||||
ext.obj.parent_plugin_type))
|
||||
for index in list(index_names.values()):
|
||||
|
|
|
@ -452,3 +452,23 @@ def get_search_plugins():
|
|||
loaded_plugin.obj.register_parent(parent_plugin.obj)
|
||||
|
||||
return plugins
|
||||
|
||||
|
||||
def expand_type_matches(types, document_types):
|
||||
"""Returns a list that will be at least the length of `types`. If an item
|
||||
ends with a wildcard character and matches at least one plugin document
|
||||
type it'll be replaced with the matches in the return list.
|
||||
"""
|
||||
expanded_types = []
|
||||
for _type in types:
|
||||
if _type.endswith('*'):
|
||||
matches = [doc_type for doc_type in document_types
|
||||
if doc_type.startswith(_type[:-1])]
|
||||
if matches:
|
||||
expanded_types.extend(matches)
|
||||
continue
|
||||
|
||||
# If we got here, either wasn't a wildcard or didn't match anything
|
||||
expanded_types.append(_type)
|
||||
|
||||
return expanded_types
|
||||
|
|
|
@ -64,7 +64,8 @@ ROUTING_DATA = [
|
|||
|
||||
SIMPLE_DATA = [
|
||||
{
|
||||
"id": "simple1"
|
||||
"id": "simple1",
|
||||
"updated_at": "2015-08-06T12:48:14.000000"
|
||||
}
|
||||
]
|
||||
|
||||
|
@ -72,7 +73,8 @@ SIMPLE_DATA = [
|
|||
CHILD_DATA = [
|
||||
{
|
||||
"id": "child1",
|
||||
"parent_id": "simple1"
|
||||
"parent_id": "simple1",
|
||||
"updated_at": "2015-08-06T12:48:14.000000"
|
||||
}
|
||||
]
|
||||
|
||||
|
|
|
@ -134,7 +134,7 @@ class TestCinderNotifications(test_listener.TestSearchListenerBase):
|
|||
self.snapshot_events = self._load_fixture_data('events/snapshots.json')
|
||||
|
||||
notification_plugins = {
|
||||
plugin.document_type: test_listener.StevedoreMock(plugin)
|
||||
plugin.document_type: utils.StevedoreMock(plugin)
|
||||
for plugin in (self.volume_plugin, self.snapshot_plugin)}
|
||||
self.notification_endpoint = NotificationEndpoint(notification_plugins)
|
||||
|
||||
|
|
|
@ -18,6 +18,7 @@ import mock
|
|||
from searchlight.listener import NotificationEndpoint
|
||||
from searchlight.tests import functional
|
||||
from searchlight.tests.functional import test_listener
|
||||
from searchlight.tests import utils
|
||||
|
||||
|
||||
PROJECT1 = "34518c16d95e40a19b1a95c1916d8335"
|
||||
|
@ -85,7 +86,7 @@ class TestDesignateListener(test_listener.TestSearchListenerBase):
|
|||
self.addCleanup(osclient_patcher.stop)'''
|
||||
|
||||
notification_plugins = {
|
||||
plugin.document_type: test_listener.StevedoreMock(plugin)
|
||||
plugin.document_type: utils.StevedoreMock(plugin)
|
||||
for plugin in (self.zones_plugin, self.recordsets_plugin)}
|
||||
self.notification_endpoint = NotificationEndpoint(notification_plugins)
|
||||
|
||||
|
|
|
@ -25,6 +25,7 @@ from searchlight.tests.functional import generate_load_data
|
|||
from searchlight.tests.functional import mock_glance_pyclient
|
||||
from searchlight.tests.functional import test_api
|
||||
from searchlight.tests.functional import test_listener
|
||||
from searchlight.tests import utils
|
||||
|
||||
|
||||
member_list = 'glanceclient.v2.image_members.Controller.list'
|
||||
|
@ -406,7 +407,7 @@ class TestGlanceListener(test_listener.TestSearchListenerBase):
|
|||
self.metadefs_plugin = self.initialized_plugins['OS::Glance::Metadef']
|
||||
|
||||
notification_plugins = {
|
||||
plugin.document_type: test_listener.StevedoreMock(plugin)
|
||||
plugin.document_type: utils.StevedoreMock(plugin)
|
||||
for plugin in (self.images_plugin, self.metadefs_plugin)}
|
||||
self.notification_endpoint = NotificationEndpoint(notification_plugins)
|
||||
|
||||
|
|
|
@ -25,13 +25,6 @@ OWNER1 = str(uuid.uuid4())
|
|||
NETWORK_TENANT_ID = '8eaac046b2c44ab99246cb0850c7f06d'
|
||||
|
||||
|
||||
class StevedoreMock(object):
|
||||
"""Act like a stevedore-loaded plugin for the sake of the listener code"""
|
||||
def __init__(self, plugin):
|
||||
self.obj = plugin
|
||||
self.name = plugin.document_type
|
||||
|
||||
|
||||
class TestSearchListenerBase(functional.FunctionalTest):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
|
|
@ -13,15 +13,18 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import mock
|
||||
|
||||
from oslo_config import cfg
|
||||
import oslo_utils
|
||||
|
||||
from searchlight.cmd import manage
|
||||
from searchlight.elasticsearch.plugins import utils as es_utils
|
||||
from searchlight.elasticsearch import ROLE_USER_FIELD
|
||||
from searchlight.tests import fake_plugins
|
||||
from searchlight.tests import functional
|
||||
from searchlight.tests import utils as test_utils
|
||||
|
||||
|
||||
now = oslo_utils.timeutils.utcnow()
|
||||
|
@ -118,3 +121,130 @@ class TestSearchLoad(functional.FunctionalTest):
|
|||
|
||||
finally:
|
||||
es_utils.delete_index(index_name)
|
||||
|
||||
@mock.patch('oslo_utils.timeutils.utcnow')
|
||||
@mock.patch('searchlight.common.utils.get_search_plugins')
|
||||
def test_manage(self, mock_get_plugins, mock_utcnow):
|
||||
"""Test that manage index sync works from end to end. Uses fake plugins
|
||||
because it avoids having to fake service data and is less dependent
|
||||
on functional tests for each service plugin.
|
||||
"""
|
||||
mock_utcnow.return_value = datetime.datetime(year=2016, month=1, day=1)
|
||||
expected_index_name = 'searchlight-2016_01_01_00_00_00'
|
||||
|
||||
simple_plugin = fake_plugins.FakeSimplePlugin(self.elastic_connection)
|
||||
child_plugin = fake_plugins.FakeChildPlugin(self.elastic_connection)
|
||||
non_role_plugin = fake_plugins.NonRoleSeparatedPlugin(
|
||||
self.elastic_connection)
|
||||
child_plugin.register_parent(simple_plugin)
|
||||
|
||||
mock_get_plugins.return_value = {
|
||||
plugin.get_document_type(): test_utils.StevedoreMock(plugin)
|
||||
for plugin in (simple_plugin, child_plugin, non_role_plugin)
|
||||
}
|
||||
index_command = manage.IndexCommands()
|
||||
# The fake plugins all have hardcoded data for get_objects that will
|
||||
# be indexed. Use force=True to avoid the 'are you sure?' prompt
|
||||
try:
|
||||
index_command.sync(force=True)
|
||||
|
||||
es_results = self._get_all_elasticsearch_docs()
|
||||
es_hits = self._get_hit_source(es_results)
|
||||
finally:
|
||||
es_utils.delete_index(expected_index_name)
|
||||
|
||||
self.assertEqual(expected_index_name,
|
||||
es_results['hits']['hits'][0]['_index'])
|
||||
|
||||
expected = ['simple1', 'child1', 'non-role-fake1', 'non-role-fake2']
|
||||
self.assertEqual(len(expected), len(es_hits))
|
||||
self.assertEqual(
|
||||
set(expected),
|
||||
set(hit['_id'] for hit in es_results['hits']['hits']))
|
||||
|
||||
@mock.patch('oslo_utils.timeutils.utcnow')
|
||||
@mock.patch('searchlight.common.utils.get_search_plugins')
|
||||
def test_manage_type(self, mock_get_plugins, mock_utcnow):
|
||||
mock_utcnow.return_value = datetime.datetime(year=2016, month=2, day=2)
|
||||
expected_index_name = 'searchlight-2016_02_02_00_00_00'
|
||||
|
||||
simple_plugin = fake_plugins.FakeSimplePlugin(self.elastic_connection)
|
||||
non_role_plugin = fake_plugins.NonRoleSeparatedPlugin(
|
||||
self.elastic_connection)
|
||||
|
||||
mock_get_plugins.return_value = {
|
||||
plugin.get_document_type(): test_utils.StevedoreMock(plugin)
|
||||
for plugin in (simple_plugin, non_role_plugin)
|
||||
}
|
||||
index_command = manage.IndexCommands()
|
||||
|
||||
# Expect this to match simple plugin and child plugin, but not
|
||||
# non_role_plugin. Patch the index->index function call since it won't
|
||||
# find any data, and we want to check it's called correctly
|
||||
with mock.patch.object(index_command,
|
||||
'_es_reindex_worker') as patch_es_reindex:
|
||||
try:
|
||||
index_command.sync(_type='fake-simple', force=True)
|
||||
|
||||
es_results = self._get_all_elasticsearch_docs()
|
||||
es_hits = self._get_hit_source(es_results)
|
||||
finally:
|
||||
es_utils.delete_index(expected_index_name)
|
||||
|
||||
patch_es_reindex.assert_called_with(
|
||||
[non_role_plugin.get_document_type()],
|
||||
[('searchlight', 'searchlight-search',
|
||||
'searchlight-listener')],
|
||||
{'searchlight': expected_index_name}
|
||||
)
|
||||
|
||||
expected = ['simple1']
|
||||
self.assertEqual(len(expected), len(es_hits))
|
||||
self.assertEqual(
|
||||
set(expected),
|
||||
set(hit['_id'] for hit in es_results['hits']['hits']))
|
||||
|
||||
@mock.patch('oslo_utils.timeutils.utcnow')
|
||||
@mock.patch('searchlight.common.utils.get_search_plugins')
|
||||
def test_manage_type_glob(self, mock_get_plugins, mock_utcnow):
|
||||
mock_utcnow.return_value = datetime.datetime(year=2016, month=3, day=3)
|
||||
expected_index_name = 'searchlight-2016_03_03_00_00_00'
|
||||
|
||||
simple_plugin = fake_plugins.FakeSimplePlugin(self.elastic_connection)
|
||||
child_plugin = fake_plugins.FakeChildPlugin(self.elastic_connection)
|
||||
non_role_plugin = fake_plugins.NonRoleSeparatedPlugin(
|
||||
self.elastic_connection)
|
||||
child_plugin.register_parent(simple_plugin)
|
||||
|
||||
mock_get_plugins.return_value = {
|
||||
plugin.get_document_type(): test_utils.StevedoreMock(plugin)
|
||||
for plugin in (simple_plugin, child_plugin, non_role_plugin)
|
||||
}
|
||||
index_command = manage.IndexCommands()
|
||||
|
||||
# Expect this to match simple plugin and child plugin, but not
|
||||
# non_role_plugin. Patch the index->index function call since it won't
|
||||
# find any data, and we want to check it's called correctly
|
||||
with mock.patch.object(index_command,
|
||||
'_es_reindex_worker') as patch_es_reindex:
|
||||
try:
|
||||
# Use two wildcard matches
|
||||
index_command.sync(_type='fake-sim*,fake-chi*', force=True)
|
||||
|
||||
es_results = self._get_all_elasticsearch_docs()
|
||||
es_hits = self._get_hit_source(es_results)
|
||||
finally:
|
||||
es_utils.delete_index(expected_index_name)
|
||||
|
||||
patch_es_reindex.assert_called_with(
|
||||
[non_role_plugin.get_document_type()],
|
||||
[('searchlight', 'searchlight-search',
|
||||
'searchlight-listener')],
|
||||
{'searchlight': expected_index_name}
|
||||
)
|
||||
|
||||
expected = ['simple1', 'child1']
|
||||
self.assertEqual(len(expected), len(es_hits))
|
||||
self.assertEqual(
|
||||
set(expected),
|
||||
set(hit['_id'] for hit in es_results['hits']['hits']))
|
||||
|
|
|
@ -21,6 +21,7 @@ from searchlight.listener import NotificationEndpoint
|
|||
from searchlight.tests import functional
|
||||
from searchlight.tests.functional import test_api
|
||||
from searchlight.tests.functional import test_listener
|
||||
from searchlight.tests import utils
|
||||
|
||||
# These is in the load file
|
||||
TENANT1 = "8eaac046b2c44ab99246cb0850c7f06d"
|
||||
|
@ -299,7 +300,7 @@ class TestNeutronListeners(test_listener.TestSearchListenerBase):
|
|||
self.initialized_plugins['OS::Neutron::SecurityGroup'])
|
||||
|
||||
notification_plugins = {
|
||||
plugin.document_type: test_listener.StevedoreMock(plugin)
|
||||
plugin.document_type: utils.StevedoreMock(plugin)
|
||||
for plugin in (self.networks_plugin, self.ports_plugin,
|
||||
self.subnets_plugin, self.routers_plugin,
|
||||
self.fip_plugin, self.secgroup_plugin)}
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
import mock
|
||||
|
||||
from elasticsearch import exceptions as es_exc
|
||||
from searchlight.common import utils as common_utils
|
||||
from searchlight.elasticsearch.plugins import utils as plugin_utils
|
||||
from searchlight.tests import utils as test_utils
|
||||
|
||||
|
@ -333,3 +334,32 @@ class TestReindexingUtils(test_utils.BaseTestCase):
|
|||
mock.call(ignore=404, index='ndx2'),
|
||||
mock.call(ignore=404, index='ndx3')]
|
||||
mock_engine.indices.delete.assert_has_calls(calls, any_order=True)
|
||||
|
||||
def test_type_expansion(self):
|
||||
doc_types = ['simple-plugin', 'simple-child', 'something-else']
|
||||
|
||||
# Don't do any expansion
|
||||
self.assertEqual(doc_types,
|
||||
common_utils.expand_type_matches(doc_types,
|
||||
doc_types))
|
||||
|
||||
self.assertEqual([], common_utils.expand_type_matches([], doc_types))
|
||||
|
||||
# Expansion time!
|
||||
self.assertEqual(
|
||||
['simple-plugin', 'simple-child'],
|
||||
common_utils.expand_type_matches(['simple-*'], doc_types))
|
||||
|
||||
self.assertEqual(
|
||||
['something-else'],
|
||||
common_utils.expand_type_matches(['something-*'], doc_types))
|
||||
|
||||
# Doesn't match. Sad!
|
||||
self.assertEqual(
|
||||
['does-not-match-*'],
|
||||
common_utils.expand_type_matches(['does-not-match-*'], doc_types))
|
||||
|
||||
self.assertEqual(
|
||||
doc_types,
|
||||
common_utils.expand_type_matches(['simple-*', 'something-*'],
|
||||
doc_types))
|
||||
|
|
|
@ -580,3 +580,10 @@ class FlavorDictObj(DictObj):
|
|||
d = copy.deepcopy(super(FlavorDictObj, self).to_dict())
|
||||
d.pop("extra_spec", None)
|
||||
return d
|
||||
|
||||
|
||||
class StevedoreMock(object):
|
||||
"""Act like a stevedore-loaded plugin"""
|
||||
def __init__(self, plugin):
|
||||
self.obj = plugin
|
||||
self.name = plugin.document_type
|
||||
|
|
Loading…
Reference in New Issue