openstack
/
searchlight
Code Issues Proposed changes

Merge "Allow wildcard in index --type argument"

This commit is contained in:
Jenkins 2016-07-28 20:16:18 +00:00 committed by Gerrit Code Review
parent 698c39421e 5cda277269
commit e8e8b6a5b0
13 changed files with 223 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doc/source/indexingservice.rst
View File

@ -113,6 +113,14 @@ up-to-date information from the Glance API. Data for other plugins will be
bulk-copied from a preexisting index into the new one using the scroll_ and
bulk_ features of Elasticsearch.
You can use the wildcard character * at the *end* of the ``type`` argument.
For instance, the following will match all cinder plugins::
$ searchlight-manage index sync --type OS::Cinder::*
Wildcard characters are only allowed at the end of the argument; they will not
be matched anywhere else.
To index all resources in the 'searchlight' resource group::
$ searchlight-manage index sync --index searchlight

7
releasenotes/notes/bp-indexing-type-wildcard-94c38a719a9210af.yaml Normal file
View File

@ -0,0 +1,7 @@
---
prelude: >
Allow wildcard character as an argument to
searchlight-manage index sync --type
features:
- The --type argument to searchlight-manage index sync
accepts a wildcard as the final character.

13
searchlight/cmd/manage.py
View File

@ -165,11 +165,18 @@ class IndexCommands(object):
"than 0."))
sys.exit(3)
# Grab the list of plugins registered as entry points through stevedore
search_plugins = utils.get_search_plugins()
# Verify all indices and types have registered plugins.
# index and _type are lists because of nargs='*'
group = group.split(',') if group else []
_type = _type.split(',') if _type else []
_type = utils.expand_type_matches(
_type, six.viewkeys(search_plugins))
LOG.debug("After expansion, 'type' argument: %s", ", ".join(_type))
group_set = set(group)
type_set = set(_type)
@ -198,7 +205,7 @@ class IndexCommands(object):
# First Pass: Document Types.
if _type:
for res_type, ext in six.iteritems(utils.get_search_plugins()):
for res_type, ext in six.iteritems(search_plugins):
plugin_obj = ext.obj
type_set.discard(plugin_obj.get_document_type())
if plugin_obj.get_document_type() in _type:
@ -211,7 +218,7 @@ class IndexCommands(object):
resource_groups = []
plugin_objs = {}
plugins_list = []
for res_type, ext in six.iteritems(utils.get_search_plugins()):
for res_type, ext in six.iteritems(search_plugins):
plugin_obj = ext.obj
group_set.discard(plugin_obj.resource_group_name)
if (not group) or (plugin_obj.resource_group_name in group):
@ -336,7 +343,7 @@ class IndexCommands(object):
# from the plugins and add a mapping for them as needed to the newly
# created indices.
doc_type_info = []
for res_type, ext in six.iteritems(utils.get_search_plugins()):
for res_type, ext in six.iteritems(search_plugins):
doc_type_info.append((ext.obj.get_document_type(),
ext.obj.parent_plugin_type))
for index in list(index_names.values()):

20
searchlight/common/utils.py
View File

@ -452,3 +452,23 @@ def get_search_plugins():
loaded_plugin.obj.register_parent(parent_plugin.obj)
return plugins
def expand_type_matches(types, document_types):
"""Returns a list that will be at least the length of `types`. If an item
ends with a wildcard character and matches at least one plugin document
type it'll be replaced with the matches in the return list.
"""
expanded_types = []
for _type in types:
if _type.endswith('*'):
matches = [doc_type for doc_type in document_types
if doc_type.startswith(_type[:-1])]
if matches:
expanded_types.extend(matches)
continue
# If we got here, either wasn't a wildcard or didn't match anything
expanded_types.append(_type)
return expanded_types

6
searchlight/tests/fake_plugins.py
View File

@ -64,7 +64,8 @@ ROUTING_DATA = [
SIMPLE_DATA = [
{
"id": "simple1"
"id": "simple1",
"updated_at": "2015-08-06T12:48:14.000000"
}
]
@ -72,7 +73,8 @@ SIMPLE_DATA = [
CHILD_DATA = [
{
"id": "child1",
"parent_id": "simple1"
"parent_id": "simple1",
"updated_at": "2015-08-06T12:48:14.000000"
}
]

2
searchlight/tests/functional/test_cinder_plugins.py
View File

@ -134,7 +134,7 @@ class TestCinderNotifications(test_listener.TestSearchListenerBase):
self.snapshot_events = self._load_fixture_data('events/snapshots.json')
notification_plugins = {
plugin.document_type: test_listener.StevedoreMock(plugin)
plugin.document_type: utils.StevedoreMock(plugin)
for plugin in (self.volume_plugin, self.snapshot_plugin)}
self.notification_endpoint = NotificationEndpoint(notification_plugins)

3
searchlight/tests/functional/test_designate_plugins.py
View File

@ -18,6 +18,7 @@ import mock
from searchlight.listener import NotificationEndpoint
from searchlight.tests import functional
from searchlight.tests.functional import test_listener
from searchlight.tests import utils
PROJECT1 = "34518c16d95e40a19b1a95c1916d8335"
@ -85,7 +86,7 @@ class TestDesignateListener(test_listener.TestSearchListenerBase):
self.addCleanup(osclient_patcher.stop)'''
notification_plugins = {
plugin.document_type: test_listener.StevedoreMock(plugin)
plugin.document_type: utils.StevedoreMock(plugin)
for plugin in (self.zones_plugin, self.recordsets_plugin)}
self.notification_endpoint = NotificationEndpoint(notification_plugins)

3
searchlight/tests/functional/test_glance_plugins.py
View File

@ -25,6 +25,7 @@ from searchlight.tests.functional import generate_load_data
from searchlight.tests.functional import mock_glance_pyclient
from searchlight.tests.functional import test_api
from searchlight.tests.functional import test_listener
from searchlight.tests import utils
member_list = 'glanceclient.v2.image_members.Controller.list'
@ -406,7 +407,7 @@ class TestGlanceListener(test_listener.TestSearchListenerBase):
self.metadefs_plugin = self.initialized_plugins['OS::Glance::Metadef']
notification_plugins = {
plugin.document_type: test_listener.StevedoreMock(plugin)
plugin.document_type: utils.StevedoreMock(plugin)
for plugin in (self.images_plugin, self.metadefs_plugin)}
self.notification_endpoint = NotificationEndpoint(notification_plugins)

7
searchlight/tests/functional/test_listener.py
View File

@ -25,13 +25,6 @@ OWNER1 = str(uuid.uuid4())
NETWORK_TENANT_ID = '8eaac046b2c44ab99246cb0850c7f06d'
class StevedoreMock(object):
"""Act like a stevedore-loaded plugin for the sake of the listener code"""
def __init__(self, plugin):
self.obj = plugin
self.name = plugin.document_type
class TestSearchListenerBase(functional.FunctionalTest):
def __init__(self, *args, **kwargs):

130
searchlight/tests/functional/test_load.py
View File

@ -13,15 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import mock
from oslo_config import cfg
import oslo_utils
from searchlight.cmd import manage
from searchlight.elasticsearch.plugins import utils as es_utils
from searchlight.elasticsearch import ROLE_USER_FIELD
from searchlight.tests import fake_plugins
from searchlight.tests import functional
from searchlight.tests import utils as test_utils
now = oslo_utils.timeutils.utcnow()
@ -118,3 +121,130 @@ class TestSearchLoad(functional.FunctionalTest):
finally:
es_utils.delete_index(index_name)
@mock.patch('oslo_utils.timeutils.utcnow')
@mock.patch('searchlight.common.utils.get_search_plugins')
def test_manage(self, mock_get_plugins, mock_utcnow):
"""Test that manage index sync works from end to end. Uses fake plugins
because it avoids having to fake service data and is less dependent
on functional tests for each service plugin.
"""
mock_utcnow.return_value = datetime.datetime(year=2016, month=1, day=1)
expected_index_name = 'searchlight-2016_01_01_00_00_00'
simple_plugin = fake_plugins.FakeSimplePlugin(self.elastic_connection)
child_plugin = fake_plugins.FakeChildPlugin(self.elastic_connection)
non_role_plugin = fake_plugins.NonRoleSeparatedPlugin(
self.elastic_connection)
child_plugin.register_parent(simple_plugin)
mock_get_plugins.return_value = {
plugin.get_document_type(): test_utils.StevedoreMock(plugin)
for plugin in (simple_plugin, child_plugin, non_role_plugin)
}
index_command = manage.IndexCommands()
# The fake plugins all have hardcoded data for get_objects that will
# be indexed. Use force=True to avoid the 'are you sure?' prompt
try:
index_command.sync(force=True)
es_results = self._get_all_elasticsearch_docs()
es_hits = self._get_hit_source(es_results)
finally:
es_utils.delete_index(expected_index_name)
self.assertEqual(expected_index_name,
es_results['hits']['hits'][0]['_index'])
expected = ['simple1', 'child1', 'non-role-fake1', 'non-role-fake2']
self.assertEqual(len(expected), len(es_hits))
self.assertEqual(
set(expected),
set(hit['_id'] for hit in es_results['hits']['hits']))
@mock.patch('oslo_utils.timeutils.utcnow')
@mock.patch('searchlight.common.utils.get_search_plugins')
def test_manage_type(self, mock_get_plugins, mock_utcnow):
mock_utcnow.return_value = datetime.datetime(year=2016, month=2, day=2)
expected_index_name = 'searchlight-2016_02_02_00_00_00'
simple_plugin = fake_plugins.FakeSimplePlugin(self.elastic_connection)
non_role_plugin = fake_plugins.NonRoleSeparatedPlugin(
self.elastic_connection)
mock_get_plugins.return_value = {
plugin.get_document_type(): test_utils.StevedoreMock(plugin)
for plugin in (simple_plugin, non_role_plugin)
}
index_command = manage.IndexCommands()
# Expect this to match simple plugin and child plugin, but not
# non_role_plugin. Patch the index->index function call since it won't
# find any data, and we want to check it's called correctly
with mock.patch.object(index_command,
'_es_reindex_worker') as patch_es_reindex:
try:
index_command.sync(_type='fake-simple', force=True)
es_results = self._get_all_elasticsearch_docs()
es_hits = self._get_hit_source(es_results)
finally:
es_utils.delete_index(expected_index_name)
patch_es_reindex.assert_called_with(
[non_role_plugin.get_document_type()],
[('searchlight', 'searchlight-search',
'searchlight-listener')],
{'searchlight': expected_index_name}
)
expected = ['simple1']
self.assertEqual(len(expected), len(es_hits))
self.assertEqual(
set(expected),
set(hit['_id'] for hit in es_results['hits']['hits']))
@mock.patch('oslo_utils.timeutils.utcnow')
@mock.patch('searchlight.common.utils.get_search_plugins')
def test_manage_type_glob(self, mock_get_plugins, mock_utcnow):
mock_utcnow.return_value = datetime.datetime(year=2016, month=3, day=3)
expected_index_name = 'searchlight-2016_03_03_00_00_00'
simple_plugin = fake_plugins.FakeSimplePlugin(self.elastic_connection)
child_plugin = fake_plugins.FakeChildPlugin(self.elastic_connection)
non_role_plugin = fake_plugins.NonRoleSeparatedPlugin(
self.elastic_connection)
child_plugin.register_parent(simple_plugin)
mock_get_plugins.return_value = {
plugin.get_document_type(): test_utils.StevedoreMock(plugin)
for plugin in (simple_plugin, child_plugin, non_role_plugin)
}
index_command = manage.IndexCommands()
# Expect this to match simple plugin and child plugin, but not
# non_role_plugin. Patch the index->index function call since it won't
# find any data, and we want to check it's called correctly
with mock.patch.object(index_command,
'_es_reindex_worker') as patch_es_reindex:
try:
# Use two wildcard matches
index_command.sync(_type='fake-sim*,fake-chi*', force=True)
es_results = self._get_all_elasticsearch_docs()
es_hits = self._get_hit_source(es_results)
finally:
es_utils.delete_index(expected_index_name)
patch_es_reindex.assert_called_with(
[non_role_plugin.get_document_type()],
[('searchlight', 'searchlight-search',
'searchlight-listener')],
{'searchlight': expected_index_name}
)
expected = ['simple1', 'child1']
self.assertEqual(len(expected), len(es_hits))
self.assertEqual(
set(expected),
set(hit['_id'] for hit in es_results['hits']['hits']))

3
searchlight/tests/functional/test_neutron_plugins.py
View File

@ -21,6 +21,7 @@ from searchlight.listener import NotificationEndpoint
from searchlight.tests import functional
from searchlight.tests.functional import test_api
from searchlight.tests.functional import test_listener
from searchlight.tests import utils
# These is in the load file
TENANT1 = "8eaac046b2c44ab99246cb0850c7f06d"
@ -299,7 +300,7 @@ class TestNeutronListeners(test_listener.TestSearchListenerBase):
self.initialized_plugins['OS::Neutron::SecurityGroup'])
notification_plugins = {
plugin.document_type: test_listener.StevedoreMock(plugin)
plugin.document_type: utils.StevedoreMock(plugin)
for plugin in (self.networks_plugin, self.ports_plugin,
self.subnets_plugin, self.routers_plugin,
self.fip_plugin, self.secgroup_plugin)}

30
searchlight/tests/unit/test_reindexing.py
View File

@ -16,6 +16,7 @@
import mock
from elasticsearch import exceptions as es_exc
from searchlight.common import utils as common_utils
from searchlight.elasticsearch.plugins import utils as plugin_utils
from searchlight.tests import utils as test_utils
@ -333,3 +334,32 @@ class TestReindexingUtils(test_utils.BaseTestCase):
mock.call(ignore=404, index='ndx2'),
mock.call(ignore=404, index='ndx3')]
mock_engine.indices.delete.assert_has_calls(calls, any_order=True)
def test_type_expansion(self):
doc_types = ['simple-plugin', 'simple-child', 'something-else']
# Don't do any expansion
self.assertEqual(doc_types,
common_utils.expand_type_matches(doc_types,
doc_types))
self.assertEqual([], common_utils.expand_type_matches([], doc_types))
# Expansion time!
self.assertEqual(
['simple-plugin', 'simple-child'],
common_utils.expand_type_matches(['simple-*'], doc_types))
self.assertEqual(
['something-else'],
common_utils.expand_type_matches(['something-*'], doc_types))
# Doesn't match. Sad!
self.assertEqual(
['does-not-match-*'],
common_utils.expand_type_matches(['does-not-match-*'], doc_types))
self.assertEqual(
doc_types,
common_utils.expand_type_matches(['simple-*', 'something-*'],
doc_types))

7
searchlight/tests/utils.py
View File

@ -580,3 +580,10 @@ class FlavorDictObj(DictObj):
d = copy.deepcopy(super(FlavorDictObj, self).to_dict())
d.pop("extra_spec", None)
return d
class StevedoreMock(object):
"""Act like a stevedore-loaded plugin"""
def __init__(self, plugin):
self.obj = plugin
self.name = plugin.document_type