diff --git a/devstack/local.conf b/devstack/local.conf index a918c8a2..110f2e9c 100644 --- a/devstack/local.conf +++ b/devstack/local.conf @@ -177,6 +177,9 @@ driver = messaging #notifications_topic = notifications #resource_group_name = searchlight +[service_credentials:nova] +compute_api_version = 2.1 + [resource_plugin:os_nova_server] enabled = True #admin_only_fields = OS-EXT-STS:vm_state diff --git a/doc/source/plugins.rst b/doc/source/plugins.rst index 6343919f..13a225ce 100644 --- a/doc/source/plugins.rst +++ b/doc/source/plugins.rst @@ -75,6 +75,9 @@ Please read the rest of the guide for detailed information.:: [resource_plugin] resource_group_name = searchlight + [service_credentials:nova] + compute_api_version = 2.1 + [resource_plugin:os_nova_server] enabled = True admin_only_fields = OS-EXT-SRV*,OS-EXT-STS:vm_state diff --git a/doc/source/plugins/nova.rst b/doc/source/plugins/nova.rst index 1dba54fb..dec32da7 100644 --- a/doc/source/plugins/nova.rst +++ b/doc/source/plugins/nova.rst @@ -39,6 +39,19 @@ general configuration information, and an example complete configuration. searchlight.conf ---------------- +Nova microversions +^^^^^^^^^^^^^^^^^^ +:: + + [service_credentials:nova] + compute_api_version = 2.1 + +.. note:: + + Nova adds/removes fields using microversion mechanism, check + http://git.openstack.org/cgit/openstack/nova/tree/nova/api/openstack/rest_api_version_history.rst + for detailed Nova microversion history. + Plugin: OS::Nova::Server ^^^^^^^^^^^^^^^^^^^^^^^^ :: diff --git a/releasenotes/notes/add-nova-microversion-support-8b40da4458678f20.yaml b/releasenotes/notes/add-nova-microversion-support-8b40da4458678f20.yaml new file mode 100644 index 00000000..830eb6d1 --- /dev/null +++ b/releasenotes/notes/add-nova-microversion-support-8b40da4458678f20.yaml @@ -0,0 +1,6 @@ +--- +features: + - Added microversion support for Nova plugins, the default + API version for Nova plugins is 2.1. User can change the + API microversion for Nova using compute_api_version config + option added in section service_credentials:nova. diff --git a/searchlight/common/exception.py b/searchlight/common/exception.py index 76216347..fa733dce 100644 --- a/searchlight/common/exception.py +++ b/searchlight/common/exception.py @@ -121,3 +121,9 @@ class InvalidJsonPatchPath(JsonPatchException): class IndexingException(SearchlightException): message = _("An error occurred during index creation or initial loading") + + +class InvalidAPIVersionProvided(SearchlightException): + message = _("The provided API version is not supported, " + "the current available version range for %(service)s " + "is: from %(min_version)s to %(max_version)s.") diff --git a/searchlight/elasticsearch/plugins/base.py b/searchlight/elasticsearch/plugins/base.py index d42a3f8f..18a9dab8 100644 --- a/searchlight/elasticsearch/plugins/base.py +++ b/searchlight/elasticsearch/plugins/base.py @@ -240,8 +240,7 @@ class IndexBase(plugin.Plugin): # See https://www.elastic.co/guide/en/elasticsearch/ # reference/2.1/mapping-meta-field.html if facet_name in meta_mapping: - facet['resource_type'] = \ - meta_mapping[facet_name]['resource_type'] + facet.update(meta_mapping[facet_name]) if (self.get_parent_id_field() and name == self.get_parent_id_field()): diff --git a/searchlight/elasticsearch/plugins/nova/servers.py b/searchlight/elasticsearch/plugins/nova/servers.py index 3f1881cf..2934e0c6 100644 --- a/searchlight/elasticsearch/plugins/nova/servers.py +++ b/searchlight/elasticsearch/plugins/nova/servers.py @@ -29,7 +29,7 @@ class ServerIndex(base.IndexBase): NotificationHandlerCls = notification_handler.InstanceHandler # Will be combined with 'admin_only_fields' from config - ADMIN_ONLY_FIELDS = ['OS-EXT-SRV-ATTR:*'] + ADMIN_ONLY_FIELDS = ['OS-EXT-SRV-ATTR:*', 'host_status'] @classmethod def get_document_type(self): @@ -95,6 +95,17 @@ class ServerIndex(base.IndexBase): # maintains compatibility with both 'security_groups': {'type': 'string', 'index': 'not_analyzed'}, 'status': {'type': 'string', 'index': 'not_analyzed'}, + # Nova adds/removes fields using microversion mechanism, check + # http://git.openstack.org/cgit/openstack/nova/tree/nova/api/openstack/rest_api_version_history.rst + # for detailed Nova microversion history. + # Added in microversion 2.9 + 'locked': {'type': 'string', 'index': 'not_analyzed'}, + # Added in microversion 2.16 + 'host_status': {'type': 'string', 'index': 'not_analyzed'}, + # Added in microversion 2.19 + 'description': {'type': 'string'}, + # Added in microversion 2.26 + 'tags': {'type': 'string'}, }, "_meta": { "image.id": { @@ -117,6 +128,18 @@ class ServerIndex(base.IndexBase): }, "security_groups": { "resource_type": resource_types.NOVA_SECURITY_GROUP + }, + "locked": { + "min_version": "2.9" + }, + "host_status": { + "min_version": "2.16" + }, + "description": { + "min_version": "2.19" + }, + "tags": { + "min_version": "2.26" } }, } @@ -131,7 +154,7 @@ class ServerIndex(base.IndexBase): return ('OS-EXT-AZ:availability_zone', 'status', 'image.id', 'flavor.id', 'networks.name', 'networks.OS-EXT-IPS:type', 'networks.version', - 'security_groups') + 'security_groups', 'host_status', 'locked') @property def facets_excluded(self): @@ -139,7 +162,7 @@ class ServerIndex(base.IndexBase): fields should not be offered as facet options, or those that should only be available to administrators. """ - return {'tenant_id': True, 'project_id': True, + return {'tenant_id': True, 'project_id': True, 'host_status': True, 'created': False, 'updated': False} def _get_rbac_field_filters(self, request_context): diff --git a/searchlight/elasticsearch/plugins/openstack_clients.py b/searchlight/elasticsearch/plugins/openstack_clients.py index 2bd273a2..b87f7321 100644 --- a/searchlight/elasticsearch/plugins/openstack_clients.py +++ b/searchlight/elasticsearch/plugins/openstack_clients.py @@ -12,6 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +from distutils.version import LooseVersion import os from cinderclient import client as cinder_client @@ -21,10 +22,11 @@ from keystoneclient import auth as ks_auth from keystoneclient import session as ks_session import keystoneclient.v2_0.client as ks_client import neutronclient.v2_0.client as neutron_client -from novaclient import api_versions from novaclient import client as nova_client import swiftclient +from searchlight.common import exception + from oslo_config import cfg @@ -38,10 +40,20 @@ client_opts = [ 'use for communication with OpenStack services.'), ] +compute_api_version = cfg.StrOpt( + 'compute_api_version', + default='2.1', + help='The compute API (micro)version, the provided ' + 'compute API (micro)version should be not smaller ' + 'than 2.1 and not larger than the max supported ' + 'Compute API microversion. The current supported ' + 'Compute API versions can be checked using: ' + 'nova version-list.') GROUP = "service_credentials" cfg.CONF.register_opts(client_opts, group=GROUP) +cfg.CONF.register_opt(compute_api_version, group="service_credentials:nova") ks_session.Session.register_conf_options(cfg.CONF, GROUP) @@ -49,6 +61,8 @@ ks_auth.register_conf_options(cfg.CONF, GROUP) _session = None +NOVA_MIN_API_VERSION = '2.1' + def _get_session(): global _session @@ -73,14 +87,26 @@ def get_glanceclient(): def get_novaclient(): - session = _get_session() - return nova_client.Client( - version=api_versions.APIVersion('2.1'), - session=session, - region_name=cfg.CONF.service_credentials.os_region_name, - endpoint_type=cfg.CONF.service_credentials.os_endpoint_type - ) + def do_get_client(api_version=2.1): + session = _get_session() + return nova_client.Client( + version=api_version, + session=session, + region_name=cfg.CONF.service_credentials.os_region_name, + endpoint_type=cfg.CONF.service_credentials.os_endpoint_type + ) + + version = cfg.CONF["service_credentials:nova"].compute_api_version + # Check whether Nova can support the provided microversion. + max_version = do_get_client().versions.list()[-1].version + if LooseVersion(version) > LooseVersion(max_version) or \ + LooseVersion(version) < LooseVersion(NOVA_MIN_API_VERSION): + raise exception.InvalidAPIVersionProvided( + service='compute service', min_version=NOVA_MIN_API_VERSION, + max_version=max_version) + + return do_get_client(version) def get_designateclient(): diff --git a/searchlight/tests/functional/test_api.py b/searchlight/tests/functional/test_api.py index 39c1ceec..b67ba7c0 100644 --- a/searchlight/tests/functional/test_api.py +++ b/searchlight/tests/functional/test_api.py @@ -14,6 +14,7 @@ # limitations under the License. import json +import mock import six import time import uuid @@ -28,6 +29,11 @@ TENANT3 = str(uuid.uuid4()) USER1 = str(uuid.uuid4()) +fake_version_list = [test_utils.FakeVersion('2.1'), + test_utils.FakeVersion('2.1')] + +nova_version_getter = 'novaclient.v2.client.versions.VersionManager.list' + MATCH_ALL = {"query": {"match_all": {}}, "sort": [{"name": {"order": "asc"}}]} EMPTY_RESPONSE = {"hits": {"hits": [], "total": 0, "max_score": 0.0}, "_shards": {"successful": 0, "failed": 0, "total": 0}, @@ -231,10 +237,12 @@ class TestSearchApi(functional.FunctionalTest): u'updated_at': u'2016-04-07T15:51:35Z', u'user_id': u'27f4d76b-be62-4e4e-aa33bb11cc55' } - self._index( - servers_plugin, - [test_utils.DictObj(**server1), test_utils.DictObj(**server2), - test_utils.DictObj(**server3)]) + with mock.patch(nova_version_getter, return_value=fake_version_list): + self._index( + servers_plugin, + [test_utils.DictObj(**server1), + test_utils.DictObj(**server2), + test_utils.DictObj(**server3)]) response, json_content = self._facet_request( TENANT1, @@ -308,9 +316,11 @@ class TestSearchApi(functional.FunctionalTest): u'user_id': u'27f4d76b-be62-4e4e-aa33bb11cc55' } - self._index( - servers_plugin, - [test_utils.DictObj(**server1), test_utils.DictObj(**server2)]) + with mock.patch(nova_version_getter, return_value=fake_version_list): + self._index( + servers_plugin, + [test_utils.DictObj(**server1), + test_utils.DictObj(**server2)]) response, json_content = self._facet_request( TENANT1, @@ -360,9 +370,10 @@ class TestSearchApi(functional.FunctionalTest): } servers_plugin = self.initialized_plugins['OS::Nova::Server'] - self._index( - servers_plugin, - [test_utils.DictObj(**s1)]) + with mock.patch(nova_version_getter, return_value=fake_version_list): + self._index( + servers_plugin, + [test_utils.DictObj(**s1)]) response, json_content = self._search_request(MATCH_ALL, TENANT1, @@ -413,9 +424,10 @@ class TestSearchApi(functional.FunctionalTest): } servers_plugin = self.initialized_plugins['OS::Nova::Server'] - self._index( - servers_plugin, - [test_utils.DictObj(**s1)]) + with mock.patch(nova_version_getter, return_value=fake_version_list): + self._index( + servers_plugin, + [test_utils.DictObj(**s1)]) # For each of these queries (which are really looking for the same # thing) we expect a result for an admin, and no result for a user @@ -484,7 +496,8 @@ class TestSearchApi(functional.FunctionalTest): "created_at": "2016-04-06T12:48:18Z" } - self._index(servers_plugin, [test_utils.DictObj(**server_doc)]) + with mock.patch(nova_version_getter, return_value=fake_version_list): + self._index(servers_plugin, [test_utils.DictObj(**server_doc)]) self._index(images_plugin, [image_doc]) # Modify the policy file to disallow some things diff --git a/searchlight/tests/functional/test_nova_plugin.py b/searchlight/tests/functional/test_nova_plugin.py index 202271da..c9aa14ad 100644 --- a/searchlight/tests/functional/test_nova_plugin.py +++ b/searchlight/tests/functional/test_nova_plugin.py @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import mock from searchlight.tests import functional from searchlight.tests import utils @@ -18,6 +19,11 @@ from searchlight.tests import utils TENANT1 = u"1816a16093df465dbc609cf638422a05" TENANT_ID = u"1dd2c5280b4e45fc9d7d08a81228c891" +fake_version_list = [utils.FakeVersion('2.1'), + utils.FakeVersion('2.1')] + +nova_version_getter = 'novaclient.v2.client.versions.VersionManager.list' + class TestNovaPlugins(functional.FunctionalTest): def setUp(self): @@ -27,7 +33,8 @@ class TestNovaPlugins(functional.FunctionalTest): self.server_plugin = self.initialized_plugins['OS::Nova::Server'] self.server_objects = self._load_fixture_data('load/servers.json') - def test_hypervisor_rbac(self): + @mock.patch(nova_version_getter, return_value=fake_version_list) + def test_hypervisor_rbac(self, mock_version): self._index(self.hyper_plugin, [utils.DictObj(**hyper) for hyper in self.hyper_objects]) response, json_content = self._search_request( @@ -88,7 +95,8 @@ class TestNovaPlugins(functional.FunctionalTest): actual_sources = [process(hit['_source']) for hit in hits] self.assertEqual(expected_sources, actual_sources) - def _index_data(self): + @mock.patch(nova_version_getter, return_value=fake_version_list) + def _index_data(self, mock_version): self._index(self.server_plugin, [utils.DictObj(**server) for server in self.server_objects] diff --git a/searchlight/tests/unit/test_nova_server_plugin.py b/searchlight/tests/unit/test_nova_server_plugin.py index 3d4dc419..dd2dba53 100644 --- a/searchlight/tests/unit/test_nova_server_plugin.py +++ b/searchlight/tests/unit/test_nova_server_plugin.py @@ -90,6 +90,10 @@ net_ip4_6 = { } ] } + +fake_version_list = [test_utils.FakeVersion('2.1'), + test_utils.FakeVersion('2.1')] + net_ipv4 = {u'net4': [dict(net_ip4_6[u'net4'][0])]} _now = datetime.datetime.utcnow() @@ -98,6 +102,7 @@ created_now = _five_minutes_ago.strftime('%Y-%m-%dT%H:%M:%SZ') updated_now = _now.strftime('%Y-%m-%dT%H:%M:%SZ') nova_server_getter = 'novaclient.v2.client.servers.ServerManager.get' +nova_version_getter = 'novaclient.v2.client.versions.VersionManager.list' def _instance_fixture(instance_id, name, tenant_id, **kwargs): @@ -197,7 +202,8 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): def test_document_type(self): self.assertEqual('OS::Nova::Server', self.plugin.get_document_type()) - def test_serialize(self): + @mock.patch(nova_version_getter, return_value=fake_version_list) + def test_serialize(self, mock_version): expected = { u'OS-DCF:diskConfig': u'MANUAL', u'OS-EXT-AZ:availability_zone': u'az1', @@ -258,7 +264,8 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): self.assertEqual(expected, serialized) - def test_serialize_no_image(self): + @mock.patch(nova_version_getter, return_value=fake_version_list) + def test_serialize_no_image(self, mock_version): instance = _instance_fixture( ID3, u'instance3', tenant_id=TENANT1, flavor=flavor1, image='', addresses=net_ipv4, @@ -341,9 +348,10 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): network_facets = ('name', 'version', 'ipv6_addr', 'ipv4_addr', 'OS-EXT-IPS-MAC:mac_addr', 'OS-EXT-IPS:type') expected_facet_names = [ - 'OS-EXT-AZ:availability_zone', 'created_at', 'flavor.id', 'id', - 'image.id', 'name', 'owner', 'security_groups', 'status', - 'updated_at', 'user_id'] + 'OS-EXT-AZ:availability_zone', 'created_at', 'description', + 'flavor.id', 'id', 'image.id', 'locked', 'name', + 'owner', 'security_groups', 'status', 'tags', 'updated_at', + 'user_id'] expected_facet_names.extend(['networks.' + f for f in network_facets]) self.assertEqual(set(expected_facet_names), set(facet_names)) @@ -356,7 +364,8 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): for name in complex_facet_option_fields) simple_facet_option_fields = ( - 'status', 'OS-EXT-AZ:availability_zone', 'security_groups' + 'status', 'OS-EXT-AZ:availability_zone', 'security_groups', + 'locked' ) aggs.update(dict(unit_test_utils.simple_facet_field_agg(name) for name in simple_facet_option_fields)) @@ -397,9 +406,10 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): network_facets = ('name', 'version', 'ipv6_addr', 'ipv4_addr', 'OS-EXT-IPS-MAC:mac_addr', 'OS-EXT-IPS:type') expected_facet_names = [ - 'OS-EXT-AZ:availability_zone', 'created_at', 'flavor.id', 'id', - 'image.id', 'name', 'owner', 'project_id', 'security_groups', - 'status', 'tenant_id', 'updated_at', 'user_id'] + 'OS-EXT-AZ:availability_zone', 'created_at', 'description', + 'flavor.id', 'host_status', 'id', 'image.id', 'locked', + 'name', 'owner', 'project_id', 'security_groups', 'status', + 'tags', 'tenant_id', 'updated_at', 'user_id'] expected_facet_names.extend(['networks.' + f for f in network_facets]) self.assertEqual(set(expected_facet_names), set(facet_names)) @@ -411,7 +421,8 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): for name in complex_facet_option_fields) simple_facet_option_fields = ( - 'status', 'OS-EXT-AZ:availability_zone', 'security_groups' + 'status', 'OS-EXT-AZ:availability_zone', 'security_groups', + 'host_status', 'locked' ) aggs.update(dict(unit_test_utils.simple_facet_field_agg(name) for name in simple_facet_option_fields)) @@ -514,7 +525,8 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): self.assertEqual(expected_status, status_facet) self.assertEqual(expected_image, image_facet) - def test_created_at_updated_at(self): + @mock.patch(nova_version_getter, return_value=fake_version_list) + def test_created_at_updated_at(self, mock_version): self.assertTrue('created_at' not in self.instance1.to_dict()) self.assertTrue('updated_at' not in self.instance1.to_dict()) @@ -524,7 +536,8 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): self.assertEqual(serialized['created_at'], created_now) self.assertEqual(serialized['updated_at'], updated_now) - def test_update_404_deletes(self): + @mock.patch(nova_version_getter, return_value=fake_version_list) + def test_update_404_deletes(self, mock_version): """Test that if a server is missing on a notification event, it gets deleted from the index """ @@ -570,7 +583,8 @@ class TestServerLoaderPlugin(test_utils.BaseTestCase): mock_update.assert_called_with(vol_payload, "a", 1234) - def test_filter_result(self): + @mock.patch(nova_version_getter, return_value=fake_version_list) + def test_filter_result(self, mock_version): """We reformat outgoing results so that security group looks like the response we get from the nova API. """ diff --git a/searchlight/tests/utils.py b/searchlight/tests/utils.py index fe01c6eb..775121f1 100644 --- a/searchlight/tests/utils.py +++ b/searchlight/tests/utils.py @@ -561,3 +561,8 @@ class DictObj(object): def to_dict(self): return self.__dict__ + + +class FakeVersion(object): + def __init__(self, version): + self.version = version