From 879d2a6a6a87d7f1fdda30286af0a0fadc8dc5c8 Mon Sep 17 00:00:00 2001 From: Vitaly Gridnev Date: Thu, 23 Jun 2016 17:17:59 +0300 Subject: [PATCH] plugins api impl things implemented in this change: * api for updating labels * exposing labels data * unit tests coming soon: * validations based on labels Change-Id: If854e812492cef1b1d32ae01f74e6b32cf456ee3 bp: plugin-management-api --- sahara/api/v10.py | 8 + sahara/api/v2/plugins.py | 9 + sahara/config.py | 2 +- sahara/plugins/base.py | 25 ++- sahara/plugins/fake/plugin.py | 11 + sahara/plugins/labels.py | 194 +++++++++++++++++ sahara/plugins/opts.py | 26 +++ sahara/plugins/provisioning.py | 12 ++ sahara/service/api/v10.py | 4 + sahara/service/api/v2/plugins.py | 4 + sahara/service/validations/plugins.py | 9 + sahara/tests/unit/plugins/test_labels.py | 241 ++++++++++++++++++++++ sahara/tests/unit/service/api/test_v10.py | 13 +- 13 files changed, 542 insertions(+), 16 deletions(-) create mode 100644 sahara/plugins/labels.py create mode 100644 sahara/plugins/opts.py create mode 100644 sahara/tests/unit/plugins/test_labels.py diff --git a/sahara/api/v10.py b/sahara/api/v10.py index bc0085b6..4da06115 100644 --- a/sahara/api/v10.py +++ b/sahara/api/v10.py @@ -207,6 +207,14 @@ def plugins_get_version(plugin_name, version): return u.render(api.get_plugin(plugin_name, version).wrapped_dict) +@rest.patch('/plugins/') +@acl.enforce("data-processing:plugins:patch") +@v.check_exists(api.get_plugin, plugin_name='plugin_name') +@v.validate(v_p.plugin_update_validation_jsonschema(), v_p.check_plugin_update) +def plugins_update(plugin_name, data): + return u.render(api.update_plugin(plugin_name, data).wrapped_dict) + + @rest.post_file('/plugins///convert-config/') @acl.enforce("data-processing:plugins:convert_config") @v.check_exists(api.get_plugin, plugin_name='plugin_name', version='version') diff --git a/sahara/api/v2/plugins.py b/sahara/api/v2/plugins.py index b9ab2172..f6f9434c 100644 --- a/sahara/api/v2/plugins.py +++ b/sahara/api/v2/plugins.py @@ -16,6 +16,7 @@ from sahara.api import acl from sahara.service.api.v2 import plugins as api from sahara.service import validation as v +from sahara.service.validations import plugins as v_p import sahara.utils.api as u @@ -40,3 +41,11 @@ def plugins_get(plugin_name): @v.check_exists(api.get_plugin, plugin_name='plugin_name', version='version') def plugins_get_version(plugin_name, version): return u.render(api.get_plugin(plugin_name, version).wrapped_dict) + + +@rest.patch('/plugins/') +@acl.enforce("data-processing:plugins:patch") +@v.check_exists(api.get_plugin, plugin_name='plugin_name') +@v.validate(v_p.plugin_update_validation_jsonschema(), v_p.check_plugin_update) +def plugins_update(plugin_name, data): + return u.render(api.update_plugin(plugin_name, data).wrapped_dict) diff --git a/sahara/config.py b/sahara/config.py index d0644292..39f61b96 100644 --- a/sahara/config.py +++ b/sahara/config.py @@ -22,7 +22,7 @@ from oslo_log import log from sahara import exceptions as ex from sahara.i18n import _ -from sahara.plugins import base as plugins_base +from sahara.plugins import opts as plugins_base from sahara.service.castellan import config as castellan from sahara.topology import topology_helper from sahara.utils.notification import sender diff --git a/sahara/plugins/base.py b/sahara/plugins/base.py index 8cc7889e..054e2b15 100644 --- a/sahara/plugins/base.py +++ b/sahara/plugins/base.py @@ -20,23 +20,17 @@ from oslo_log import log as logging import six from stevedore import enabled +from sahara import conductor as cond from sahara import exceptions as ex from sahara.i18n import _ from sahara.i18n import _LI +from sahara.plugins import labels from sahara.utils import resources +conductor = cond.API LOG = logging.getLogger(__name__) - -opts = [ - cfg.ListOpt('plugins', - default=['vanilla', 'spark', 'cdh', 'ambari'], - help='List of plugins to be loaded. Sahara preserves the ' - 'order of the list when returning it.'), -] - CONF = cfg.CONF -CONF.register_opts(opts) def required(fun): @@ -87,7 +81,9 @@ class PluginInterface(resources.BaseResource): class PluginManager(object): def __init__(self): self.plugins = {} + self.default_label_schema = {} self._load_cluster_plugins() + self.label_handler = labels.LabelHandler(self.plugins) def _load_cluster_plugins(self): config_plugins = CONF.plugins @@ -134,6 +130,8 @@ class PluginManager(object): plugin = self.get_plugin(plugin_name) if plugin: res = plugin.as_resource() + res._info.update(self.label_handler.get_label_full_details( + plugin_name)) if version: if version in plugin.get_versions(): res._info.update(plugin.get_version_details(version)) @@ -141,6 +139,15 @@ class PluginManager(object): return None return res + def update_plugin(self, plugin_name, values): + self.label_handler.update_plugin(plugin_name, values) + return self.serialize_plugin(plugin_name) + + def validate_plugin_update(self, plugin_name, values): + return self.label_handler.validate_plugin_update(plugin_name, values) + + def get_plugin_update_validation_jsonschema(self): + return self.label_handler.get_plugin_update_validation_jsonschema() PLUGINS = None diff --git a/sahara/plugins/fake/plugin.py b/sahara/plugins/fake/plugin.py index 3e77441e..0915df25 100644 --- a/sahara/plugins/fake/plugin.py +++ b/sahara/plugins/fake/plugin.py @@ -34,6 +34,17 @@ class FakePluginProvider(p.ProvisioningPluginBase): def get_versions(self): return ["0.1"] + def get_labels(self): + return { + 'plugin_labels': { + 'enabled': {'status': True}, + 'hidden': {'status': True}, + }, + 'version_labels': { + '0.1': {'enabled': {'status': True}} + } + } + def get_node_processes(self, hadoop_version): return { "HDFS": ["namenode", "datanode"], diff --git a/sahara/plugins/labels.py b/sahara/plugins/labels.py new file mode 100644 index 00000000..38163e60 --- /dev/null +++ b/sahara/plugins/labels.py @@ -0,0 +1,194 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from sahara import conductor as cond +from sahara import context +from sahara import exceptions as ex +from sahara.i18n import _ + +conductor = cond.API + +STABLE = { + 'name': 'stable', + 'mutable': False, + 'description': "Indicates that plugin or its version are stable to be used" +} + +DEPRECATED = { + 'name': 'deprecated', + 'mutable': False, + 'description': "Plugin or its version is deprecated and will be removed " + "in future releases. Please, consider to use another " + "plugin or its version." +} + +ENABLED = { + 'name': 'enabled', + 'mutable': True, + 'description': "Plugin or its version is enabled and can be used by user." +} + +HIDDEN = { + 'name': 'hidden', + 'mutable': True, + 'description': "Existence of plugin or its version is hidden, but " + "still can be used for cluster creation by CLI and " + "directly by client." +} + +PLUGIN_LABELS_SCOPE = 'plugin_labels' +VERSION_LABELS_SCOPE = 'version_labels' +MUTABLE = 'mutable' + +LABEL_OBJECT = { + 'type': 'object', + 'properties': { + 'status': { + 'type': 'boolean', + } + }, + "additionalProperties": False, +} + + +class LabelHandler(object): + def __init__(self, loaded_plugins): + self.plugins = loaded_plugins + + def get_plugin_update_validation_jsonschema(self): + schema = { + 'type': 'object', "additionalProperties": False, + 'properties': { + VERSION_LABELS_SCOPE: { + 'type': 'object', 'additionalProperties': False, + }, + }, + } + ln = [label['name'] for label in self.get_labels()] + labels_descr_object = { + 'type': 'object', + "properties": {name: copy.deepcopy(LABEL_OBJECT) for name in ln}, + "additionalProperties": False + } + schema['properties'][PLUGIN_LABELS_SCOPE] = copy.deepcopy( + labels_descr_object) + all_versions = [] + for plugin_name in self.plugins.keys(): + plugin = self.plugins[plugin_name] + all_versions.extend(plugin.get_versions()) + all_versions = set(all_versions) + schema['properties'][VERSION_LABELS_SCOPE]['properties'] = { + ver: copy.deepcopy(labels_descr_object) for ver in all_versions + } + return schema + + def get_default_label_details(self, plugin_name): + plugin = self.plugins.get(plugin_name) + return plugin.get_labels() + + def get_label_details(self, plugin_name): + plugin = conductor.plugin_get(context.ctx(), plugin_name) + if not plugin: + plugin = self.get_default_label_details(plugin_name) + # keep only tenant + fields = ['name', 'id', 'updated_at', 'created_at'] + for field in fields: + if field in plugin: + del plugin[field] + return plugin + + def get_label_full_details(self, plugin_name): + return self.expand_data(self.get_label_details(plugin_name)) + + def get_labels(self): + return [HIDDEN, STABLE, ENABLED, DEPRECATED] + + def get_labels_map(self): + return { + label['name']: label for label in self.get_labels() + } + + def expand_data(self, plugin): + plugin_labels = plugin.get(PLUGIN_LABELS_SCOPE) + labels_map = self.get_labels_map() + for key in plugin_labels.keys(): + key_desc = labels_map.get(key) + plugin_labels[key].update(key_desc) + del plugin_labels[key]['name'] + + for version in plugin.get(VERSION_LABELS_SCOPE): + vers_labels = plugin.get(VERSION_LABELS_SCOPE).get(version) + for key in vers_labels.keys(): + key_desc = labels_map.get(key) + vers_labels[key].update(key_desc) + del vers_labels[key]['name'] + + return plugin + + def _validate_labels_update(self, default_data, update_values): + for label in update_values.keys(): + if label not in default_data.keys(): + raise ex.InvalidDataException( + _("Label '%s' can't be updated because it's not " + "available for plugin or its version") % label) + if not default_data[label][MUTABLE]: + raise ex.InvalidDataException( + _("Label '%s' can't be updated because it's not " + "mutable") % label) + + def validate_plugin_update(self, plugin_name, values): + plugin = self.plugins[plugin_name] + # it's important to get full details since we have mutability + default = self.get_label_full_details(plugin_name) + if values.get(PLUGIN_LABELS_SCOPE): + pl = values.get(PLUGIN_LABELS_SCOPE) + self._validate_labels_update(default[PLUGIN_LABELS_SCOPE], pl) + + if values.get(VERSION_LABELS_SCOPE): + vl = values.get(VERSION_LABELS_SCOPE) + for version in vl.keys(): + if version not in plugin.get_versions(): + raise ex.InvalidDataException( + _("Unknown plugin version '%(version)s' of " + "%(plugin)s") % { + 'version': version, 'plugin': plugin_name}) + self._validate_labels_update( + default[VERSION_LABELS_SCOPE][version], vl[version]) + + def update_plugin(self, plugin_name, values): + ctx = context.ctx() + current = self.get_label_details(plugin_name) + if not conductor.plugin_get(ctx, plugin_name): + current['name'] = plugin_name + conductor.plugin_create(ctx, current) + del current['name'] + + if values.get(PLUGIN_LABELS_SCOPE): + for label in values.get(PLUGIN_LABELS_SCOPE).keys(): + current[PLUGIN_LABELS_SCOPE][label].update( + values.get(PLUGIN_LABELS_SCOPE).get(label)) + else: + del current[PLUGIN_LABELS_SCOPE] + + if values.get(VERSION_LABELS_SCOPE): + vl = values.get(VERSION_LABELS_SCOPE) + for version in vl.keys(): + for label in vl.get(version).keys(): + current[VERSION_LABELS_SCOPE][version][label].update( + vl[version][label]) + else: + del current[VERSION_LABELS_SCOPE] + + conductor.plugin_update(context.ctx(), plugin_name, current) diff --git a/sahara/plugins/opts.py b/sahara/plugins/opts.py new file mode 100644 index 00000000..01ce02b1 --- /dev/null +++ b/sahara/plugins/opts.py @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File contains plugins opts to avoid cyclic imports issue + +from oslo_config import cfg + +opts = [ + cfg.ListOpt('plugins', + default=['vanilla', 'spark', 'cdh', 'ambari'], + help='List of plugins to be loaded. Sahara preserves the ' + 'order of the list when returning it.'), +] + +CONF = cfg.CONF +CONF.register_opts(opts) diff --git a/sahara/plugins/provisioning.py b/sahara/plugins/provisioning.py index 0f6706fd..dac28cf4 100644 --- a/sahara/plugins/provisioning.py +++ b/sahara/plugins/provisioning.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy from sahara import exceptions as ex from sahara.i18n import _ @@ -29,6 +30,17 @@ class ProvisioningPluginBase(plugins_base.PluginInterface): def get_configs(self, hadoop_version): pass + @plugins_base.required_with_default + def get_labels(self): + versions = self.get_versions() + default = {'enabled': {'status': True}} + return { + 'plugin_labels': copy.deepcopy(default), + 'version_labels': { + version: copy.deepcopy(default) for version in versions + } + } + @plugins_base.required def get_node_processes(self, hadoop_version): pass diff --git a/sahara/service/api/v10.py b/sahara/service/api/v10.py index db22938e..31001773 100644 --- a/sahara/service/api/v10.py +++ b/sahara/service/api/v10.py @@ -225,6 +225,10 @@ def get_plugin(plugin_name, version=None): return plugin_base.PLUGINS.serialize_plugin(plugin_name, version) +def update_plugin(plugin_name, values): + return plugin_base.PLUGINS.update_plugin(plugin_name, values) + + def construct_ngs_for_scaling(cluster, additional_node_groups): ctx = context.ctx() additional = {} diff --git a/sahara/service/api/v2/plugins.py b/sahara/service/api/v2/plugins.py index 8fdf62ec..041307d0 100644 --- a/sahara/service/api/v2/plugins.py +++ b/sahara/service/api/v2/plugins.py @@ -31,6 +31,10 @@ def get_plugin(plugin_name, version=None): return plugin_base.PLUGINS.serialize_plugin(plugin_name, version) +def update_plugin(plugin_name, values): + return plugin_base.PLUGINS.update_plugin(plugin_name, values) + + def construct_ngs_for_scaling(cluster, additional_node_groups): ctx = context.ctx() additional = {} diff --git a/sahara/service/validations/plugins.py b/sahara/service/validations/plugins.py index a6c78c06..145c7aef 100644 --- a/sahara/service/validations/plugins.py +++ b/sahara/service/validations/plugins.py @@ -15,9 +15,18 @@ import sahara.exceptions as ex from sahara.i18n import _ +from sahara.plugins import base + + +def plugin_update_validation_jsonschema(): + return base.PLUGINS.get_plugin_update_validation_jsonschema() def check_convert_to_template(plugin_name, version, **kwargs): raise ex.InvalidReferenceException( _("Requested plugin '%s' doesn't support converting config files " "to cluster templates") % plugin_name) + + +def check_plugin_update(plugin_name, values, **kwargs): + base.PLUGINS.validate_plugin_update(plugin_name, values) diff --git a/sahara/tests/unit/plugins/test_labels.py b/sahara/tests/unit/plugins/test_labels.py new file mode 100644 index 00000000..d09cf340 --- /dev/null +++ b/sahara/tests/unit/plugins/test_labels.py @@ -0,0 +1,241 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import jsonschema.exceptions as json_exc +import testtools + +from sahara import conductor as cond +from sahara import exceptions as ex +from sahara.plugins import base +from sahara.tests.unit import base as unit_base +from sahara.utils import api_validator + +conductor = cond.API + +EXPECTED_SCHEMA = { + "type": "object", + "additionalProperties": False, + "properties": { + "plugin_labels": { + "type": "object", + "additionalProperties": False, + "properties": { + "hidden": { + "type": "object", + "additionalProperties": False, + "properties": { + "status": { + "type": "boolean" + } + } + }, + "stable": { + "type": "object", + "additionalProperties": False, + "properties": { + "status": { + "type": "boolean" + } + } + }, + "enabled": { + "type": "object", + "additionalProperties": False, + "properties": { + "status": { + "type": "boolean" + } + } + }, + "deprecated": { + "type": "object", + "additionalProperties": False, + "properties": { + "status": { + "type": "boolean" + } + } + } + } + }, + "version_labels": { + "type": "object", + "additionalProperties": False, + "properties": { + "0.1": { + "type": "object", + "additionalProperties": False, + "properties": { + "hidden": { + "type": "object", + "additionalProperties": False, + "properties": { + "status": { + "type": "boolean" + } + } + }, + "stable": { + "type": "object", + "additionalProperties": False, + "properties": { + "status": { + "type": "boolean" + } + } + }, + "enabled": { + "type": "object", + "additionalProperties": False, + "properties": { + "status": { + "type": "boolean" + } + } + }, + "deprecated": { + "type": "object", + "additionalProperties": False, + "properties": { + "status": { + "type": "boolean" + } + } + } + } + } + } + }, + } +} + + +class TestPluginLabels(unit_base.SaharaWithDbTestCase): + def test_validate_default_labels_load(self): + all_plugins = ['cdh', 'ambari', 'fake', 'storm', 'mapr', 'spark', + 'vanilla'] + self.override_config('plugins', all_plugins) + manager = base.PluginManager() + for plugin in all_plugins: + data = manager.label_handler.get_label_details(plugin) + self.assertIsNotNone(data) + # order doesn't play a role + self.assertIsNotNone(data['plugin_labels']) + self.assertEqual( + sorted(list(manager.get_plugin(plugin).get_versions())), + sorted(list(data.get('version_labels').keys()))) + + def test_get_label_full_details(self): + self.override_config('plugins', ['fake']) + lh = base.PluginManager().label_handler + + result = lh.get_label_full_details('fake') + self.assertIsNotNone(result.get('plugin_labels')) + self.assertIsNotNone(result.get('version_labels')) + pl = result.get('plugin_labels') + + self.assertEqual( + ['enabled', 'hidden'], + sorted(list(pl.keys())) + ) + for lb in ['hidden', 'enabled']: + self.assertEqual( + ['description', 'mutable', 'status'], + sorted(list(pl[lb])) + ) + vl = result.get('version_labels') + self.assertEqual(['0.1'], list(vl.keys())) + vl = vl.get('0.1') + + self.assertEqual( + ['enabled'], list(vl.keys())) + + self.assertEqual( + ['description', 'mutable', 'status'], + sorted(list(vl['enabled'])) + ) + + def test_validate_plugin_update(self): + def validate(plugin_name, values, validator, lh): + validator.validate(values) + lh.validate_plugin_update(plugin_name, values) + + values = {'plugin_labels': {'enabled': {'status': False}}} + self.override_config('plugins', ['fake', 'spark']) + lh = base.PluginManager() + validator = api_validator.ApiValidator( + lh.get_plugin_update_validation_jsonschema()) + validate('fake', values, validator, lh) + values = {'plugin_labels': {'not_exists': {'status': False}}} + + with testtools.ExpectedException(json_exc.ValidationError): + validate('fake', values, validator, lh) + + values = {'plugin_labels': {'enabled': {'status': 'False'}}} + with testtools.ExpectedException(json_exc.ValidationError): + validate('fake', values, validator, lh) + + values = {'field': {'blala': 'blalalalalala'}} + + with testtools.ExpectedException(json_exc.ValidationError): + validate('fake', values, validator, lh) + + values = {'plugin_labels': {'hidden': {'status': True}}} + + with testtools.ExpectedException(ex.InvalidDataException): + # valid under schema, but not valid under validator + # hidden is not available to spark + validate('spark', values, validator, lh) + + values = {'plugin_labels': {'enabled': {'mutable': False}}} + with testtools.ExpectedException(json_exc.ValidationError): + validate('spark', values, validator, lh) + + values = {'version_labels': {'enabled': {'status': False}}} + with testtools.ExpectedException(json_exc.ValidationError): + validate('spark', values, validator, lh) + + values = {'version_labels': {'0.1': {'enabled': {'status': False}}}} + validate('fake', values, validator, lh) + + values = {'version_labels': {'0.1': {'enabled': {'status': False}}}} + with testtools.ExpectedException(ex.InvalidDataException): + validate('spark', values, validator, lh) + + values = {'version_labels': {'0.1': {'hidden': {'status': True}}}} + with testtools.ExpectedException(ex.InvalidDataException): + validate('fake', values, validator, lh) + + def test_jsonschema(self): + self.override_config('plugins', ['fake']) + lh = base.PluginManager() + schema = lh.get_plugin_update_validation_jsonschema() + self.assertEqual(EXPECTED_SCHEMA, schema) + + def test_update(self): + self.override_config('plugins', ['fake']) + lh = base.PluginManager() + + data = lh.update_plugin('fake', values={ + 'plugin_labels': {'enabled': {'status': False}}}).dict + + # enabled is updated, but hidden still same + self.assertFalse(data['plugin_labels']['enabled']['status']) + self.assertTrue(data['plugin_labels']['hidden']['status']) + + data = lh.update_plugin('fake', values={ + 'version_labels': {'0.1': {'enabled': {'status': False}}}}).dict + + self.assertFalse(data['plugin_labels']['enabled']['status']) + self.assertTrue(data['plugin_labels']['hidden']['status']) + self.assertFalse(data['version_labels']['0.1']['enabled']['status']) diff --git a/sahara/tests/unit/service/api/test_v10.py b/sahara/tests/unit/service/api/test_v10.py index f3347025..1af63084 100644 --- a/sahara/tests/unit/service/api/test_v10.py +++ b/sahara/tests/unit/service/api/test_v10.py @@ -135,12 +135,8 @@ class FakePlugin(pr_base.ProvisioningPluginBase): class FakePluginManager(pl_base.PluginManager): def __init__(self, calls_order): - self.calls = calls_order - - def get_plugin(self, plugin_name): - if plugin_name == "fake": - return FakePlugin(self.calls) - return None + super(FakePluginManager, self).__init__() + self.plugins['fake'] = FakePlugin(calls_order) class FakeOps(object): @@ -296,6 +292,11 @@ class TestApi(base.SaharaWithDbTestCase): self.assertIsNone(api.get_plugin('fake', '0.3')) data = api.get_plugin('fake').dict + self.assertIsNotNone(data.get('version_labels')) + self.assertIsNotNone(data.get('plugin_labels')) + del data['plugin_labels'] + del data['version_labels'] + self.assertEqual({ 'description': "Some description", 'name': 'fake',