openstack/fuel-web
Code Issues Proposed changes

[Nailgun] Plugin Sync API

Browse Source 
Current commit provide implementation of API call for plugin
metadata synchronization from config files into DB

* New PluginSync handler, validator and new url
* New plugin wrapper for 3.0.0 plugin version with
  sync metadata method

Change-Id: I223ec14afbfa0852f231ad8dd2c5289758997b0d
Implements: blueprint role-as-a-plugin
This commit is contained in:
Andriy Popovych 2015-06-25 10:42:11 +03:00 committed by Sebastian Kalinowski
parent 502eb7c5d4
commit ac96a16bbb
12 changed files with 398 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
nailgun/nailgun/api/v1/handlers/plugin.py
View File

@ -14,10 +14,14 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
from nailgun.api.v1.handlers import base
from nailgun.api.v1.handlers.base import content
from nailgun.api.v1.validators import plugin
from nailgun.errors import errors
from nailgun import objects
from nailgun.plugins.manager import PluginManager
class PluginHandler(base.SingleHandler):
@ -44,3 +48,25 @@ class PluginCollectionHandler(base.CollectionHandler):
if obj:
raise self.http(409, self.collection.single.to_json(obj))
return super(PluginCollectionHandler, self).POST()
class PluginSyncHandler(base.BaseHandler):
validator = plugin.PluginSyncValidator
@content
def POST(self):
""":returns: JSONized REST object.
:http: * 200 (plugins successfully synced)
* 404 (plugin not found in db)
* 400 (problem with parsing metadata file)
"""
data = self.checked_data()
ids = data.get('ids', None)
try:
PluginManager.sync_plugins_metadata(plugin_ids=ids)
except errors.ParseError as exc:
raise self.http(400, msg=six.text_type(exc))
raise self.http(200)

3
nailgun/nailgun/api/v1/urls.py
View File

@ -62,6 +62,7 @@ from nailgun.api.v1.handlers.node import NodesAllocationStatsHandler
from nailgun.api.v1.handlers.plugin import PluginCollectionHandler
from nailgun.api.v1.handlers.plugin import PluginHandler
from nailgun.api.v1.handlers.plugin import PluginSyncHandler
from nailgun.api.v1.handlers.node import NodeCollectionNICsDefaultHandler
from nailgun.api.v1.handlers.node import NodeCollectionNICsHandler
@ -215,6 +216,8 @@ urls = (
PluginHandler,
r'/plugins/?$',
PluginCollectionHandler,
r'/plugins/sync/?$',
PluginSyncHandler,
r'/notifications/?$',
NotificationCollectionHandler,

13
nailgun/nailgun/api/v1/validators/json_schema/plugin.py
View File

@ -27,6 +27,7 @@ PLUGIN_RELEASE_SCHEMA = {
'required': ['version', 'os', 'mode']
}
PLUGIN_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'title': 'plugin',
@ -75,3 +76,15 @@ TASK_SCHEMA = {
}
}
}
SYNC_SCHEMA = {
'type': 'object',
'properties': {
'ids': {
'type': 'array',
'items': {'type': 'integer'}
}
},
'required': ['ids']
}

21
nailgun/nailgun/api/v1/validators/plugin.py
View File

@ -15,9 +15,9 @@
from nailgun.api.v1.validators.base import BasicValidator
from nailgun.errors import errors
from nailgun.api.v1.validators.json_schema import plugin
from nailgun.errors import errors
from nailgun.objects import Plugin
class PluginValidator(BasicValidator):
@ -43,3 +43,20 @@ class PluginValidator(BasicValidator):
@classmethod
def validate_create(cls, data):
return cls.validate(data)
class PluginSyncValidator(BasicValidator):
@classmethod
def validate(cls, data):
if data:
parsed = super(PluginSyncValidator, cls).validate(data)
cls.validate_schema(parsed, plugin.SYNC_SCHEMA)
# Check plugin with given id exists in DB
# otherwise raise ObjectNotFound exception
for plugin_id in parsed.get('ids'):
Plugin.get_by_uid(plugin_id, fail_if_not_found=True)
return parsed
else:
return {}

11
nailgun/nailgun/objects/plugin.py
View File

@ -64,3 +64,14 @@ class PluginCollection(base.NailgunCollection):
newest_plugins.append(newest_plugin)
return newest_plugins
@classmethod
def get_by_uids(cls, plugin_ids):
"""Returns plugins by given ids.
:param plugin_ids: list of plugin ids
:type plugin_ids: list
:returns: iterable (SQLAlchemy query)
"""
return cls.filter_by_id_list(
cls.all(), plugin_ids)

52
nailgun/nailgun/plugins/attr_plugin.py
View File

@ -24,6 +24,7 @@ import yaml
from nailgun.errors import errors
from nailgun.logger import logger
from nailgun.objects.plugin import Plugin
from nailgun.settings import settings
@ -38,6 +39,7 @@ class ClusterAttributesPluginBase(object):
"""
environment_config_name = 'environment_config.yaml'
plugin_metadata = 'metadata.yaml'
task_config_name = 'tasks.yaml'
def __init__(self, plugin):
@ -56,10 +58,24 @@ class ClusterAttributesPluginBase(object):
plugin related scripts and repositories
"""
def sync_metadata_to_db(self):
"""Sync metadata from config yaml files into DB
"""
metadata_file_path = os.path.join(
self.plugin_path, self.plugin_metadata)
metadata = self._load_config(metadata_file_path) or {}
Plugin.update(self.plugin, metadata)
def _load_config(self, config):
if os.access(config, os.R_OK):
with open(config, "r") as conf:
return yaml.load(conf.read())
try:
return yaml.safe_load(conf.read())
except yaml.YAMLError as exc:
logger.warning(exc)
raise errors.ParseError(
'Problem with loading YAML file {0}'.format(config))
else:
logger.warning("Config {0} is not readable.".format(config))
@ -263,9 +279,41 @@ class ClusterAttributesPluginV2(ClusterAttributesPluginBase):
return major
class PluginAdapterV3(ClusterAttributesPluginV2):
"""Plugin wrapper class for package version >= 3.0.0
"""
node_roles_config_name = 'node_roles.yaml'
volumes_config_name = 'volumes.yaml'
deployment_tasks_config_name = 'deployment_tasks.yaml'
def sync_metadata_to_db(self):
super(PluginAdapterV3, self).sync_metadata_to_db()
data_to_update = {}
db_config_metadata_mapping = {
'attributes_metadata': self.environment_config_name,
'roles_metadata': self.node_roles_config_name,
'volumes_metadata': self.volumes_config_name,
'deployment_tasks': self.deployment_tasks_config_name,
'tasks': self.task_config_name
}
for attribute, config in six.iteritems(db_config_metadata_mapping):
config_file_path = os.path.join(self.plugin_path, config)
attribute_data = self._load_config(config_file_path)
# Plugin columns have constraints for nullable data, so
# we need to check it
if attribute_data:
data_to_update[attribute] = attribute_data
Plugin.update(self.plugin, data_to_update)
__version_mapping = {
'1.0.': ClusterAttributesPluginV1,
'2.0.': ClusterAttributesPluginV2
'2.0.': ClusterAttributesPluginV2,
'3.0.': PluginAdapterV3
}

14
nailgun/nailgun/plugins/manager.py
View File

@ -79,3 +79,17 @@ class PluginManager(object):
attr_pl.set_cluster_tasks(cluster)
attr_plugins.append(attr_pl)
return attr_plugins
@classmethod
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given ids. If there is not
ids all newest plugins will be synced
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all_newest()
for plugin in plugins:
plugin_wrapper = wrap_plugin(plugin)
plugin_wrapper.sync_metadata_to_db()

62
nailgun/nailgun/test/base.py
View File

@ -530,6 +530,68 @@ class EnvironmentManager(object):
'weight': kwargs.get('weight', 25),
'label': kwargs.get('label', 'label')}}}
def get_default_plugin_node_roles_config(self, **kwargs):
node_roles = {
'test_node_role': {
'metadata': {
'name': 'Some plugin role',
'description': 'Some description'
}
}
}
node_roles.update(kwargs)
return node_roles
def get_default_plugin_volumes_config(self, **kwargs):
volumes = {
'volumes': [
{
'id': 'test_node_volume',
'type': 'vg'
}
]
}
volumes.update(kwargs)
return volumes
def get_default_plugin_deployment_tasks(self, **kwargs):
deployment_tasks = [
{
'id': 'role-name',
'type': 'group',
'role': '[role-name]',
'requires': '[controller]',
'required_for': '[deploy_end]',
'parameters': {
'strategy': {
'type': 'parallel'
}
}
}
]
deployment_tasks[0].update(kwargs)
return deployment_tasks
def get_default_plugin_tasks(self, **kwargs):
default_tasks = [
{
'role': '[test_role]',
'stage': 'post_deployment',
'type': 'puppet',
'parameters': {
'puppet_manifest': '/etc/puppet/modules/test_manigest.pp',
'puppet_modules': '/etc/puppet/modules',
'timeout': 720
}
}
]
default_tasks[0].update(kwargs)
return default_tasks
def get_default_plugin_metadata(self, **kwargs):
sample_plugin = {
'version': '0.1.0',

116
nailgun/nailgun/test/integration/test_plugins_api.py
View File

@ -134,6 +134,18 @@ class BasePluginTest(base.BaseIntegrationTest):
headers=self.default_headers)
return resp
def sync_plugins(self, params=None, expect_errors=False):
post_data = jsonutils.dumps(params) if params else ''
resp = self.app.post(
base.reverse('PluginSyncHandler'),
post_data,
headers=self.default_headers,
expect_errors=expect_errors
)
return resp
class TestPluginsApi(BasePluginTest):
@ -219,6 +231,93 @@ class TestPluginsApi(BasePluginTest):
self.disable_plugin(cluster, 'multiversion_plugin')
self.assertEqual(len(cluster.plugins), 0)
def test_sync_all_plugins(self):
self._create_new_and_old_version_plugins_for_sync()
resp = self.sync_plugins()
self.assertEqual(resp.status_code, 200)
def test_sync_specific_plugins(self):
plugin_ids = self._create_new_and_old_version_plugins_for_sync()
ids = plugin_ids[:1]
resp = self.sync_plugins(params={'ids': ids})
self.assertEqual(resp.status_code, 200)
def test_sync_failed_when_plugin_not_found(self):
plugin_ids = self._create_new_and_old_version_plugins_for_sync()
ids = [plugin_ids.pop() + 1]
resp = self.sync_plugins(params={'ids': ids}, expect_errors=True)
self.assertEqual(resp.status_code, 404)
@mock.patch('nailgun.plugins.attr_plugin.open', create=True)
@mock.patch('nailgun.plugins.attr_plugin.os.access')
def test_sync_with_invalid_yaml_files(self, maccess, mopen):
maccess.return_value = True
self._create_new_and_old_version_plugins_for_sync()
with mock.patch.object(yaml, 'safe_load') as yaml_safe_load:
yaml_safe_load.side_effect = yaml.YAMLError()
resp = self.sync_plugins(expect_errors=True)
self.assertEqual(resp.status_code, 400)
self.assertRegexpMatches(
resp.json_body["message"],
'Problem with loading YAML file')
def _create_new_and_old_version_plugins_for_sync(self):
plugin_ids = []
old_version_plugin = {
'name': 'test_name_0',
'version': '0.1.1',
'fuel_version': ['6.0'],
'title': 'Test plugin',
'package_version': '1.0.0',
'releases': [
{'os': 'Ubuntu',
'mode': ['ha', 'multinode'],
'version': '2014.2.1-5.1'}
],
}
resp = self.create_plugin(sample=old_version_plugin)
self.assertEqual(resp.status_code, 201)
new_version_plugin_1 = {
'name': 'test_name_1',
'version': '0.1.1',
'fuel_version': ['7.0'],
'title': 'Test plugin',
'package_version': '3.0.0',
'releases': [
{'os': 'Ubuntu',
'mode': ['ha', 'multinode'],
'version': '2014.2.1-5.1'}
],
}
resp = self.create_plugin(sample=new_version_plugin_1)
self.assertEqual(resp.status_code, 201)
# Only plugins with version 3.0.0 will be synced
plugin_ids.append(resp.json['id'])
new_version_plugin_2 = {
'name': 'test_name_2',
'version': '0.1.1',
'fuel_version': ['7.0'],
'title': 'Test plugin',
'package_version': '3.0.0',
'releases': [
{'os': 'Ubuntu',
'mode': ['ha'],
'version': '2014.2.1-5.1'}
],
}
resp = self.create_plugin(sample=new_version_plugin_2)
self.assertEqual(resp.status_code, 201)
plugin_ids.append(resp.json['id'])
return plugin_ids
class TestPrePostHooks(BasePluginTest):
@ -344,3 +443,20 @@ class TestPluginValidation(BasePluginTest):
}
resp = self.create_plugin(sample=sample, expect_errors=True)
self.assertEqual(resp.status_code, 400)
class TestPluginSyncValidation(BasePluginTest):
def test_valid(self):
resp = self.sync_plugins()
self.assertEqual(resp.status_code, 200)
def test_ids_not_present(self):
sample = {'test': '1'}
resp = self.sync_plugins(params=sample, expect_errors=True)
self.assertEqual(resp.status_code, 400)
def test_ids_not_array_type(self):
sample = {'ids': {}}
resp = self.sync_plugins(params=sample, expect_errors=True)
self.assertEqual(resp.status_code, 400)

58
nailgun/nailgun/test/unit/test_attributes_plugin.py
View File

@ -124,6 +124,17 @@ class TestPluginBase(base.BaseTestCase):
self.assertEqual(
expected, self.attr_plugin.master_scripts_path(self.cluster))
def test_sync_metadata_to_db(self):
plugin_metadata = self.env.get_default_plugin_metadata()
with mock.patch.object(self.attr_plugin, '_load_config') as load_conf:
load_conf.return_value = plugin_metadata
self.attr_plugin.sync_metadata_to_db()
for key, val in six.iteritems(plugin_metadata):
self.assertEqual(
getattr(self.plugin, key), val)
class TestPluginV1(TestPluginBase):
@ -165,6 +176,53 @@ class TestPluginV2(TestPluginBase):
'{0}-{1}'.format(self.plugin.name, '0.1'))
class TestPluginV3(TestPluginBase):
__test__ = True
package_version = '3.0.0'
def test_sync_metadata_to_db(self):
plugin_metadata = self.env.get_default_plugin_metadata()
attributes_metadata = self.env.get_default_plugin_env_config()
roles_metadata = self.env.get_default_plugin_node_roles_config()
volumes_metadata = self.env.get_default_plugin_volumes_config()
deployment_tasks = self.env.get_default_plugin_deployment_tasks()
tasks = self.env.get_default_plugin_tasks()
mocked_metadata = {
self._find_path('metadata'): plugin_metadata,
self._find_path('environment_config'): attributes_metadata,
self._find_path('node_roles'): roles_metadata,
self._find_path('volumes'): volumes_metadata,
self._find_path('deployment_tasks'): deployment_tasks,
self._find_path('tasks'): tasks,
}
with mock.patch.object(self.attr_plugin, '_load_config') as load_conf:
load_conf.side_effect = lambda key: mocked_metadata[key]
self.attr_plugin.sync_metadata_to_db()
for key, val in six.iteritems(plugin_metadata):
self.assertEqual(
getattr(self.plugin, key), val)
self.assertEqual(
self.plugin.attributes_metadata, attributes_metadata)
self.assertEqual(
self.plugin.roles_metadata, roles_metadata)
self.assertEqual(
self.plugin.volumes_metadata, volumes_metadata)
self.assertEqual(
self.plugin.deployment_tasks, deployment_tasks)
self.assertEqual(
self.plugin.tasks, tasks)
def _find_path(self, config_name):
return os.path.join(
self.attr_plugin.plugin_path,
'{0}.yaml'.format(config_name))
class TestClusterCompatiblityValidation(base.BaseTestCase):
def setUp(self):

1
nailgun/nailgun/test/unit/test_migration_fuel_7_0.py
View File

@ -102,7 +102,6 @@ def prepare_nodes_with_attributes(meta):
def prepare():
meta = base.reflect_db_metadata()
db.execute(
meta.tables['plugins'].insert(),
[{

36
nailgun/nailgun/test/unit/test_object_plugin.py
View File

@ -23,15 +23,7 @@ from nailgun.test import base
class TestPluginCollection(base.BaseTestCase):
def test_all_newest(self):
for version in ['1.0.0', '2.0.0', '0.0.1']:
plugin_data = self.env.get_default_plugin_metadata(
version=version,
name='multiversion_plugin')
Plugin.create(plugin_data)
single_plugin_data = self.env.get_default_plugin_metadata(
name='single_plugin')
Plugin.create(single_plugin_data)
self._create_test_plugins()
newest_plugins = PluginCollection.all_newest()
self.assertEqual(len(newest_plugins), 2)
@ -46,4 +38,28 @@ class TestPluginCollection(base.BaseTestCase):
self.assertEqual(len(single_plugin), 1)
self.assertEqual(len(multiversion_plugin), 1)
self.assertEqual(multiversion_plugin[0].version, '2.0.0')
self.assertEqual(multiversion_plugin[0].version, '3.0.0')
def test_get_by_uids(self):
plugin_ids = self._create_test_plugins()
ids = plugin_ids[:2]
plugins = PluginCollection.get_by_uids(ids)
self.assertEqual(len(list(plugins)), 2)
self.assertListEqual(
[plugin.id for plugin in plugins], ids)
def _create_test_plugins(self):
plugin_ids = []
for version in ['1.0.0', '2.0.0', '0.0.1', '3.0.0']:
plugin_data = self.env.get_default_plugin_metadata(
version=version,
name='multiversion_plugin')
plugin = Plugin.create(plugin_data)
plugin_ids.append(plugin.id)
single_plugin_data = self.env.get_default_plugin_metadata(
name='single_plugin')
plugin = Plugin.create(single_plugin_data)
plugin_ids.append(plugin.id)
return plugin_ids