From 86e90616dc40b1c5cae8a62357947b5bc55ba5e7 Mon Sep 17 00:00:00 2001 From: Artem Roma Date: Mon, 6 Jul 2015 11:57:26 +0300 Subject: [PATCH] Add plugin deployment tasks to deployment graph `get_deployment_tasks` for nailgun cluster object modified so that it now by default (if deployment graph was not defined for the cluster) returns summary list of deployment task for release and all enabled for the cluster plugins. Change-Id: I4e0b063a80a6d94bfb792b80cd6880a2a471f135 Implements: blueprint role-as-a-plugin --- nailgun/nailgun/errors/__init__.py | 2 + nailgun/nailgun/objects/cluster.py | 9 +- nailgun/nailgun/objects/release.py | 2 + .../orchestrator/plugins_serializers.py | 12 +- .../nailgun/orchestrator/tasks_serializer.py | 19 +- nailgun/nailgun/plugins/adapters.py | 10 +- nailgun/nailgun/plugins/manager.py | 30 +- .../test_orchestrator_serializer_70.py | 263 ++++++++++++++++++ nailgun/nailgun/test/unit/test_objects.py | 48 ++++ .../nailgun/test/unit/test_plugin_adapters.py | 17 ++ .../test/unit/test_plugins_serializers.py | 4 +- 11 files changed, 403 insertions(+), 13 deletions(-) diff --git a/nailgun/nailgun/errors/__init__.py b/nailgun/nailgun/errors/__init__.py index 9457874047..2d6e659c8d 100644 --- a/nailgun/nailgun/errors/__init__.py +++ b/nailgun/nailgun/errors/__init__.py @@ -91,6 +91,8 @@ default_messages = { # Plugin errors "PackageVersionIsNotCompatible": "Package version is not compatible", + "PluginsTasksOverlapping": + "There is task with same id supplied by another plugin", # unknown "UnknownError": "Unknown error" diff --git a/nailgun/nailgun/objects/cluster.py b/nailgun/nailgun/objects/cluster.py index 620ab6d644..179adf3c8c 100644 --- a/nailgun/nailgun/objects/cluster.py +++ b/nailgun/nailgun/objects/cluster.py @@ -788,14 +788,19 @@ class Cluster(NailgunObject): - if there is deployment_graph defined by user - use it instead of defined - if instance assigned for patching - return custom patching graph - - else return default for release deployment graph + - else return default for release and enabled plugins + deployment graph """ if instance.deployment_tasks: return instance.deployment_tasks elif instance.pending_release_id: return yaml.load(graph_configuration.PATCHING) else: - return Release.get_deployment_tasks(instance.release) + release_deployment_tasks = \ + Release.get_deployment_tasks(instance.release) + plugin_deployment_tasks = \ + PluginManager.get_plugins_deployment_tasks(instance) + return release_deployment_tasks + plugin_deployment_tasks @classmethod def create_vmware_attributes(cls, instance): diff --git a/nailgun/nailgun/objects/release.py b/nailgun/nailgun/objects/release.py index 5eb1102448..9793bfbf03 100644 --- a/nailgun/nailgun/objects/release.py +++ b/nailgun/nailgun/objects/release.py @@ -177,6 +177,8 @@ class Release(NailgunObject): elif env_version.startswith('5.1') or env_version.startswith('6.0'): return yaml.load(graph_configuration.DEPLOYMENT_51_60) + return [] + @classmethod def get_min_controller_count(cls, instance): return instance.roles_metadata['controller']['limits']['min'] diff --git a/nailgun/nailgun/orchestrator/plugins_serializers.py b/nailgun/nailgun/orchestrator/plugins_serializers.py index 7c9467df42..30905d6573 100644 --- a/nailgun/nailgun/orchestrator/plugins_serializers.py +++ b/nailgun/nailgun/orchestrator/plugins_serializers.py @@ -134,7 +134,11 @@ class PluginsPreDeploymentHooksSerializer(BasePluginDeploymentHooksSerializer): repo_tasks = [] for plugin in plugins: - uids = get_uids_for_tasks(self.nodes, plugin.tasks) + # TODO(aroma): remove this concatenation when unified way of + # processing will be introduced for deployment tasks and existing + # plugin tasks + tasks_of_plugin = plugin.tasks + plugin.deployment_tasks + uids = get_uids_for_tasks(self.nodes, tasks_of_plugin) # If there are no nodes for tasks execution # or if there are no files in repository @@ -177,7 +181,11 @@ class PluginsPreDeploymentHooksSerializer(BasePluginDeploymentHooksSerializer): def sync_scripts(self, plugins): tasks = [] for plugin in plugins: - uids = get_uids_for_tasks(self.nodes, plugin.tasks) + # TODO(aroma): remove this concatenation when unified way of + # processing will be introduced for deployment tasks and existing + # plugin tasks + tasks_of_plugin = plugin.tasks + plugin.deployment_tasks + uids = get_uids_for_tasks(self.nodes, tasks_of_plugin) if not uids: continue tasks.append( diff --git a/nailgun/nailgun/orchestrator/tasks_serializer.py b/nailgun/nailgun/orchestrator/tasks_serializer.py index a2005c2626..8888792ed8 100644 --- a/nailgun/nailgun/orchestrator/tasks_serializer.py +++ b/nailgun/nailgun/orchestrator/tasks_serializer.py @@ -37,15 +37,22 @@ def get_uids_for_tasks(nodes, tasks): """ roles = [] for task in tasks: - if task['role'] == consts.ALL_ROLES: + # plugin tasks may store information about node + # role not only in `role` key but also in `groups` + task_role = task.get('role', task.get('groups')) + if task_role == consts.ALL_ROLES: return get_uids_for_roles(nodes, consts.ALL_ROLES) - elif task['role'] == consts.MASTER_ROLE: + elif task_role == consts.MASTER_ROLE: return ['master'] - elif isinstance(task['role'], list): - roles.extend(task['role']) - else: + elif isinstance(task_role, list): + roles.extend(task_role) + # if task has 'skipped' status it is allowed that 'roles' and + # 'groups' are not be specified + elif task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped: logger.warn( - 'Wrong roles format, `roles` should be a list or "*" in %s', + 'Wrong roles format in task %s: either ' + '`roles` or `groups` must be specified and contain ' + 'a list of roles or "*"', task) return get_uids_for_roles(nodes, roles) diff --git a/nailgun/nailgun/plugins/adapters.py b/nailgun/nailgun/plugins/adapters.py index 9fdcb37f14..99a2799ec4 100644 --- a/nailgun/nailgun/plugins/adapters.py +++ b/nailgun/nailgun/plugins/adapters.py @@ -160,7 +160,7 @@ class PluginAdapterBase(object): u'weight': 70, u'label': self.plugin.title, 'plugin_id': self.plugin.id} - def set_cluster_tasks(self, cluster): + def set_cluster_tasks(self): """Loads plugins provided tasks from tasks config file and sets them to instance tasks variable. """ @@ -192,6 +192,14 @@ class PluginAdapterBase(object): return settings.PLUGINS_SLAVES_SCRIPTS_PATH.format( plugin_name=self.path_name) + @property + def deployment_tasks(self): + deployment_tasks = [] + for task in self.plugin.deployment_tasks: + task['parameters'].setdefault('cwd', self.slaves_scripts_path) + deployment_tasks.append(task) + return deployment_tasks + def get_release_info(self, release): """Returns plugin release information which corresponds to a provided release. diff --git a/nailgun/nailgun/plugins/manager.py b/nailgun/nailgun/plugins/manager.py index e0622368b7..8519c1cdc0 100644 --- a/nailgun/nailgun/plugins/manager.py +++ b/nailgun/nailgun/plugins/manager.py @@ -13,7 +13,9 @@ # under the License. import six +from six.moves import map +from nailgun.errors import errors from nailgun.logger import logger from nailgun.objects.plugin import Plugin from nailgun.objects.plugin import PluginCollection @@ -76,7 +78,7 @@ class PluginManager(object): cluster_plugins = [] for plugin_db in cluster.plugins: plugin_adapter = wrap_plugin(plugin_db) - plugin_adapter.set_cluster_tasks(cluster) + plugin_adapter.set_cluster_tasks() cluster_plugins.append(plugin_adapter) return cluster_plugins @@ -93,3 +95,29 @@ class PluginManager(object): for plugin in plugins: plugin_adapter = wrap_plugin(plugin) plugin_adapter.sync_metadata_to_db() + + @classmethod + def get_plugins_deployment_tasks(cls, cluster): + deployment_tasks = [] + + processed_tasks = {} + for plugin_adapter in map(wrap_plugin, cluster.plugins): + depl_tasks = plugin_adapter.deployment_tasks + + for t in depl_tasks: + t_id = t['id'] + if t_id in processed_tasks: + raise errors.PluginsTasksOverlapping( + 'Plugin {0} is overlapping with plugin {1} ' + 'by introducing the same deployment task with ' + 'id {2}' + .format(plugin_adapter.full_name, + processed_tasks[t_id], + t_id) + ) + + processed_tasks[t_id] = plugin_adapter.full_name + + deployment_tasks.extend(depl_tasks) + + return deployment_tasks diff --git a/nailgun/nailgun/test/integration/test_orchestrator_serializer_70.py b/nailgun/nailgun/test/integration/test_orchestrator_serializer_70.py index fa27b4ce00..a0634b86cf 100644 --- a/nailgun/nailgun/test/integration/test_orchestrator_serializer_70.py +++ b/nailgun/nailgun/test/integration/test_orchestrator_serializer_70.py @@ -24,6 +24,8 @@ from nailgun import objects from nailgun.orchestrator.deployment_graph import AstuteGraph from nailgun.orchestrator.deployment_serializers import \ get_serializer_for_cluster +from nailgun.orchestrator import stages +from nailgun.test import base from nailgun.test.integration.test_orchestrator_serializer import \ BaseDeploymentSerializer @@ -309,3 +311,264 @@ class TestDeploymentSerializationForNovaNetwork70(BaseDeploymentSerializer): def test_generate_vmware_attributes_data(self): self.check_generate_vmware_attributes_data() + + +class TestPluginDeploymentTasksInjection(base.BaseIntegrationTest): + + release_deployment_tasks = [ + {'id': 'pre_deployment_start', + 'type': 'stage'}, + {'id': 'pre_deployment_end', + 'type': 'stage', + 'requires': ['pre_deployment_start']}, + {'id': 'deploy_start', + 'type': 'stage'}, + {'id': 'deploy_end', + 'requires': ['deploy_start'], + 'type': 'stage'}, + {'id': 'post_deployment_start', + 'type': 'stage', + 'requires': ['deploy_end']}, + {'id': 'post_deployment_end', + 'type': 'stage', + 'requires': ['post_deployment_start']}, + {'id': 'primary-controller', + 'parameters': {'strategy': {'type': 'one_by_one'}}, + 'required_for': ['deploy_end'], + 'requires': ['deploy_start'], + 'role': ['primary-controller'], + 'type': 'group'}, + {'id': 'first-fake-depl-task', + 'required_for': ['deploy_end'], + 'requires': ['deploy_start'], + 'type': 'puppet', + 'parameters': {'puppet_manifest': 'first-fake-depl-task', + 'puppet_modules': 'test', + 'timeout': 0}, + 'groups': ['primary-controller']}, + {'id': 'second-fake-depl-task', + 'required_for': ['deploy_end'], + 'requires': ['deploy_start'], + 'type': 'puppet', + 'parameters': {'puppet_manifest': 'second-fake-depl-task', + 'puppet_modules': 'test', + 'timeout': 0}, + 'groups': ['primary-controller']}, + ] + + def setUp(self): + super(TestPluginDeploymentTasksInjection, self).setUp() + + self.cluster = self._prepare_cluster() + + def _prepare_cluster(self): + self.env.create( + release_kwargs={ + 'version': '2015.1.0-7.0', + 'deployment_tasks': self.release_deployment_tasks, + }, + cluster_kwargs={ + 'mode': 'ha_compact', + 'net_provider': 'neutron', + 'net_segment_type': 'vlan', + }, + nodes_kwargs=[ + {'roles': ['controller'], 'primary_roles': ['controller'], + 'pending_addition': True} + ] + ) + return self.env.clusters[0] + + def prepare_plugins_for_cluster(self, cluster, plugins_kw_list): + plugins = [ + self._create_plugin(**kw) + for kw in plugins_kw_list + ] + cluster.plugins.extend(plugins) + self.db.flush() + + def _create_plugin(self, **plugin_kwargs): + plugin_kwargs.update( + { + 'releases': [ + { + 'repository_path': 'plugin_test', + 'version': self.cluster.release.version, + 'os': + self.cluster.release.operating_system.lower(), + 'mode': ['ha', 'multinode'], + 'deployment_scripts_path': 'plugin_test/' + }, + ], + } + ) + plugin_data = self.env.get_default_plugin_metadata( + **plugin_kwargs + ) + + return objects.Plugin.create(plugin_data) + + def _check_pre_deployment_tasks(self, serialized, task_type): + self.assertTrue(serialized) + + needed_task = next( + t for t in serialized + if t['type'] == task_type) + self.assertIsNotNone(needed_task) + self.assertIsNotNone(needed_task.get('parameters')) + self.assertItemsEqual( + (n.uid for n in self.cluster.nodes), + needed_task['uids'] + ) + + def test_plugin_depl_tasks_proper_injections(self): + self.prepare_plugins_for_cluster( + self.cluster, + [ + { + 'name': 'between_rel_tasks', + 'deployment_tasks': [ + { + 'id': 'between-rel-tasks', + 'type': 'puppet', + 'groups': ['primary-controller'], + 'requires': ['first-fake-depl-task'], + 'required_for': ['second-fake-depl-task'], + 'parameters': { + 'puppet_manifest': 'between-rel-tasks', + 'puppet_modules': 'test', + 'timeout': 0, + } + }, + ], + }, + ] + ) + + graph = AstuteGraph(self.cluster) + objects.NodeCollection.prepare_for_deployment(self.cluster.nodes) + serializer = \ + get_serializer_for_cluster(self.cluster)(graph) + serialized = serializer.serialize(self.cluster, self.cluster.nodes) + + serialized_tasks = serialized[0]['tasks'] + + expected_priority = { + 100: 'first-fake-depl-task', + 200: 'between-rel-tasks', + 300: 'second-fake-depl-task', + } + + for task in serialized_tasks: + task_identificator = task['parameters']['puppet_manifest'] + self.assertEqual( + task_identificator, expected_priority[task['priority']] + ) + + def test_plugin_depl_task_overwrite_from_rel(self): + self.prepare_plugins_for_cluster( + self.cluster, + [ + { + 'name': 'between_rel_tasks', + 'deployment_tasks': [ + { + 'id': 'first-fake-depl-task', + 'type': 'puppet', + 'groups': ['primary-controller'], + 'requires': ['deploy_start'], + 'required_for': ['second-fake-depl-task'], + 'parameters': { + 'puppet_manifest': 'plugin_task', + 'puppet_modules': 'test', + 'timeout': 0, + } + }, + ], + }, + ] + ) + + graph = AstuteGraph(self.cluster) + objects.NodeCollection.prepare_for_deployment(self.cluster.nodes) + serializer = \ + get_serializer_for_cluster(self.cluster)(graph) + serialized = serializer.serialize(self.cluster, self.cluster.nodes) + + serialized_tasks = serialized[0]['tasks'] + + needed_task_priority = next( + t['priority'] for t in serialized_tasks + if t['parameters']['puppet_manifest'] == 'plugin_task' + ) + # first task in graph has priority equal 100 + self.assertEqual(needed_task_priority, 100) + + def test_plugin_depl_task_in_pre_depl(self): + self.prepare_plugins_for_cluster( + self.cluster, + [ + { + 'name': 'pre_depl_plugin_task', + 'deployment_tasks': [ + { + 'id': 'pre-depl-plugin-task', + 'type': 'puppet', + 'role': ['primary-controller'], + 'requires': ['pre_deployment_start'], + 'required_for': ['pre_deployment_end'], + 'parameters': { + 'puppet_manifest': 'pre_depl_plugin_task', + 'puppet_modules': 'test', + 'timeout': 0, + } + }, + ], + }, + ] + ) + + graph = AstuteGraph(self.cluster) + objects.NodeCollection.prepare_for_deployment(self.cluster.nodes) + with mock.patch('nailgun.plugins.adapters.glob.glob', + mock.Mock(return_value='path/to/test/repos')): + pre_deployment = stages.pre_deployment_serialize( + graph, self.cluster, self.cluster.nodes) + + for task_type in (consts.ORCHESTRATOR_TASK_TYPES.sync, + consts.ORCHESTRATOR_TASK_TYPES.upload_file): + self._check_pre_deployment_tasks(pre_deployment, task_type) + + def test_plugin_depl_task_in_post_depl(self): + self.prepare_plugins_for_cluster( + self.cluster, + [ + { + 'name': 'post-depl-plugin-task', + 'deployment_tasks': [ + { + 'id': 'post-depl-plugin-task', + 'type': 'puppet', + 'role': ['primary-controller'], + 'requires': ['post_deployment_start'], + 'required_for': ['post_deployment_end'], + 'parameters': { + 'puppet_manifest': 'post_depl_plugin_task', + 'puppet_modules': 'test', + 'timeout': 0, + } + }, + ], + }, + ] + ) + + graph = AstuteGraph(self.cluster) + objects.NodeCollection.prepare_for_deployment(self.cluster.nodes) + post_deployment = stages.post_deployment_serialize( + graph, self.cluster, self.cluster.nodes) + + self.assertEqual( + post_deployment[0]['parameters']['puppet_manifest'], + 'post_depl_plugin_task' + ) diff --git a/nailgun/nailgun/test/unit/test_objects.py b/nailgun/nailgun/test/unit/test_objects.py index 2800bbf24c..534ad79776 100644 --- a/nailgun/nailgun/test/unit/test_objects.py +++ b/nailgun/nailgun/test/unit/test_objects.py @@ -743,6 +743,14 @@ class TestClusterObject(BaseTestCase): {'roles': ['compute']}, {'roles': ['cinder']}]) + def _create_cluster_with_plugins(self, plugins_kw_list): + cluster = self.env.create_cluster(api=False) + + for kw in plugins_kw_list: + cluster.plugins.append(objects.Plugin.create(kw)) + + return cluster + def test_all_controllers(self): self.assertEqual(len(objects.Cluster.get_nodes_by_role( self.env.clusters[0], 'controller')), 2) @@ -780,6 +788,46 @@ class TestClusterObject(BaseTestCase): objects.Cluster.get_network_roles(cluster), cluster.release.network_roles_metadata) + def test_get_deployment_tasks(self): + deployment_tasks = self.env.get_default_plugin_deployment_tasks() + plugin_metadata = self.env.get_default_plugin_metadata( + deployment_tasks=deployment_tasks + ) + + cluster = self._create_cluster_with_plugins([plugin_metadata]) + + cluster_deployment_tasks = \ + objects.Cluster.get_deployment_tasks(cluster) + + tasks_ids = [t['id'] for t in cluster_deployment_tasks] + depl_task_id = deployment_tasks[0]['id'] + self.assertIn(depl_task_id, tasks_ids) + + default_tasks_count = len(cluster.release.deployment_tasks) + self.assertEqual(len(cluster_deployment_tasks), + default_tasks_count + + len(cluster.plugins[0].deployment_tasks)) + + def test_get_deployment_tasks_overlapping_error(self): + deployment_tasks = self.env.get_default_plugin_deployment_tasks() + plugins_kw_list = [ + self.env.get_default_plugin_metadata( + name=plugin_name, + deployment_tasks=deployment_tasks) + for plugin_name in ('test_plugin_first', 'test_plugin_second') + ] + + cluster = self._create_cluster_with_plugins(plugins_kw_list) + + expected_message = ( + 'Plugin test_plugin_second-0.1.0 is overlapping with plugin ' + 'test_plugin_first-0.1.0 by introducing the same ' + 'deployment task with id role-name' + ) + with self.assertRaisesRegexp(errors.PluginsTasksOverlapping, + expected_message): + objects.Cluster.get_deployment_tasks(cluster) + class TestClusterObjectGetNetworkManager(BaseTestCase): def setUp(self): diff --git a/nailgun/nailgun/test/unit/test_plugin_adapters.py b/nailgun/nailgun/test/unit/test_plugin_adapters.py index 06bad0c724..e5214d869a 100644 --- a/nailgun/nailgun/test/unit/test_plugin_adapters.py +++ b/nailgun/nailgun/test/unit/test_plugin_adapters.py @@ -136,6 +136,23 @@ class TestPluginBase(base.BaseTestCase): self.assertEqual( getattr(self.plugin, key), val) + def test_get_deployment_tasks(self): + self.plugin.deployment_tasks = \ + self.env.get_default_plugin_deployment_tasks() + + depl_task = self.plugin_adapter.deployment_tasks[0] + self.assertEqual(depl_task['parameters'].get('cwd'), + self.plugin_adapter.slaves_scripts_path) + + def test_get_deployment_tasks_params_not_changed(self): + expected = 'path/to/some/dir' + self.plugin.deployment_tasks = \ + self.env.get_default_plugin_deployment_tasks( + parameters={'cwd': expected} + ) + depl_task = self.plugin_adapter.deployment_tasks[0] + self.assertEqual(depl_task['parameters'].get('cwd'), expected) + class TestPluginV1(TestPluginBase): diff --git a/nailgun/nailgun/test/unit/test_plugins_serializers.py b/nailgun/nailgun/test/unit/test_plugins_serializers.py index b7445f6b6b..2935f43274 100644 --- a/nailgun/nailgun/test/unit/test_plugins_serializers.py +++ b/nailgun/nailgun/test/unit/test_plugins_serializers.py @@ -197,7 +197,9 @@ class TestPluginsPreDeploymentHooksSerializer(base.BaseTestCase): self.hook = PluginsPreDeploymentHooksSerializer( self.cluster, self.nodes) - self.plugins = [mock.Mock()] + + plugin = mock.Mock(tasks=[], deployment_tasks=[]) + self.plugins = [plugin] @mock.patch( 'nailgun.orchestrator.plugins_serializers.get_uids_for_tasks',