From b61567addaf57ede7299ba0f0448b88aa4c182e2 Mon Sep 17 00:00:00 2001 From: Luigi Toscano Date: Wed, 11 Mar 2015 18:35:33 +0100 Subject: [PATCH] Decouple Sahara templates test from vanilla plugin The change introduces a list of template data for node templates and clusters for each plugin. The first known plugin and its first known version are used. At least one default template is added for each plugin, so that all the real deployment can be covered. The tests are skipped if the plugins key in tempest configuration is empty; they fail if no known plugin or no known version for known plugins are found (which probably means that the test configuration should be updated). The list must be maintained; on the other way, after some discussions, the alternative solution - expose all the node group and cluster templates parameters in tempest configuration - would have made the testing more random and too dependent on the cloud operator in charge of running tempest. The list of plugins is not probably going to change massively for each release, so it seems feasible. Change-Id: I397616f7923e696082fe67da3ff9841f89e9a505 --- tempest/api/data_processing/base.py | 304 ++++++++++++++++++ .../data_processing/test_cluster_templates.py | 68 ++-- .../test_node_group_templates.py | 29 +- 3 files changed, 337 insertions(+), 64 deletions(-) diff --git a/tempest/api/data_processing/base.py b/tempest/api/data_processing/base.py index 5992921d70..7f3474b7dc 100644 --- a/tempest/api/data_processing/base.py +++ b/tempest/api/data_processing/base.py @@ -12,14 +12,216 @@ # License for the specific language governing permissions and limitations # under the License. +from collections import OrderedDict + +import six from tempest_lib import exceptions as lib_exc from tempest import config +from tempest import exceptions import tempest.test CONF = config.CONF +"""Default templates. +There should always be at least a master1 and a worker1 node +group template.""" +DEFAULT_TEMPLATES = { + 'vanilla': OrderedDict([ + ('2.6.0', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['namenode', 'resourcemanager', + 'hiveserver'] + }, + 'master2': { + 'count': 1, + 'node_processes': ['oozie', 'historyserver', + 'secondarynamenode'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['datanode', 'nodemanager'], + 'node_configs': { + 'MapReduce': { + 'yarn.app.mapreduce.am.resource.mb': 256, + 'yarn.app.mapreduce.am.command-opts': '-Xmx256m' + }, + 'YARN': { + 'yarn.scheduler.minimum-allocation-mb': 256, + 'yarn.scheduler.maximum-allocation-mb': 1024, + 'yarn.nodemanager.vmem-check-enabled': False + } + } + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs.replication': 1 + } + } + }), + ('1.2.1', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['namenode', 'jobtracker'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['datanode', 'tasktracker'], + 'node_configs': { + 'HDFS': { + 'Data Node Heap Size': 1024 + }, + 'MapReduce': { + 'Task Tracker Heap Size': 1024 + } + } + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs.replication': 1 + }, + 'MapReduce': { + 'mapred.map.tasks.speculative.execution': False, + 'mapred.child.java.opts': '-Xmx500m' + }, + 'general': { + 'Enable Swift': False + } + } + }) + ]), + 'hdp': OrderedDict([ + ('2.0.6', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['NAMENODE', 'SECONDARY_NAMENODE', + 'ZOOKEEPER_SERVER', 'AMBARI_SERVER', + 'HISTORYSERVER', 'RESOURCEMANAGER', + 'GANGLIA_SERVER', 'NAGIOS_SERVER', + 'OOZIE_SERVER'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['HDFS_CLIENT', 'DATANODE', + 'YARN_CLIENT', 'ZOOKEEPER_CLIENT', + 'MAPREDUCE2_CLIENT', 'NODEMANAGER', + 'PIG', 'OOZIE_CLIENT'] + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs.replication': 1 + } + } + }) + ]), + 'spark': OrderedDict([ + ('1.0.0', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['namenode', 'master'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['datanode', 'slave'] + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs.replication': 1 + } + } + }) + ]), + 'cdh': OrderedDict([ + ('5.3.0', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['CLOUDERA_MANAGER'] + }, + 'master2': { + 'count': 1, + 'node_processes': ['HDFS_NAMENODE', + 'YARN_RESOURCEMANAGER'] + }, + 'master3': { + 'count': 1, + 'node_processes': ['OOZIE_SERVER', 'YARN_JOBHISTORY', + 'HDFS_SECONDARYNAMENODE', + 'HIVE_METASTORE', 'HIVE_SERVER2'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['YARN_NODEMANAGER', 'HDFS_DATANODE'] + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs_replication': 1 + } + } + }), + ('5', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['CLOUDERA_MANAGER'] + }, + 'master2': { + 'count': 1, + 'node_processes': ['HDFS_NAMENODE', + 'YARN_RESOURCEMANAGER'] + }, + 'master3': { + 'count': 1, + 'node_processes': ['OOZIE_SERVER', 'YARN_JOBHISTORY', + 'HDFS_SECONDARYNAMENODE', + 'HIVE_METASTORE', 'HIVE_SERVER2'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['YARN_NODEMANAGER', 'HDFS_DATANODE'] + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs_replication': 1 + } + } + }) + ]), + 'mapr': OrderedDict([ + ('4.0.1.mrv2', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['CLDB', 'FileServer', 'ZooKeeper', + 'NodeManager', 'ResourceManager', + 'HistoryServer', 'Oozie'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['FileServer', 'NodeManager', 'Pig'] + } + }, + 'cluster_configs': { + 'Hive': { + 'Hive Version': '0.13', + } + } + }) + ]), +} + class BaseDataProcessingTest(tempest.test.BaseTestCase): @@ -28,6 +230,7 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase): super(BaseDataProcessingTest, cls).skip_checks() if not CONF.service_available.sahara: raise cls.skipException('Sahara support is required') + cls.default_plugin = cls._get_default_plugin() @classmethod def setup_credentials(cls): @@ -43,6 +246,10 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase): def resource_setup(cls): super(BaseDataProcessingTest, cls).resource_setup() + cls.default_version = cls._get_default_version() + if cls.default_plugin is not None and cls.default_version is None: + raise exceptions.InvalidConfiguration( + message="No known Sahara plugin version was found") cls.flavor_ref = CONF.compute.flavor_ref # add lists for watched resources @@ -173,3 +380,100 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase): cls._jobs.append(resp_body['id']) return resp_body + + @classmethod + def _get_default_plugin(cls): + """Returns the default plugin used for testing.""" + if len(CONF.data_processing_feature_enabled.plugins) == 0: + return None + + for plugin in CONF.data_processing_feature_enabled.plugins: + if plugin in DEFAULT_TEMPLATES.keys(): + break + else: + plugin = '' + return plugin + + @classmethod + def _get_default_version(cls): + """Returns the default plugin version used for testing. + This is gathered separately from the plugin to allow + the usage of plugin name in skip_checks. This method is + rather invoked into resource_setup, which allows API calls + and exceptions. + """ + if not cls.default_plugin: + return None + plugin = cls.client.get_plugin(cls.default_plugin) + + for version in DEFAULT_TEMPLATES[cls.default_plugin].keys(): + if version in plugin['versions']: + break + else: + version = None + + return version + + @classmethod + def get_node_group_template(cls, nodegroup='worker1'): + """Returns a node group template for the default plugin.""" + try: + plugin_data = ( + DEFAULT_TEMPLATES[cls.default_plugin][cls.default_version] + ) + nodegroup_data = plugin_data['NODES'][nodegroup] + node_group_template = { + 'description': 'Test node group template', + 'plugin_name': cls.default_plugin, + 'hadoop_version': cls.default_version, + 'node_processes': nodegroup_data['node_processes'], + 'flavor_id': cls.flavor_ref, + 'node_configs': nodegroup_data.get('node_configs', {}), + } + return node_group_template + except (IndexError, KeyError): + return None + + @classmethod + def get_cluster_template(cls, node_group_template_ids=None): + """Returns a cluster template for the default plugin. + node_group_template_defined contains the type and ID of pre-defined + node group templates that have to be used in the cluster template + (instead of dynamically defining them with 'node_processes'). + """ + if node_group_template_ids is None: + node_group_template_ids = {} + try: + plugin_data = ( + DEFAULT_TEMPLATES[cls.default_plugin][cls.default_version] + ) + + all_node_groups = [] + for ng_name, ng_data in six.iteritems(plugin_data['NODES']): + node_group = { + 'name': '%s-node' % (ng_name), + 'flavor_id': cls.flavor_ref, + 'count': ng_data['count'] + } + if ng_name in node_group_template_ids.keys(): + # node group already defined, use it + node_group['node_group_template_id'] = ( + node_group_template_ids[ng_name] + ) + else: + # node_processes list defined on-the-fly + node_group['node_processes'] = ng_data['node_processes'] + if 'node_configs' in ng_data: + node_group['node_configs'] = ng_data['node_configs'] + all_node_groups.append(node_group) + + cluster_template = { + 'description': 'Test cluster template', + 'plugin_name': cls.default_plugin, + 'hadoop_version': cls.default_version, + 'cluster_configs': plugin_data.get('cluster_configs', {}), + 'node_groups': all_node_groups, + } + return cluster_template + except (IndexError, KeyError): + return None diff --git a/tempest/api/data_processing/test_cluster_templates.py b/tempest/api/data_processing/test_cluster_templates.py index 8a63c3f687..cebf493fbf 100644 --- a/tempest/api/data_processing/test_cluster_templates.py +++ b/tempest/api/data_processing/test_cluster_templates.py @@ -15,6 +15,7 @@ from tempest_lib.common.utils import data_utils from tempest.api.data_processing import base as dp_base +from tempest import exceptions from tempest import test @@ -22,56 +23,31 @@ class ClusterTemplateTest(dp_base.BaseDataProcessingTest): """Link to the API documentation is http://docs.openstack.org/developer/ sahara/restapi/rest_api_v1.0.html#cluster-templates """ + @classmethod + def skip_checks(cls): + super(ClusterTemplateTest, cls).skip_checks() + if cls.default_plugin is None: + raise cls.skipException("No Sahara plugins configured") + @classmethod def resource_setup(cls): super(ClusterTemplateTest, cls).resource_setup() - # create node group template - node_group_template = { - 'name': data_utils.rand_name('sahara-ng-template'), - 'description': 'Test node group template', - 'plugin_name': 'vanilla', - 'hadoop_version': '1.2.1', - 'node_processes': ['datanode'], - 'flavor_id': cls.flavor_ref, - 'node_configs': { - 'HDFS': { - 'Data Node Heap Size': 1024 - } - } - } - resp_body = cls.create_node_group_template(**node_group_template) - node_group_template_id = resp_body['id'] - cls.full_cluster_template = { - 'description': 'Test cluster template', - 'plugin_name': 'vanilla', - 'hadoop_version': '1.2.1', - 'cluster_configs': { - 'HDFS': { - 'dfs.replication': 2 - }, - 'MapReduce': { - 'mapred.map.tasks.speculative.execution': False, - 'mapred.child.java.opts': '-Xmx500m' - }, - 'general': { - 'Enable Swift': False - } - }, - 'node_groups': [ - { - 'name': 'master-node', - 'flavor_id': cls.flavor_ref, - 'node_processes': ['namenode'], - 'count': 1 - }, - { - 'name': 'worker-node', - 'node_group_template_id': node_group_template_id, - 'count': 3 - } - ] - } + # pre-define a node group templates + node_group_template_w = cls.get_node_group_template('worker1') + if node_group_template_w is None: + raise exceptions.InvalidConfiguration( + message="No known Sahara plugin was found") + + node_group_template_w['name'] = data_utils.rand_name( + 'sahara-ng-template') + resp_body = cls.create_node_group_template(**node_group_template_w) + node_group_template_id = resp_body['id'] + configured_node_group_templates = {'worker1': node_group_template_id} + + cls.full_cluster_template = cls.get_cluster_template( + configured_node_group_templates) + # create cls.cluster_template variable to use for comparison to cluster # template response body. The 'node_groups' field in the response body # has some extra info that post body does not have. The 'node_groups' diff --git a/tempest/api/data_processing/test_node_group_templates.py b/tempest/api/data_processing/test_node_group_templates.py index d7381f4d0e..4068027e9f 100644 --- a/tempest/api/data_processing/test_node_group_templates.py +++ b/tempest/api/data_processing/test_node_group_templates.py @@ -19,27 +19,16 @@ from tempest import test class NodeGroupTemplateTest(dp_base.BaseDataProcessingTest): + + @classmethod + def skip_checks(cls): + super(NodeGroupTemplateTest, cls).skip_checks() + if cls.default_plugin is None: + raise cls.skipException("No Sahara plugins configured") + @classmethod def resource_setup(cls): super(NodeGroupTemplateTest, cls).resource_setup() - cls.node_group_template = { - 'description': 'Test node group template', - 'plugin_name': 'vanilla', - 'hadoop_version': '1.2.1', - 'node_processes': [ - 'datanode', - 'tasktracker' - ], - 'flavor_id': cls.flavor_ref, - 'node_configs': { - 'HDFS': { - 'Data Node Heap Size': 1024 - }, - 'MapReduce': { - 'Task Tracker Heap Size': 1024 - } - } - } def _create_node_group_template(self, template_name=None): """Creates Node Group Template with optional name specified. @@ -47,6 +36,10 @@ class NodeGroupTemplateTest(dp_base.BaseDataProcessingTest): It creates template, ensures template name and response body. Returns id and name of created template. """ + self.node_group_template = self.get_node_group_template() + self.assertIsNotNone(self.node_group_template, + "No known Sahara plugin was found") + if not template_name: # generate random name if it's not specified template_name = data_utils.rand_name('sahara-ng-template')