diff --git a/tempest/api/data_processing/base.py b/tempest/api/data_processing/base.py index d91fbaa400..5a903b738b 100644 --- a/tempest/api/data_processing/base.py +++ b/tempest/api/data_processing/base.py @@ -12,14 +12,216 @@ # License for the specific language governing permissions and limitations # under the License. +from collections import OrderedDict + +import six from tempest_lib import exceptions as lib_exc from tempest import config +from tempest import exceptions import tempest.test CONF = config.CONF +"""Default templates. +There should always be at least a master1 and a worker1 node +group template.""" +DEFAULT_TEMPLATES = { + 'vanilla': OrderedDict([ + ('2.6.0', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['namenode', 'resourcemanager', + 'hiveserver'] + }, + 'master2': { + 'count': 1, + 'node_processes': ['oozie', 'historyserver', + 'secondarynamenode'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['datanode', 'nodemanager'], + 'node_configs': { + 'MapReduce': { + 'yarn.app.mapreduce.am.resource.mb': 256, + 'yarn.app.mapreduce.am.command-opts': '-Xmx256m' + }, + 'YARN': { + 'yarn.scheduler.minimum-allocation-mb': 256, + 'yarn.scheduler.maximum-allocation-mb': 1024, + 'yarn.nodemanager.vmem-check-enabled': False + } + } + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs.replication': 1 + } + } + }), + ('1.2.1', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['namenode', 'jobtracker'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['datanode', 'tasktracker'], + 'node_configs': { + 'HDFS': { + 'Data Node Heap Size': 1024 + }, + 'MapReduce': { + 'Task Tracker Heap Size': 1024 + } + } + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs.replication': 1 + }, + 'MapReduce': { + 'mapred.map.tasks.speculative.execution': False, + 'mapred.child.java.opts': '-Xmx500m' + }, + 'general': { + 'Enable Swift': False + } + } + }) + ]), + 'hdp': OrderedDict([ + ('2.0.6', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['NAMENODE', 'SECONDARY_NAMENODE', + 'ZOOKEEPER_SERVER', 'AMBARI_SERVER', + 'HISTORYSERVER', 'RESOURCEMANAGER', + 'GANGLIA_SERVER', 'NAGIOS_SERVER', + 'OOZIE_SERVER'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['HDFS_CLIENT', 'DATANODE', + 'YARN_CLIENT', 'ZOOKEEPER_CLIENT', + 'MAPREDUCE2_CLIENT', 'NODEMANAGER', + 'PIG', 'OOZIE_CLIENT'] + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs.replication': 1 + } + } + }) + ]), + 'spark': OrderedDict([ + ('1.0.0', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['namenode', 'master'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['datanode', 'slave'] + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs.replication': 1 + } + } + }) + ]), + 'cdh': OrderedDict([ + ('5.3.0', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['CLOUDERA_MANAGER'] + }, + 'master2': { + 'count': 1, + 'node_processes': ['HDFS_NAMENODE', + 'YARN_RESOURCEMANAGER'] + }, + 'master3': { + 'count': 1, + 'node_processes': ['OOZIE_SERVER', 'YARN_JOBHISTORY', + 'HDFS_SECONDARYNAMENODE', + 'HIVE_METASTORE', 'HIVE_SERVER2'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['YARN_NODEMANAGER', 'HDFS_DATANODE'] + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs_replication': 1 + } + } + }), + ('5', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['CLOUDERA_MANAGER'] + }, + 'master2': { + 'count': 1, + 'node_processes': ['HDFS_NAMENODE', + 'YARN_RESOURCEMANAGER'] + }, + 'master3': { + 'count': 1, + 'node_processes': ['OOZIE_SERVER', 'YARN_JOBHISTORY', + 'HDFS_SECONDARYNAMENODE', + 'HIVE_METASTORE', 'HIVE_SERVER2'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['YARN_NODEMANAGER', 'HDFS_DATANODE'] + } + }, + 'cluster_configs': { + 'HDFS': { + 'dfs_replication': 1 + } + } + }) + ]), + 'mapr': OrderedDict([ + ('4.0.1.mrv2', { + 'NODES': { + 'master1': { + 'count': 1, + 'node_processes': ['CLDB', 'FileServer', 'ZooKeeper', + 'NodeManager', 'ResourceManager', + 'HistoryServer', 'Oozie'] + }, + 'worker1': { + 'count': 1, + 'node_processes': ['FileServer', 'NodeManager', 'Pig'] + } + }, + 'cluster_configs': { + 'Hive': { + 'Hive Version': '0.13', + } + } + }) + ]), +} + class BaseDataProcessingTest(tempest.test.BaseTestCase): @@ -28,6 +230,7 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase): super(BaseDataProcessingTest, cls).skip_checks() if not CONF.service_available.sahara: raise cls.skipException('Sahara support is required') + cls.default_plugin = cls._get_default_plugin() @classmethod def setup_credentials(cls): @@ -43,6 +246,10 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase): def resource_setup(cls): super(BaseDataProcessingTest, cls).resource_setup() + cls.default_version = cls._get_default_version() + if cls.default_plugin is not None and cls.default_version is None: + raise exceptions.InvalidConfiguration( + message="No known Sahara plugin version was found") cls.flavor_ref = CONF.compute.flavor_ref # add lists for watched resources @@ -172,3 +379,100 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase): cls._jobs.append(resp_body['id']) return resp_body + + @classmethod + def _get_default_plugin(cls): + """Returns the default plugin used for testing.""" + if len(CONF.data_processing_feature_enabled.plugins) == 0: + return None + + for plugin in CONF.data_processing_feature_enabled.plugins: + if plugin in DEFAULT_TEMPLATES.keys(): + break + else: + plugin = '' + return plugin + + @classmethod + def _get_default_version(cls): + """Returns the default plugin version used for testing. + This is gathered separately from the plugin to allow + the usage of plugin name in skip_checks. This method is + rather invoked into resource_setup, which allows API calls + and exceptions. + """ + if not cls.default_plugin: + return None + plugin = cls.client.get_plugin(cls.default_plugin) + + for version in DEFAULT_TEMPLATES[cls.default_plugin].keys(): + if version in plugin['versions']: + break + else: + version = None + + return version + + @classmethod + def get_node_group_template(cls, nodegroup='worker1'): + """Returns a node group template for the default plugin.""" + try: + plugin_data = ( + DEFAULT_TEMPLATES[cls.default_plugin][cls.default_version] + ) + nodegroup_data = plugin_data['NODES'][nodegroup] + node_group_template = { + 'description': 'Test node group template', + 'plugin_name': cls.default_plugin, + 'hadoop_version': cls.default_version, + 'node_processes': nodegroup_data['node_processes'], + 'flavor_id': cls.flavor_ref, + 'node_configs': nodegroup_data.get('node_configs', {}), + } + return node_group_template + except (IndexError, KeyError): + return None + + @classmethod + def get_cluster_template(cls, node_group_template_ids=None): + """Returns a cluster template for the default plugin. + node_group_template_defined contains the type and ID of pre-defined + node group templates that have to be used in the cluster template + (instead of dynamically defining them with 'node_processes'). + """ + if node_group_template_ids is None: + node_group_template_ids = {} + try: + plugin_data = ( + DEFAULT_TEMPLATES[cls.default_plugin][cls.default_version] + ) + + all_node_groups = [] + for ng_name, ng_data in six.iteritems(plugin_data['NODES']): + node_group = { + 'name': '%s-node' % (ng_name), + 'flavor_id': cls.flavor_ref, + 'count': ng_data['count'] + } + if ng_name in node_group_template_ids.keys(): + # node group already defined, use it + node_group['node_group_template_id'] = ( + node_group_template_ids[ng_name] + ) + else: + # node_processes list defined on-the-fly + node_group['node_processes'] = ng_data['node_processes'] + if 'node_configs' in ng_data: + node_group['node_configs'] = ng_data['node_configs'] + all_node_groups.append(node_group) + + cluster_template = { + 'description': 'Test cluster template', + 'plugin_name': cls.default_plugin, + 'hadoop_version': cls.default_version, + 'cluster_configs': plugin_data.get('cluster_configs', {}), + 'node_groups': all_node_groups, + } + return cluster_template + except (IndexError, KeyError): + return None diff --git a/tempest/api/data_processing/test_cluster_templates.py b/tempest/api/data_processing/test_cluster_templates.py index 8a63c3f687..cebf493fbf 100644 --- a/tempest/api/data_processing/test_cluster_templates.py +++ b/tempest/api/data_processing/test_cluster_templates.py @@ -15,6 +15,7 @@ from tempest_lib.common.utils import data_utils from tempest.api.data_processing import base as dp_base +from tempest import exceptions from tempest import test @@ -22,56 +23,31 @@ class ClusterTemplateTest(dp_base.BaseDataProcessingTest): """Link to the API documentation is http://docs.openstack.org/developer/ sahara/restapi/rest_api_v1.0.html#cluster-templates """ + @classmethod + def skip_checks(cls): + super(ClusterTemplateTest, cls).skip_checks() + if cls.default_plugin is None: + raise cls.skipException("No Sahara plugins configured") + @classmethod def resource_setup(cls): super(ClusterTemplateTest, cls).resource_setup() - # create node group template - node_group_template = { - 'name': data_utils.rand_name('sahara-ng-template'), - 'description': 'Test node group template', - 'plugin_name': 'vanilla', - 'hadoop_version': '1.2.1', - 'node_processes': ['datanode'], - 'flavor_id': cls.flavor_ref, - 'node_configs': { - 'HDFS': { - 'Data Node Heap Size': 1024 - } - } - } - resp_body = cls.create_node_group_template(**node_group_template) - node_group_template_id = resp_body['id'] - cls.full_cluster_template = { - 'description': 'Test cluster template', - 'plugin_name': 'vanilla', - 'hadoop_version': '1.2.1', - 'cluster_configs': { - 'HDFS': { - 'dfs.replication': 2 - }, - 'MapReduce': { - 'mapred.map.tasks.speculative.execution': False, - 'mapred.child.java.opts': '-Xmx500m' - }, - 'general': { - 'Enable Swift': False - } - }, - 'node_groups': [ - { - 'name': 'master-node', - 'flavor_id': cls.flavor_ref, - 'node_processes': ['namenode'], - 'count': 1 - }, - { - 'name': 'worker-node', - 'node_group_template_id': node_group_template_id, - 'count': 3 - } - ] - } + # pre-define a node group templates + node_group_template_w = cls.get_node_group_template('worker1') + if node_group_template_w is None: + raise exceptions.InvalidConfiguration( + message="No known Sahara plugin was found") + + node_group_template_w['name'] = data_utils.rand_name( + 'sahara-ng-template') + resp_body = cls.create_node_group_template(**node_group_template_w) + node_group_template_id = resp_body['id'] + configured_node_group_templates = {'worker1': node_group_template_id} + + cls.full_cluster_template = cls.get_cluster_template( + configured_node_group_templates) + # create cls.cluster_template variable to use for comparison to cluster # template response body. The 'node_groups' field in the response body # has some extra info that post body does not have. The 'node_groups' diff --git a/tempest/api/data_processing/test_node_group_templates.py b/tempest/api/data_processing/test_node_group_templates.py index d7381f4d0e..4068027e9f 100644 --- a/tempest/api/data_processing/test_node_group_templates.py +++ b/tempest/api/data_processing/test_node_group_templates.py @@ -19,27 +19,16 @@ from tempest import test class NodeGroupTemplateTest(dp_base.BaseDataProcessingTest): + + @classmethod + def skip_checks(cls): + super(NodeGroupTemplateTest, cls).skip_checks() + if cls.default_plugin is None: + raise cls.skipException("No Sahara plugins configured") + @classmethod def resource_setup(cls): super(NodeGroupTemplateTest, cls).resource_setup() - cls.node_group_template = { - 'description': 'Test node group template', - 'plugin_name': 'vanilla', - 'hadoop_version': '1.2.1', - 'node_processes': [ - 'datanode', - 'tasktracker' - ], - 'flavor_id': cls.flavor_ref, - 'node_configs': { - 'HDFS': { - 'Data Node Heap Size': 1024 - }, - 'MapReduce': { - 'Task Tracker Heap Size': 1024 - } - } - } def _create_node_group_template(self, template_name=None): """Creates Node Group Template with optional name specified. @@ -47,6 +36,10 @@ class NodeGroupTemplateTest(dp_base.BaseDataProcessingTest): It creates template, ensures template name and response body. Returns id and name of created template. """ + self.node_group_template = self.get_node_group_template() + self.assertIsNotNone(self.node_group_template, + "No known Sahara plugin was found") + if not template_name: # generate random name if it's not specified template_name = data_utils.rand_name('sahara-ng-template')