diff --git a/rally/benchmark/scenarios/sahara/consts.py b/rally/benchmark/scenarios/sahara/consts.py new file mode 100644 index 0000000000..d90b0511eb --- /dev/null +++ b/rally/benchmark/scenarios/sahara/consts.py @@ -0,0 +1,120 @@ +# Copyright 2015: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +NODE_PROCESSES = { + "vanilla": { + "1.2.1": { + "master": ["namenode", "jobtracker", "oozie"], + "worker": ["datanode", "tasktracker"] + }, + "2.3.0": { + "master": ["namenode", "resourcemanager", "historyserver", + "oozie"], + "worker": ["datanode", "nodemanager"] + }, + "2.4.1": { + "master": ["namenode", "resourcemanager", "historyserver", + "oozie"], + "worker": ["datanode", "nodemanager"] + }, + "2.6.0": { + "master": ["namenode", "resourcemanager", "historyserver", + "oozie"], + "worker": ["datanode", "nodemanager"] + } + }, + "hdp": { + "1.3.2": { + "master": ["JOBTRACKER", "NAMENODE", "SECONDARY_NAMENODE", + "GANGLIA_SERVER", "NAGIOS_SERVER", + "AMBARI_SERVER", "OOZIE_SERVER"], + "worker": ["TASKTRACKER", "DATANODE", "HDFS_CLIENT", + "MAPREDUCE_CLIENT", "OOZIE_CLIENT", "PIG"] + }, + "2.0.6": { + "manager": ["AMBARI_SERVER", "GANGLIA_SERVER", + "NAGIOS_SERVER"], + "master": ["NAMENODE", "SECONDARY_NAMENODE", + "ZOOKEEPER_SERVER", "ZOOKEEPER_CLIENT", + "HISTORYSERVER", "RESOURCEMANAGER", + "OOZIE_SERVER"], + "worker": ["DATANODE", "HDFS_CLIENT", "ZOOKEEPER_CLIENT", + "PIG", "MAPREDUCE2_CLIENT", "YARN_CLIENT", + "NODEMANAGER", "OOZIE_CLIENT"] + }, + "2.2": { + "manager": ["AMBARI_SERVER", "GANGLIA_SERVER", + "NAGIOS_SERVER"], + "master": ["NAMENODE", "SECONDARY_NAMENODE", + "ZOOKEEPER_SERVER", "ZOOKEEPER_CLIENT", + "HISTORYSERVER", "RESOURCEMANAGER", + "OOZIE_SERVER"], + "worker": ["DATANODE", "HDFS_CLIENT", "ZOOKEEPER_CLIENT", + "PIG", "MAPREDUCE2_CLIENT", "YARN_CLIENT", + "NODEMANAGER", "OOZIE_CLIENT", "TEZ_CLIENT"] + } + }, + "cdh": { + "5": { + "manager": ["CLOUDERA_MANAGER"], + "master": ["HDFS_NAMENODE", "YARN_RESOURCEMANAGER", + "OOZIE_SERVER", "YARN_JOBHISTORY", + "HDFS_SECONDARYNAMENODE", "HIVE_METASTORE", + "HIVE_SERVER2"], + "worker": ["YARN_NODEMANAGER", "HDFS_DATANODE"] + } + } +} + +REPLICATION_CONFIGS = { + "vanilla": { + "1.2.1": { + "target": "HDFS", + "config_name": "dfs.replication" + }, + "2.3.0": { + "target": "HDFS", + "config_name": "dfs.replication" + }, + "2.4.1": { + "target": "HDFS", + "config_name": "dfs.replication" + }, + "2.6.0": { + "target": "HDFS", + "config_name": "dfs.replication" + } + }, + "hdp": { + "1.3.2": { + "target": "HDFS", + "config_name": "dfs.replication" + }, + "2.0.6": { + "target": "HDFS", + "config_name": "dfs.replication" + }, + "2.2": { + "target": "HDFS", + "config_name": "dfs.replication" + } + }, + "cdh": { + "5": { + "target": "HDFS", + "config_name": "dfs_replication" + } + } +} diff --git a/rally/benchmark/scenarios/sahara/utils.py b/rally/benchmark/scenarios/sahara/utils.py index 931eaddc3c..e2f3db070a 100644 --- a/rally/benchmark/scenarios/sahara/utils.py +++ b/rally/benchmark/scenarios/sahara/utils.py @@ -20,6 +20,7 @@ from oslo_utils import uuidutils from saharaclient.api import base as sahara_base from rally.benchmark.scenarios import base +from rally.benchmark.scenarios.sahara import consts as sahara_consts from rally.benchmark import utils as bench_utils from rally.common.i18n import _ from rally.common import log as logging @@ -51,87 +52,6 @@ class SaharaScenario(base.Scenario): RESOURCE_NAME_LENGTH = 20 - # TODO(nkonovalov): Add other provisioning plugins - NODE_PROCESSES = { - "vanilla": { - "1.2.1": { - "master": ["namenode", "jobtracker", "oozie"], - "worker": ["datanode", "tasktracker"] - }, - "2.3.0": { - "master": ["namenode", "resourcemanager", "historyserver", - "oozie"], - "worker": ["datanode", "nodemanager"] - }, - "2.4.1": { - "master": ["namenode", "resourcemanager", "historyserver", - "oozie"], - "worker": ["datanode", "nodemanager"] - } - }, - "hdp": { - "1.3.2": { - "master": ["JOBTRACKER", "NAMENODE", "SECONDARY_NAMENODE", - "GANGLIA_SERVER", "NAGIOS_SERVER", - "AMBARI_SERVER", "OOZIE_SERVER"], - "worker": ["TASKTRACKER", "DATANODE", "HDFS_CLIENT", - "MAPREDUCE_CLIENT", "OOZIE_CLIENT", "PIG"] - }, - "2.0.6": { - "manager": ["AMBARI_SERVER", "GANGLIA_SERVER", - "NAGIOS_SERVER"], - "master": ["NAMENODE", "SECONDARY_NAMENODE", - "ZOOKEEPER_SERVER", "ZOOKEEPER_CLIENT", - "HISTORYSERVER", "RESOURCEMANAGER", - "OOZIE_SERVER"], - "worker": ["DATANODE", "HDFS_CLIENT", "ZOOKEEPER_CLIENT", - "PIG", "MAPREDUCE2_CLIENT", "YARN_CLIENT", - "NODEMANAGER", "OOZIE_CLIENT"] - } - }, - "cdh": { - "5": { - "manager": ["MANAGER"], - "master": ["NAMENODE", "SECONDARYNAMENODE", "RESOURCEMANAGER", - "JOBHISTORY", "OOZIE_SERVER"], - "worker": ["DATANODE", "NODEMANAGER"] - } - } - } - - REPLICATION_CONFIGS = { - "vanilla": { - "1.2.1": { - "target": "HDFS", - "config_name": "dfs.replication" - }, - "2.3.0": { - "target": "HDFS", - "config_name": "dfs.replication" - }, - "2.4.1": { - "target": "HDFS", - "config_name": "dfs.replication" - } - }, - "hdp": { - "1.3.2": { - "target": "HDFS", - "config_name": "dfs.replication" - }, - "2.0.6": { - "target": "HDFS", - "config_name": "dfs.replication" - } - }, - "cdh": { - "5": { - "target": "HDFS", - "config_name": "dfs_replication" - } - } - } - @base.atomic_action_timer("sahara.list_node_group_templates") def _list_node_group_templates(self): """Return user Node Group Templates list.""" @@ -155,8 +75,8 @@ class SaharaScenario(base.Scenario): plugin_name=plugin_name, hadoop_version=hadoop_version, flavor_id=flavor_id, - node_processes=self.NODE_PROCESSES[plugin_name][hadoop_version] - ["master"]) + node_processes=sahara_consts.NODE_PROCESSES[plugin_name] + [hadoop_version]["master"]) @base.atomic_action_timer("sahara.create_worker_node_group_template") def _create_worker_node_group_template(self, flavor_id, plugin_name, @@ -176,8 +96,8 @@ class SaharaScenario(base.Scenario): plugin_name=plugin_name, hadoop_version=hadoop_version, flavor_id=flavor_id, - node_processes=self.NODE_PROCESSES[plugin_name][hadoop_version] - ["worker"]) + node_processes=sahara_consts.NODE_PROCESSES[plugin_name] + [hadoop_version]["worker"]) @base.atomic_action_timer("sahara.delete_node_group_template") def _delete_node_group_template(self, node_group): @@ -288,7 +208,7 @@ class SaharaScenario(base.Scenario): plugin_name): replication_value = min(workers_count, 3) # 3 is a default Hadoop replication - conf = self.REPLICATION_CONFIGS[plugin_name][hadoop_version] + conf = sahara_consts.REPLICATION_CONFIGS[plugin_name][hadoop_version] LOG.debug("Using replication factor: %s" % replication_value) replication_config = { conf["target"]: { @@ -339,26 +259,27 @@ class SaharaScenario(base.Scenario): { "name": "master-ng", "flavor_id": flavor_id, - "node_processes": self.NODE_PROCESSES[plugin_name] + "node_processes": sahara_consts.NODE_PROCESSES[plugin_name] [hadoop_version]["master"], "count": 1 }, { "name": "worker-ng", "flavor_id": flavor_id, - "node_processes": self.NODE_PROCESSES[plugin_name] + "node_processes": sahara_consts.NODE_PROCESSES[plugin_name] [hadoop_version]["worker"], "count": workers_count } ] - if "manager" in self.NODE_PROCESSES[plugin_name][hadoop_version]: + if "manager" in (sahara_consts.NODE_PROCESSES[plugin_name] + [hadoop_version]): # Adding manager group separately as it is supported only in # specific configurations. node_groups.append({ "name": "manager-ng", "flavor_id": flavor_id, - "node_processes": self.NODE_PROCESSES[plugin_name] + "node_processes": sahara_consts.NODE_PROCESSES[plugin_name] [hadoop_version]["manager"], "count": 1 }) diff --git a/tests/unit/benchmark/scenarios/sahara/test_utils.py b/tests/unit/benchmark/scenarios/sahara/test_utils.py index 1ad0fd56da..421f87f13a 100644 --- a/tests/unit/benchmark/scenarios/sahara/test_utils.py +++ b/tests/unit/benchmark/scenarios/sahara/test_utils.py @@ -51,7 +51,9 @@ class SaharaUtilsTestCase(test.TestCase): @mock.patch(SAHARA_UTILS + ".SaharaScenario._generate_random_name", return_value="random_name") @mock.patch(SAHARA_UTILS + ".SaharaScenario.clients") - def test_create_node_group_templates(self, mock_clients, mock_random_name): + @mock.patch(SAHARA_UTILS + ".sahara_consts") + def test_create_node_group_templates(self, mock_constants, mock_clients, + mock_random_name): scenario = utils.SaharaScenario() mock_processes = { @@ -63,7 +65,7 @@ class SaharaUtilsTestCase(test.TestCase): } } - scenario.NODE_PROCESSES = mock_processes + mock_constants.NODE_PROCESSES = mock_processes scenario._create_master_node_group_template( flavor_id="test_flavor", @@ -117,7 +119,9 @@ class SaharaUtilsTestCase(test.TestCase): @mock.patch(SAHARA_UTILS + ".SaharaScenario._generate_random_name", return_value="random_name") @mock.patch(SAHARA_UTILS + ".SaharaScenario.clients") - def test_launch_cluster(self, mock_clients, mock_random_name): + @mock.patch(SAHARA_UTILS + ".sahara_consts") + def test_launch_cluster(self, mock_constants, + mock_clients, mock_random_name): clients_values = mock.MagicMock(return_value=[consts.Service.NEUTRON]) mock_clients.services.return_value = mock.MagicMock( @@ -181,8 +185,8 @@ class SaharaUtilsTestCase(test.TestCase): } ] - scenario.NODE_PROCESSES = mock_processes - scenario.REPLICATION_CONFIGS = mock_configs + mock_constants.NODE_PROCESSES = mock_processes + mock_constants.REPLICATION_CONFIGS = mock_configs mock_clients("sahara").clusters.create.return_value = mock.MagicMock( id="test_cluster_id") @@ -220,7 +224,9 @@ class SaharaUtilsTestCase(test.TestCase): @mock.patch(SAHARA_UTILS + ".SaharaScenario._generate_random_name", return_value="random_name") @mock.patch(SAHARA_UTILS + ".SaharaScenario.clients") - def test_launch_cluster_error(self, mock_clients, mock_random_name): + @mock.patch(SAHARA_UTILS + ".sahara_consts") + def test_launch_cluster_error(self, mock_constants, mock_clients, + mock_random_name): scenario = utils.SaharaScenario(clients=mock.MagicMock()) mock_processes = { @@ -241,8 +247,8 @@ class SaharaUtilsTestCase(test.TestCase): } } - scenario.NODE_PROCESSES = mock_processes - scenario.REPLICATION_CONFIGS = mock_configs + mock_constants.NODE_PROCESSES = mock_processes + mock_constants.REPLICATION_CONFIGS = mock_configs mock_clients("sahara").clusters.create.return_value = mock.MagicMock( id="test_cluster_id")