openstack/rally
Code Issues Proposed changes

[Sahara] Improved provisioning plugin support

Browse Source 
CDH plugin now has correct service names.
HDP plugin now supports version 2.2
Vanilla plugin now supports version 2.6.0

The config dictionaries moved to a separate file for better screnario
readability.

Change-Id: I63864daae851dd9500e2127cc821a19b0f46b35a
This commit is contained in:
Nikita Konovalov 2015-03-25 18:22:38 +03:00
parent 2efe0cb9ed
commit 1e516d32bf
3 changed files with 145 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
rally/benchmark/scenarios/sahara/consts.py Normal file
View File

@ -0,0 +1,120 @@
# Copyright 2015: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
NODE_PROCESSES = {
"vanilla": {
"1.2.1": {
"master": ["namenode", "jobtracker", "oozie"],
"worker": ["datanode", "tasktracker"]
},
"2.3.0": {
"master": ["namenode", "resourcemanager", "historyserver",
"oozie"],
"worker": ["datanode", "nodemanager"]
},
"2.4.1": {
"master": ["namenode", "resourcemanager", "historyserver",
"oozie"],
"worker": ["datanode", "nodemanager"]
},
"2.6.0": {
"master": ["namenode", "resourcemanager", "historyserver",
"oozie"],
"worker": ["datanode", "nodemanager"]
}
},
"hdp": {
"1.3.2": {
"master": ["JOBTRACKER", "NAMENODE", "SECONDARY_NAMENODE",
"GANGLIA_SERVER", "NAGIOS_SERVER",
"AMBARI_SERVER", "OOZIE_SERVER"],
"worker": ["TASKTRACKER", "DATANODE", "HDFS_CLIENT",
"MAPREDUCE_CLIENT", "OOZIE_CLIENT", "PIG"]
},
"2.0.6": {
"manager": ["AMBARI_SERVER", "GANGLIA_SERVER",
"NAGIOS_SERVER"],
"master": ["NAMENODE", "SECONDARY_NAMENODE",
"ZOOKEEPER_SERVER", "ZOOKEEPER_CLIENT",
"HISTORYSERVER", "RESOURCEMANAGER",
"OOZIE_SERVER"],
"worker": ["DATANODE", "HDFS_CLIENT", "ZOOKEEPER_CLIENT",
"PIG", "MAPREDUCE2_CLIENT", "YARN_CLIENT",
"NODEMANAGER", "OOZIE_CLIENT"]
},
"2.2": {
"manager": ["AMBARI_SERVER", "GANGLIA_SERVER",
"NAGIOS_SERVER"],
"master": ["NAMENODE", "SECONDARY_NAMENODE",
"ZOOKEEPER_SERVER", "ZOOKEEPER_CLIENT",
"HISTORYSERVER", "RESOURCEMANAGER",
"OOZIE_SERVER"],
"worker": ["DATANODE", "HDFS_CLIENT", "ZOOKEEPER_CLIENT",
"PIG", "MAPREDUCE2_CLIENT", "YARN_CLIENT",
"NODEMANAGER", "OOZIE_CLIENT", "TEZ_CLIENT"]
}
},
"cdh": {
"5": {
"manager": ["CLOUDERA_MANAGER"],
"master": ["HDFS_NAMENODE", "YARN_RESOURCEMANAGER",
"OOZIE_SERVER", "YARN_JOBHISTORY",
"HDFS_SECONDARYNAMENODE", "HIVE_METASTORE",
"HIVE_SERVER2"],
"worker": ["YARN_NODEMANAGER", "HDFS_DATANODE"]
}
}
}
REPLICATION_CONFIGS = {
"vanilla": {
"1.2.1": {
"target": "HDFS",
"config_name": "dfs.replication"
},
"2.3.0": {
"target": "HDFS",
"config_name": "dfs.replication"
},
"2.4.1": {
"target": "HDFS",
"config_name": "dfs.replication"
},
"2.6.0": {
"target": "HDFS",
"config_name": "dfs.replication"
}
},
"hdp": {
"1.3.2": {
"target": "HDFS",
"config_name": "dfs.replication"
},
"2.0.6": {
"target": "HDFS",
"config_name": "dfs.replication"
},
"2.2": {
"target": "HDFS",
"config_name": "dfs.replication"
}
},
"cdh": {
"5": {
"target": "HDFS",
"config_name": "dfs_replication"
}
}
}

101
rally/benchmark/scenarios/sahara/utils.py
View File

@ -20,6 +20,7 @@ from oslo_utils import uuidutils
from saharaclient.api import base as sahara_base
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios.sahara import consts as sahara_consts
from rally.benchmark import utils as bench_utils
from rally.common.i18n import _
from rally.common import log as logging
@ -51,87 +52,6 @@ class SaharaScenario(base.Scenario):
RESOURCE_NAME_LENGTH = 20
# TODO(nkonovalov): Add other provisioning plugins
NODE_PROCESSES = {
"vanilla": {
"1.2.1": {
"master": ["namenode", "jobtracker", "oozie"],
"worker": ["datanode", "tasktracker"]
},
"2.3.0": {
"master": ["namenode", "resourcemanager", "historyserver",
"oozie"],
"worker": ["datanode", "nodemanager"]
},
"2.4.1": {
"master": ["namenode", "resourcemanager", "historyserver",
"oozie"],
"worker": ["datanode", "nodemanager"]
}
},
"hdp": {
"1.3.2": {
"master": ["JOBTRACKER", "NAMENODE", "SECONDARY_NAMENODE",
"GANGLIA_SERVER", "NAGIOS_SERVER",
"AMBARI_SERVER", "OOZIE_SERVER"],
"worker": ["TASKTRACKER", "DATANODE", "HDFS_CLIENT",
"MAPREDUCE_CLIENT", "OOZIE_CLIENT", "PIG"]
},
"2.0.6": {
"manager": ["AMBARI_SERVER", "GANGLIA_SERVER",
"NAGIOS_SERVER"],
"master": ["NAMENODE", "SECONDARY_NAMENODE",
"ZOOKEEPER_SERVER", "ZOOKEEPER_CLIENT",
"HISTORYSERVER", "RESOURCEMANAGER",
"OOZIE_SERVER"],
"worker": ["DATANODE", "HDFS_CLIENT", "ZOOKEEPER_CLIENT",
"PIG", "MAPREDUCE2_CLIENT", "YARN_CLIENT",
"NODEMANAGER", "OOZIE_CLIENT"]
}
},
"cdh": {
"5": {
"manager": ["MANAGER"],
"master": ["NAMENODE", "SECONDARYNAMENODE", "RESOURCEMANAGER",
"JOBHISTORY", "OOZIE_SERVER"],
"worker": ["DATANODE", "NODEMANAGER"]
}
}
}
REPLICATION_CONFIGS = {
"vanilla": {
"1.2.1": {
"target": "HDFS",
"config_name": "dfs.replication"
},
"2.3.0": {
"target": "HDFS",
"config_name": "dfs.replication"
},
"2.4.1": {
"target": "HDFS",
"config_name": "dfs.replication"
}
},
"hdp": {
"1.3.2": {
"target": "HDFS",
"config_name": "dfs.replication"
},
"2.0.6": {
"target": "HDFS",
"config_name": "dfs.replication"
}
},
"cdh": {
"5": {
"target": "HDFS",
"config_name": "dfs_replication"
}
}
}
@base.atomic_action_timer("sahara.list_node_group_templates")
def _list_node_group_templates(self):
"""Return user Node Group Templates list."""
@ -155,8 +75,8 @@ class SaharaScenario(base.Scenario):
plugin_name=plugin_name,
hadoop_version=hadoop_version,
flavor_id=flavor_id,
node_processes=self.NODE_PROCESSES[plugin_name][hadoop_version]
["master"])
node_processes=sahara_consts.NODE_PROCESSES[plugin_name]
[hadoop_version]["master"])
@base.atomic_action_timer("sahara.create_worker_node_group_template")
def _create_worker_node_group_template(self, flavor_id, plugin_name,
@ -176,8 +96,8 @@ class SaharaScenario(base.Scenario):
plugin_name=plugin_name,
hadoop_version=hadoop_version,
flavor_id=flavor_id,
node_processes=self.NODE_PROCESSES[plugin_name][hadoop_version]
["worker"])
node_processes=sahara_consts.NODE_PROCESSES[plugin_name]
[hadoop_version]["worker"])
@base.atomic_action_timer("sahara.delete_node_group_template")
def _delete_node_group_template(self, node_group):
@ -288,7 +208,7 @@ class SaharaScenario(base.Scenario):
plugin_name):
replication_value = min(workers_count, 3)
# 3 is a default Hadoop replication
conf = self.REPLICATION_CONFIGS[plugin_name][hadoop_version]
conf = sahara_consts.REPLICATION_CONFIGS[plugin_name][hadoop_version]
LOG.debug("Using replication factor: %s" % replication_value)
replication_config = {
conf["target"]: {
@ -339,26 +259,27 @@ class SaharaScenario(base.Scenario):
{
"name": "master-ng",
"flavor_id": flavor_id,
"node_processes": self.NODE_PROCESSES[plugin_name]
"node_processes": sahara_consts.NODE_PROCESSES[plugin_name]
[hadoop_version]["master"],
"count": 1
}, {
"name": "worker-ng",
"flavor_id": flavor_id,
"node_processes": self.NODE_PROCESSES[plugin_name]
"node_processes": sahara_consts.NODE_PROCESSES[plugin_name]
[hadoop_version]["worker"],
"count": workers_count
}
]
if "manager" in self.NODE_PROCESSES[plugin_name][hadoop_version]:
if "manager" in (sahara_consts.NODE_PROCESSES[plugin_name]
[hadoop_version]):
# Adding manager group separately as it is supported only in
# specific configurations.
node_groups.append({
"name": "manager-ng",
"flavor_id": flavor_id,
"node_processes": self.NODE_PROCESSES[plugin_name]
"node_processes": sahara_consts.NODE_PROCESSES[plugin_name]
[hadoop_version]["manager"],
"count": 1
})

22
tests/unit/benchmark/scenarios/sahara/test_utils.py
View File

@ -51,7 +51,9 @@ class SaharaUtilsTestCase(test.TestCase):
@mock.patch(SAHARA_UTILS + ".SaharaScenario._generate_random_name",
return_value="random_name")
@mock.patch(SAHARA_UTILS + ".SaharaScenario.clients")
def test_create_node_group_templates(self, mock_clients, mock_random_name):
@mock.patch(SAHARA_UTILS + ".sahara_consts")
def test_create_node_group_templates(self, mock_constants, mock_clients,
mock_random_name):
scenario = utils.SaharaScenario()
mock_processes = {
@ -63,7 +65,7 @@ class SaharaUtilsTestCase(test.TestCase):
}
}
scenario.NODE_PROCESSES = mock_processes
mock_constants.NODE_PROCESSES = mock_processes
scenario._create_master_node_group_template(
flavor_id="test_flavor",
@ -117,7 +119,9 @@ class SaharaUtilsTestCase(test.TestCase):
@mock.patch(SAHARA_UTILS + ".SaharaScenario._generate_random_name",
return_value="random_name")
@mock.patch(SAHARA_UTILS + ".SaharaScenario.clients")
def test_launch_cluster(self, mock_clients, mock_random_name):
@mock.patch(SAHARA_UTILS + ".sahara_consts")
def test_launch_cluster(self, mock_constants,
mock_clients, mock_random_name):
clients_values = mock.MagicMock(return_value=[consts.Service.NEUTRON])
mock_clients.services.return_value = mock.MagicMock(
@ -181,8 +185,8 @@ class SaharaUtilsTestCase(test.TestCase):
}
]
scenario.NODE_PROCESSES = mock_processes
scenario.REPLICATION_CONFIGS = mock_configs
mock_constants.NODE_PROCESSES = mock_processes
mock_constants.REPLICATION_CONFIGS = mock_configs
mock_clients("sahara").clusters.create.return_value = mock.MagicMock(
id="test_cluster_id")
@ -220,7 +224,9 @@ class SaharaUtilsTestCase(test.TestCase):
@mock.patch(SAHARA_UTILS + ".SaharaScenario._generate_random_name",
return_value="random_name")
@mock.patch(SAHARA_UTILS + ".SaharaScenario.clients")
def test_launch_cluster_error(self, mock_clients, mock_random_name):
@mock.patch(SAHARA_UTILS + ".sahara_consts")
def test_launch_cluster_error(self, mock_constants, mock_clients,
mock_random_name):
scenario = utils.SaharaScenario(clients=mock.MagicMock())
mock_processes = {
@ -241,8 +247,8 @@ class SaharaUtilsTestCase(test.TestCase):
}
}
scenario.NODE_PROCESSES = mock_processes
scenario.REPLICATION_CONFIGS = mock_configs
mock_constants.NODE_PROCESSES = mock_processes
mock_constants.REPLICATION_CONFIGS = mock_configs
mock_clients("sahara").clusters.create.return_value = mock.MagicMock(
id="test_cluster_id")