openstack
/
sahara
Code Issues Proposed changes

Merge "Added secondary name node heap size param to vanilla plugin"

This commit is contained in:
Jenkins 2014-06-12 17:18:31 +00:00 committed by Gerrit Code Review
parent 021bcb3559 e139f433f0
commit 54b65cbd50
6 changed files with 25 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sahara/plugins/vanilla/v1_2_1/config_helper.py
View File

@ -57,7 +57,6 @@ XML_CONFS = {
"Hive": [HIVE_DEFAULT]
}
# TODO(aignatov): Environmental configs could be more complex
ENV_CONFS = {
"MapReduce": {
'Job Tracker Heap Size': 'HADOOP_JOBTRACKER_OPTS=\\"-Xmx%sm\\"',
@ -65,6 +64,8 @@ ENV_CONFS = {
},
"HDFS": {
'Name Node Heap Size': 'HADOOP_NAMENODE_OPTS=\\"-Xmx%sm\\"',
'Secondary Name Node Heap Size': 'HADOOP_SECONDARYNAMENODE_OPTS='
'\\"-Xmx%sm\\"',
'Data Node Heap Size': 'HADOOP_DATANODE_OPTS=\\"-Xmx%sm\\"'
},
"JobFlow": {

5
sahara/plugins/vanilla/v2_3_0/config.py
View File

@ -169,6 +169,7 @@ def _generate_xml(configs):
def _push_env_configs(instance, configs):
nn_heap = configs['HDFS']['NameNode Heap Size']
snn_heap = configs['HDFS']['SecondaryNameNode Heap Size']
dn_heap = configs['HDFS']['DataNode Heap Size']
rm_heap = configs['YARN']['ResourceManager Heap Size']
nm_heap = configs['YARN']['NodeManager Heap Size']
@ -179,6 +180,10 @@ def _push_env_configs(instance, configs):
'%s/hadoop-env.sh' % HADOOP_CONF_DIR,
'export HADOOP_NAMENODE_OPTS=.*',
'export HADOOP_NAMENODE_OPTS="-Xmx%dm"' % nn_heap)
r.replace_remote_string(
'%s/hadoop-env.sh' % HADOOP_CONF_DIR,
'export HADOOP_SECONDARYNAMENODE_OPTS=.*',
'export HADOOP_SECONDARYNAMENODE_OPTS="-Xmx%dm"' % snn_heap)
r.replace_remote_string(
'%s/hadoop-env.sh' % HADOOP_CONF_DIR,
'export HADOOP_DATANODE_OPTS=.*',

1
sahara/plugins/vanilla/v2_3_0/config_helper.py
View File

@ -56,6 +56,7 @@ ENV_CONFS = {
},
"HDFS": {
'NameNode Heap Size': 1024,
'SecondaryNameNode Heap Size': 1024,
'DataNode Heap Size': 1024
},
"MapReduce": {

7
sahara/tests/integration/tests/cluster_configs.py
View File

@ -17,8 +17,8 @@ from sahara.openstack.common import excutils
from sahara.tests.integration.tests import base
#TODO(ylobankov): add secondary nn config when bug #1217245 will be fixed
NN_CONFIG = {'Name Node Heap Size': 512}
SNN_CONFIG = {'Secondary Name Node Heap Size': 521}
JT_CONFIG = {'Job Tracker Heap Size': 514}
DN_CONFIG = {'Data Node Heap Size': 513}
@ -37,6 +37,10 @@ CONFIG_MAP = {
'service': 'HDFS',
'config': NN_CONFIG
},
'secondarynamenode': {
'service': 'HDFS',
'config': SNN_CONFIG
},
'jobtracker': {
'service': 'MapReduce',
'config': JT_CONFIG
@ -106,7 +110,6 @@ class ClusterConfigTest(base.ITestCase):
self._compare_configs_on_cluster_node(config, value)
for config, value in CLUSTER_HDFS_CONFIG.items():
self._compare_configs_on_cluster_node(config, value)
#TODO(ylobankov): add check for secondary nn when bug #1217245 will be fixed
for process in processes:
if process in CONFIG_MAP:
for config, value in self._get_config_from_config_map(

1
sahara/tests/integration/tests/gating/test_vanilla_gating.py
View File

@ -180,6 +180,7 @@ class VanillaGatingTest(cinder.CinderVolumeTest,
flavor_id=self.flavor_id,
node_processes=['secondarynamenode', 'oozie'],
node_configs={
'HDFS': cluster_configs.SNN_CONFIG,
'JobFlow': cluster_configs.OOZIE_CONFIG
},
floating_ip_pool=floating_ip_pool,

11
sahara/tests/integration/tests/resources/cluster_config_test_script.sh
View File

@ -7,6 +7,10 @@ case $1 in
FUNC="check_nn_heap_size"
;;
SecondaryNameNodeHeapSize)
FUNC="check_snn_heap_size"
;;
JobTrackerHeapSize)
FUNC="check_jt_heap_size"
;;
@ -88,6 +92,13 @@ check_nn_heap_size() {
check_heap_size "namenode"
}
check_snn_heap_size() {
echo -e "*********************** SECONDARY NAME NODE HEAP SIZE **********************\n" >> $log
check_heap_size "secondarynamenode"
}
check_jt_heap_size() {
echo -e "********************** JOB TRACKER HEAP SIZE *********************\n" >> $log