openstack
/
sahara
Code Issues Proposed changes

Merge "Added secondary name node heap size param to vanilla plugin"

This commit is contained in:
Jenkins 2014-06-12 17:18:31 +00:00 committed by Gerrit Code Review
parent 021bcb3559 e139f433f0
commit 54b65cbd50
6 changed files with 25 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sahara/plugins/vanilla/v1_2_1/config_helper.py
View File

@ -57,7 +57,6 @@ XML_CONFS = {
"Hive": [HIVE_DEFAULT] "Hive": [HIVE_DEFAULT]
} }
# TODO(aignatov): Environmental configs could be more complex
ENV_CONFS = { ENV_CONFS = {
"MapReduce": { "MapReduce": {
'Job Tracker Heap Size': 'HADOOP_JOBTRACKER_OPTS=\\"-Xmx%sm\\"', 'Job Tracker Heap Size': 'HADOOP_JOBTRACKER_OPTS=\\"-Xmx%sm\\"',
@ -65,6 +64,8 @@ ENV_CONFS = {
}, },
"HDFS": { "HDFS": {
'Name Node Heap Size': 'HADOOP_NAMENODE_OPTS=\\"-Xmx%sm\\"', 'Name Node Heap Size': 'HADOOP_NAMENODE_OPTS=\\"-Xmx%sm\\"',
'Secondary Name Node Heap Size': 'HADOOP_SECONDARYNAMENODE_OPTS='
'\\"-Xmx%sm\\"',
'Data Node Heap Size': 'HADOOP_DATANODE_OPTS=\\"-Xmx%sm\\"' 'Data Node Heap Size': 'HADOOP_DATANODE_OPTS=\\"-Xmx%sm\\"'
}, },
"JobFlow": { "JobFlow": {

5
sahara/plugins/vanilla/v2_3_0/config.py
View File

@ -169,6 +169,7 @@ def _generate_xml(configs):
def _push_env_configs(instance, configs): def _push_env_configs(instance, configs):
nn_heap = configs['HDFS']['NameNode Heap Size'] nn_heap = configs['HDFS']['NameNode Heap Size']
snn_heap = configs['HDFS']['SecondaryNameNode Heap Size']
dn_heap = configs['HDFS']['DataNode Heap Size'] dn_heap = configs['HDFS']['DataNode Heap Size']
rm_heap = configs['YARN']['ResourceManager Heap Size'] rm_heap = configs['YARN']['ResourceManager Heap Size']
nm_heap = configs['YARN']['NodeManager Heap Size'] nm_heap = configs['YARN']['NodeManager Heap Size']
@ -179,6 +180,10 @@ def _push_env_configs(instance, configs):
'%s/hadoop-env.sh' % HADOOP_CONF_DIR, '%s/hadoop-env.sh' % HADOOP_CONF_DIR,
'export HADOOP_NAMENODE_OPTS=.*', 'export HADOOP_NAMENODE_OPTS=.*',
'export HADOOP_NAMENODE_OPTS="-Xmx%dm"' % nn_heap) 'export HADOOP_NAMENODE_OPTS="-Xmx%dm"' % nn_heap)
r.replace_remote_string(
'%s/hadoop-env.sh' % HADOOP_CONF_DIR,
'export HADOOP_SECONDARYNAMENODE_OPTS=.*',
'export HADOOP_SECONDARYNAMENODE_OPTS="-Xmx%dm"' % snn_heap)
r.replace_remote_string( r.replace_remote_string(
'%s/hadoop-env.sh' % HADOOP_CONF_DIR, '%s/hadoop-env.sh' % HADOOP_CONF_DIR,
'export HADOOP_DATANODE_OPTS=.*', 'export HADOOP_DATANODE_OPTS=.*',

1
sahara/plugins/vanilla/v2_3_0/config_helper.py
View File

@ -56,6 +56,7 @@ ENV_CONFS = {
}, },
"HDFS": { "HDFS": {
'NameNode Heap Size': 1024, 'NameNode Heap Size': 1024,
'SecondaryNameNode Heap Size': 1024,
'DataNode Heap Size': 1024 'DataNode Heap Size': 1024
}, },
"MapReduce": { "MapReduce": {

7
sahara/tests/integration/tests/cluster_configs.py
View File

@ -17,8 +17,8 @@ from sahara.openstack.common import excutils
from sahara.tests.integration.tests import base from sahara.tests.integration.tests import base
#TODO(ylobankov): add secondary nn config when bug #1217245 will be fixed
NN_CONFIG = {'Name Node Heap Size': 512} NN_CONFIG = {'Name Node Heap Size': 512}
SNN_CONFIG = {'Secondary Name Node Heap Size': 521}
JT_CONFIG = {'Job Tracker Heap Size': 514} JT_CONFIG = {'Job Tracker Heap Size': 514}
DN_CONFIG = {'Data Node Heap Size': 513} DN_CONFIG = {'Data Node Heap Size': 513}
@ -37,6 +37,10 @@ CONFIG_MAP = {
'service': 'HDFS', 'service': 'HDFS',
'config': NN_CONFIG 'config': NN_CONFIG
}, },
'secondarynamenode': {
'service': 'HDFS',
'config': SNN_CONFIG
},
'jobtracker': { 'jobtracker': {
'service': 'MapReduce', 'service': 'MapReduce',
'config': JT_CONFIG 'config': JT_CONFIG
@ -106,7 +110,6 @@ class ClusterConfigTest(base.ITestCase):
self._compare_configs_on_cluster_node(config, value) self._compare_configs_on_cluster_node(config, value)
for config, value in CLUSTER_HDFS_CONFIG.items(): for config, value in CLUSTER_HDFS_CONFIG.items():
self._compare_configs_on_cluster_node(config, value) self._compare_configs_on_cluster_node(config, value)
#TODO(ylobankov): add check for secondary nn when bug #1217245 will be fixed
for process in processes: for process in processes:
if process in CONFIG_MAP: if process in CONFIG_MAP:
for config, value in self._get_config_from_config_map( for config, value in self._get_config_from_config_map(

1
sahara/tests/integration/tests/gating/test_vanilla_gating.py
View File

@ -180,6 +180,7 @@ class VanillaGatingTest(cinder.CinderVolumeTest,
flavor_id=self.flavor_id, flavor_id=self.flavor_id,
node_processes=['secondarynamenode', 'oozie'], node_processes=['secondarynamenode', 'oozie'],
node_configs={ node_configs={
'HDFS': cluster_configs.SNN_CONFIG,
'JobFlow': cluster_configs.OOZIE_CONFIG 'JobFlow': cluster_configs.OOZIE_CONFIG
}, },
floating_ip_pool=floating_ip_pool, floating_ip_pool=floating_ip_pool,

11
sahara/tests/integration/tests/resources/cluster_config_test_script.sh
View File

@ -7,6 +7,10 @@ case $1 in
FUNC="check_nn_heap_size" FUNC="check_nn_heap_size"
;; ;;
SecondaryNameNodeHeapSize)
FUNC="check_snn_heap_size"
;;
JobTrackerHeapSize) JobTrackerHeapSize)
FUNC="check_jt_heap_size" FUNC="check_jt_heap_size"
;; ;;
@ -88,6 +92,13 @@ check_nn_heap_size() {
check_heap_size "namenode" check_heap_size "namenode"
} }
check_snn_heap_size() {
echo -e "*********************** SECONDARY NAME NODE HEAP SIZE **********************\n" >> $log
check_heap_size "secondarynamenode"
}
check_jt_heap_size() { check_jt_heap_size() {
echo -e "********************** JOB TRACKER HEAP SIZE *********************\n" >> $log echo -e "********************** JOB TRACKER HEAP SIZE *********************\n" >> $log