Adding config hints for CDH plugin
This change adds an implementation of the config hints for both versions of the CDH plugin. Change-Id: I5f2f9f99aae49d53606fcd3ff11c940f17f99691 Partial-Implements: bp: edp-job-types-endpoint
This commit is contained in:
luhuichun
lu huichun
parent
4ca232ce72
commit
54fc5ec94a
46
sahara/plugins/cdh/confighints_helper.py
Normal file
46
sahara/plugins/cdh/confighints_helper.py
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
# Copyright (c) 2015 Intel Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from sahara.service.edp.oozie.workflow_creator import workflow_factory
|
||||||
|
from sahara.utils import xmlutils
|
||||||
|
|
||||||
|
|
||||||
|
def get_possible_hive_config_from(file_name):
|
||||||
|
'''Return the possible configs, args, params for a Hive job.'''
|
||||||
|
config = {
|
||||||
|
'configs': xmlutils.load_hadoop_xml_defaults(file_name),
|
||||||
|
'params': {}
|
||||||
|
}
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def get_possible_mapreduce_config_from(file_name):
|
||||||
|
'''Return the possible configs, args, params for a MapReduce job.'''
|
||||||
|
config = {
|
||||||
|
'configs': get_possible_pig_config_from(file_name).get('configs')
|
||||||
|
}
|
||||||
|
config['configs'] += workflow_factory.get_possible_mapreduce_configs()
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def get_possible_pig_config_from(file_name):
|
||||||
|
'''Return the possible configs, args, params for a Pig job.'''
|
||||||
|
config = {
|
||||||
|
'configs': xmlutils.load_hadoop_xml_defaults(file_name),
|
||||||
|
'args': [],
|
||||||
|
'params': {}
|
||||||
|
}
|
||||||
|
return config
|
||||||
@ -13,11 +13,13 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
from sahara.plugins.cdh import confighints_helper as ch_helper
|
||||||
from sahara.plugins.cdh.v5 import cloudera_utils as cu
|
from sahara.plugins.cdh.v5 import cloudera_utils as cu
|
||||||
from sahara.plugins import exceptions as ex
|
from sahara.plugins import exceptions as ex
|
||||||
from sahara.plugins import utils as u
|
from sahara.plugins import utils as u
|
||||||
from sahara.service.edp import hdfs_helper
|
from sahara.service.edp import hdfs_helper
|
||||||
from sahara.service.edp.oozie import engine as edp_engine
|
from sahara.service.edp.oozie import engine as edp_engine
|
||||||
|
from sahara.utils import edp
|
||||||
|
|
||||||
CU = cu.ClouderaUtilsV5()
|
CU = cu.ClouderaUtilsV5()
|
||||||
|
|
||||||
@ -52,3 +54,18 @@ class EdpOozieEngine(edp_engine.OozieJobEngine):
|
|||||||
'OOZIE_SERVER', '1', oo_count)
|
'OOZIE_SERVER', '1', oo_count)
|
||||||
|
|
||||||
super(EdpOozieEngine, self).validate_job_execution(cluster, job, data)
|
super(EdpOozieEngine, self).validate_job_execution(cluster, job, data)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_possible_job_config(job_type):
|
||||||
|
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
|
||||||
|
return {'job_config': ch_helper.get_possible_hive_config_from(
|
||||||
|
'plugins/cdh/v5/resources/hive-site.xml')}
|
||||||
|
if edp.compare_job_type(job_type,
|
||||||
|
edp.JOB_TYPE_MAPREDUCE,
|
||||||
|
edp.JOB_TYPE_MAPREDUCE_STREAMING):
|
||||||
|
return {'job_config': ch_helper.get_possible_mapreduce_config_from(
|
||||||
|
'plugins/cdh/v5/resources/mapred-site.xml')}
|
||||||
|
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
|
||||||
|
return {'job_config': ch_helper.get_possible_pig_config_from(
|
||||||
|
'plugins/cdh/v5/resources/mapred-site.xml')}
|
||||||
|
return edp_engine.OozieJobEngine.get_possible_job_config(job_type)
|
||||||
|
|||||||
61
sahara/plugins/cdh/v5/resources/hive-site.xml
Normal file
61
sahara/plugins/cdh/v5/resources/hive-site.xml
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
|
||||||
|
<!--Autogenerated by Cloudera Manager-->
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.local</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.uris</name>
|
||||||
|
<value>thrift://test-master-001.novalocal:9083</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.client.socket.timeout</name>
|
||||||
|
<value>300</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.warehouse.dir</name>
|
||||||
|
<value>/user/hive/warehouse</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.warehouse.subdir.inherit.perms</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapred.reduce.tasks</name>
|
||||||
|
<value>-1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.exec.reducers.bytes.per.reducer</name>
|
||||||
|
<value>1073741824</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.exec.copyfile.maxsize</name>
|
||||||
|
<value>33554432</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.exec.reducers.max</name>
|
||||||
|
<value>999</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.execute.setugi</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.cluster.delegation.token.store.class</name>
|
||||||
|
<value>org.apache.hadoop.hive.thrift.MemoryTokenStore</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.server2.enable.doAs</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.hdfs.impl.disable.cache</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.server2.use.SSL</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
157
sahara/plugins/cdh/v5/resources/mapred-site.xml
Normal file
157
sahara/plugins/cdh/v5/resources/mapred-site.xml
Normal file
@ -0,0 +1,157 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
|
||||||
|
<!--Autogenerated by Cloudera Manager-->
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.split.metainfo.maxsize</name>
|
||||||
|
<value>10000000</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.counters.max</name>
|
||||||
|
<value>120</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.output.fileoutputformat.compress</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.output.fileoutputformat.compress.type</name>
|
||||||
|
<value>BLOCK</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.output.fileoutputformat.compress.codec</name>
|
||||||
|
<value>org.apache.hadoop.io.compress.DefaultCodec</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.output.compress.codec</name>
|
||||||
|
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.output.compress</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>zlib.compress.level</name>
|
||||||
|
<value>DEFAULT_COMPRESSION</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.task.io.sort.factor</name>
|
||||||
|
<value>64</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.sort.spill.percent</name>
|
||||||
|
<value>0.8</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.shuffle.parallelcopies</name>
|
||||||
|
<value>10</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.task.timeout</name>
|
||||||
|
<value>600000</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.client.submit.file.replication</name>
|
||||||
|
<value>10</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.reduces</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.task.io.sort.mb</name>
|
||||||
|
<value>256</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.speculative</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.speculative</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
|
||||||
|
<value>0.8</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.jobhistory.address</name>
|
||||||
|
<value>test-master-001.novalocal:10020</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.jobhistory.webapp.address</name>
|
||||||
|
<value>test-master-001.novalocal:19888</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.jobhistory.webapp.https.address</name>
|
||||||
|
<value>test-master-001.novalocal:19890</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.jobhistory.admin.address</name>
|
||||||
|
<value>test-master-001.novalocal:10033</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.framework.name</name>
|
||||||
|
<value>yarn</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.staging-dir</name>
|
||||||
|
<value>/user</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.resource.mb</name>
|
||||||
|
<value>1024</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.ubertask.enable</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.command-opts</name>
|
||||||
|
<value>-Djava.net.preferIPv4Stack=true -Xmx825955249</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.java.opts</name>
|
||||||
|
<value>-Djava.net.preferIPv4Stack=true -Xmx825955249</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.java.opts</name>
|
||||||
|
<value>-Djava.net.preferIPv4Stack=true -Xmx825955249</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.admin.user.env</name>
|
||||||
|
<value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.memory.mb</name>
|
||||||
|
<value>1024</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.cpu.vcores</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.memory.mb</name>
|
||||||
|
<value>1024</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.cpu.vcores</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.application.classpath</name>
|
||||||
|
<value>$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.admin.user.env</name>
|
||||||
|
<value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.shuffle.max.connections</name>
|
||||||
|
<value>80</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
@ -13,11 +13,13 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
from sahara.plugins.cdh import confighints_helper as ch_helper
|
||||||
from sahara.plugins.cdh.v5_3_0 import cloudera_utils as cu
|
from sahara.plugins.cdh.v5_3_0 import cloudera_utils as cu
|
||||||
from sahara.plugins import exceptions as ex
|
from sahara.plugins import exceptions as ex
|
||||||
from sahara.plugins import utils as u
|
from sahara.plugins import utils as u
|
||||||
from sahara.service.edp import hdfs_helper
|
from sahara.service.edp import hdfs_helper
|
||||||
from sahara.service.edp.oozie import engine as edp_engine
|
from sahara.service.edp.oozie import engine as edp_engine
|
||||||
|
from sahara.utils import edp
|
||||||
|
|
||||||
CU = cu.ClouderaUtilsV530()
|
CU = cu.ClouderaUtilsV530()
|
||||||
|
|
||||||
@ -52,3 +54,18 @@ class EdpOozieEngine(edp_engine.OozieJobEngine):
|
|||||||
'OOZIE_SERVER', '1', oo_count)
|
'OOZIE_SERVER', '1', oo_count)
|
||||||
|
|
||||||
super(EdpOozieEngine, self).validate_job_execution(cluster, job, data)
|
super(EdpOozieEngine, self).validate_job_execution(cluster, job, data)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_possible_job_config(job_type):
|
||||||
|
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
|
||||||
|
return {'job_config': ch_helper.get_possible_hive_config_from(
|
||||||
|
'plugins/cdh/v5_3_0/resources/hive-site.xml')}
|
||||||
|
if edp.compare_job_type(job_type,
|
||||||
|
edp.JOB_TYPE_MAPREDUCE,
|
||||||
|
edp.JOB_TYPE_MAPREDUCE_STREAMING):
|
||||||
|
return {'job_config': ch_helper.get_possible_mapreduce_config_from(
|
||||||
|
'plugins/cdh/v5_3_0/resources/mapred-site.xml')}
|
||||||
|
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
|
||||||
|
return {'job_config': ch_helper.get_possible_pig_config_from(
|
||||||
|
'plugins/cdh/v5_3_0/resources/mapred-site.xml')}
|
||||||
|
return edp_engine.OozieJobEngine.get_possible_job_config(job_type)
|
||||||
|
|||||||
61
sahara/plugins/cdh/v5_3_0/resources/hive-site.xml
Normal file
61
sahara/plugins/cdh/v5_3_0/resources/hive-site.xml
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
|
||||||
|
<!--Autogenerated by Cloudera Manager-->
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.local</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.uris</name>
|
||||||
|
<value>thrift://test-master-001.novalocal:9083</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.client.socket.timeout</name>
|
||||||
|
<value>300</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.warehouse.dir</name>
|
||||||
|
<value>/user/hive/warehouse</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.warehouse.subdir.inherit.perms</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapred.reduce.tasks</name>
|
||||||
|
<value>-1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.exec.reducers.bytes.per.reducer</name>
|
||||||
|
<value>1073741824</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.exec.copyfile.maxsize</name>
|
||||||
|
<value>33554432</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.exec.reducers.max</name>
|
||||||
|
<value>999</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.metastore.execute.setugi</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.cluster.delegation.token.store.class</name>
|
||||||
|
<value>org.apache.hadoop.hive.thrift.MemoryTokenStore</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.server2.enable.doAs</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.hdfs.impl.disable.cache</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hive.server2.use.SSL</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
157
sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml
Normal file
157
sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml
Normal file
@ -0,0 +1,157 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
|
||||||
|
<!--Autogenerated by Cloudera Manager-->
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.split.metainfo.maxsize</name>
|
||||||
|
<value>10000000</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.counters.max</name>
|
||||||
|
<value>120</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.output.fileoutputformat.compress</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.output.fileoutputformat.compress.type</name>
|
||||||
|
<value>BLOCK</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.output.fileoutputformat.compress.codec</name>
|
||||||
|
<value>org.apache.hadoop.io.compress.DefaultCodec</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.output.compress.codec</name>
|
||||||
|
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.output.compress</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>zlib.compress.level</name>
|
||||||
|
<value>DEFAULT_COMPRESSION</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.task.io.sort.factor</name>
|
||||||
|
<value>64</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.sort.spill.percent</name>
|
||||||
|
<value>0.8</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.shuffle.parallelcopies</name>
|
||||||
|
<value>10</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.task.timeout</name>
|
||||||
|
<value>600000</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.client.submit.file.replication</name>
|
||||||
|
<value>10</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.reduces</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.task.io.sort.mb</name>
|
||||||
|
<value>256</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.speculative</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.speculative</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
|
||||||
|
<value>0.8</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.jobhistory.address</name>
|
||||||
|
<value>test-master-001.novalocal:10020</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.jobhistory.webapp.address</name>
|
||||||
|
<value>test-master-001.novalocal:19888</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.jobhistory.webapp.https.address</name>
|
||||||
|
<value>test-master-001.novalocal:19890</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.jobhistory.admin.address</name>
|
||||||
|
<value>test-master-001.novalocal:10033</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.framework.name</name>
|
||||||
|
<value>yarn</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.staging-dir</name>
|
||||||
|
<value>/user</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.resource.mb</name>
|
||||||
|
<value>1024</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.job.ubertask.enable</name>
|
||||||
|
<value>false</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.command-opts</name>
|
||||||
|
<value>-Djava.net.preferIPv4Stack=true -Xmx825955249</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.java.opts</name>
|
||||||
|
<value>-Djava.net.preferIPv4Stack=true -Xmx825955249</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.java.opts</name>
|
||||||
|
<value>-Djava.net.preferIPv4Stack=true -Xmx825955249</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>yarn.app.mapreduce.am.admin.user.env</name>
|
||||||
|
<value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.memory.mb</name>
|
||||||
|
<value>1024</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.map.cpu.vcores</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.memory.mb</name>
|
||||||
|
<value>1024</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.reduce.cpu.vcores</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.application.classpath</name>
|
||||||
|
<value>$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.admin.user.env</name>
|
||||||
|
<value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.shuffle.max.connections</name>
|
||||||
|
<value>80</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
59
sahara/tests/unit/plugins/cdh/test_confighints_helper.py
Normal file
59
sahara/tests/unit/plugins/cdh/test_confighints_helper.py
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
# Copyright (c) 2015 Intel Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from sahara.plugins.cdh import confighints_helper as ch_helper
|
||||||
|
from sahara.tests.unit import base as sahara_base
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigHintsHelperTest(sahara_base.SaharaTestCase):
|
||||||
|
@mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
|
||||||
|
return_value=[])
|
||||||
|
def test_get_possible_hive_config_from(self, load_hadoop_xml_defaults):
|
||||||
|
expected_config = {
|
||||||
|
'configs': [],
|
||||||
|
'params': {}
|
||||||
|
}
|
||||||
|
actual_config = ch_helper.get_possible_hive_config_from(
|
||||||
|
'sample-config.xml')
|
||||||
|
load_hadoop_xml_defaults.called_once_with('sample-config.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
|
||||||
|
return_value=[])
|
||||||
|
def test_get_possible_mapreduce_config_from(
|
||||||
|
self, load_hadoop_xml_defaults):
|
||||||
|
expected_config = {
|
||||||
|
'configs': [],
|
||||||
|
}
|
||||||
|
actual_config = ch_helper.get_possible_mapreduce_config_from(
|
||||||
|
'sample-config.xml')
|
||||||
|
load_hadoop_xml_defaults.called_once_with('sample-config.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
|
||||||
|
return_value=[])
|
||||||
|
def test_get_possible_pig_config_from(
|
||||||
|
self, load_hadoop_xml_defaults):
|
||||||
|
expected_config = {
|
||||||
|
'configs': [],
|
||||||
|
'args': [],
|
||||||
|
'params': {}
|
||||||
|
}
|
||||||
|
actual_config = ch_helper.get_possible_pig_config_from(
|
||||||
|
'sample-config.xml')
|
||||||
|
load_hadoop_xml_defaults.called_once_with('sample-config.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
0
sahara/tests/unit/plugins/cdh/v5/__init__.py
Normal file
0
sahara/tests/unit/plugins/cdh/v5/__init__.py
Normal file
94
sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py
Normal file
94
sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
# Copyright (c) 2015 Intel Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from sahara.plugins.cdh.v5 import edp_engine
|
||||||
|
from sahara.tests.unit import base as sahara_base
|
||||||
|
from sahara.utils import edp
|
||||||
|
|
||||||
|
|
||||||
|
class CDH5ConfigHintsTest(sahara_base.SaharaTestCase):
|
||||||
|
@mock.patch(
|
||||||
|
'sahara.plugins.cdh.confighints_helper.get_possible_hive_config_from',
|
||||||
|
return_value={})
|
||||||
|
def test_get_possible_job_config_hive(self,
|
||||||
|
get_possible_hive_config_from):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_HIVE)
|
||||||
|
get_possible_hive_config_from.assert_called_once_with(
|
||||||
|
'plugins/cdh/v5/resources/hive-site.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch('sahara.plugins.cdh.v5.edp_engine.EdpOozieEngine')
|
||||||
|
def test_get_possible_job_config_java(self, BaseCDHEdpOozieEngine):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
BaseCDHEdpOozieEngine.get_possible_job_config.return_value = (
|
||||||
|
expected_config)
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_JAVA)
|
||||||
|
BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with(
|
||||||
|
edp.JOB_TYPE_JAVA)
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'sahara.plugins.cdh.confighints_helper.'
|
||||||
|
'get_possible_mapreduce_config_from',
|
||||||
|
return_value={})
|
||||||
|
def test_get_possible_job_config_mapreduce(
|
||||||
|
self, get_possible_mapreduce_config_from):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_MAPREDUCE)
|
||||||
|
get_possible_mapreduce_config_from.assert_called_once_with(
|
||||||
|
'plugins/cdh/v5/resources/mapred-site.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'sahara.plugins.cdh.confighints_helper.'
|
||||||
|
'get_possible_mapreduce_config_from',
|
||||||
|
return_value={})
|
||||||
|
def test_get_possible_job_config_mapreduce_streaming(
|
||||||
|
self, get_possible_mapreduce_config_from):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_MAPREDUCE_STREAMING)
|
||||||
|
get_possible_mapreduce_config_from.assert_called_once_with(
|
||||||
|
'plugins/cdh/v5/resources/mapred-site.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'sahara.plugins.cdh.confighints_helper.get_possible_pig_config_from',
|
||||||
|
return_value={})
|
||||||
|
def test_get_possible_job_config_pig(self,
|
||||||
|
get_possible_pig_config_from):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_PIG)
|
||||||
|
get_possible_pig_config_from.assert_called_once_with(
|
||||||
|
'plugins/cdh/v5/resources/mapred-site.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch('sahara.plugins.cdh.v5.edp_engine.EdpOozieEngine')
|
||||||
|
def test_get_possible_job_config_shell(self, BaseCDHEdpOozieEngine):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
BaseCDHEdpOozieEngine.get_possible_job_config.return_value = (
|
||||||
|
expected_config)
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_SHELL)
|
||||||
|
BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with(
|
||||||
|
edp.JOB_TYPE_SHELL)
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
0
sahara/tests/unit/plugins/cdh/v5_3_0/__init__.py
Normal file
0
sahara/tests/unit/plugins/cdh/v5_3_0/__init__.py
Normal file
94
sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py
Normal file
94
sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
# Copyright (c) 2015 Intel Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from sahara.plugins.cdh.v5_3_0 import edp_engine
|
||||||
|
from sahara.tests.unit import base as sahara_base
|
||||||
|
from sahara.utils import edp
|
||||||
|
|
||||||
|
|
||||||
|
class CDH53ConfigHintsTest(sahara_base.SaharaTestCase):
|
||||||
|
@mock.patch(
|
||||||
|
'sahara.plugins.cdh.confighints_helper.get_possible_hive_config_from',
|
||||||
|
return_value={})
|
||||||
|
def test_get_possible_job_config_hive(self,
|
||||||
|
get_possible_hive_config_from):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_HIVE)
|
||||||
|
get_possible_hive_config_from.assert_called_once_with(
|
||||||
|
'plugins/cdh/v5_3_0/resources/hive-site.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch('sahara.plugins.cdh.v5_3_0.edp_engine.EdpOozieEngine')
|
||||||
|
def test_get_possible_job_config_java(self, BaseCDHEdpOozieEngine):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
BaseCDHEdpOozieEngine.get_possible_job_config.return_value = (
|
||||||
|
expected_config)
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_JAVA)
|
||||||
|
BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with(
|
||||||
|
edp.JOB_TYPE_JAVA)
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'sahara.plugins.cdh.confighints_helper.'
|
||||||
|
'get_possible_mapreduce_config_from',
|
||||||
|
return_value={})
|
||||||
|
def test_get_possible_job_config_mapreduce(
|
||||||
|
self, get_possible_mapreduce_config_from):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_MAPREDUCE)
|
||||||
|
get_possible_mapreduce_config_from.assert_called_once_with(
|
||||||
|
'plugins/cdh/v5_3_0/resources/mapred-site.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'sahara.plugins.cdh.confighints_helper.'
|
||||||
|
'get_possible_mapreduce_config_from',
|
||||||
|
return_value={})
|
||||||
|
def test_get_possible_job_config_mapreduce_streaming(
|
||||||
|
self, get_possible_mapreduce_config_from):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_MAPREDUCE_STREAMING)
|
||||||
|
get_possible_mapreduce_config_from.assert_called_once_with(
|
||||||
|
'plugins/cdh/v5_3_0/resources/mapred-site.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'sahara.plugins.cdh.confighints_helper.get_possible_pig_config_from',
|
||||||
|
return_value={})
|
||||||
|
def test_get_possible_job_config_pig(self,
|
||||||
|
get_possible_pig_config_from):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_PIG)
|
||||||
|
get_possible_pig_config_from.assert_called_once_with(
|
||||||
|
'plugins/cdh/v5_3_0/resources/mapred-site.xml')
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
|
|
||||||
|
@mock.patch('sahara.plugins.cdh.v5_3_0.edp_engine.EdpOozieEngine')
|
||||||
|
def test_get_possible_job_config_shell(self, BaseCDHEdpOozieEngine):
|
||||||
|
expected_config = {'job_config': {}}
|
||||||
|
BaseCDHEdpOozieEngine.get_possible_job_config.return_value = (
|
||||||
|
expected_config)
|
||||||
|
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
|
||||||
|
edp.JOB_TYPE_SHELL)
|
||||||
|
BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with(
|
||||||
|
edp.JOB_TYPE_SHELL)
|
||||||
|
self.assertEqual(expected_config, actual_config)
|
||||||
Loading…
Reference in New Issue
Block a user