From 54fc5ec94a10dcfcfcde43afe613afc595157050 Mon Sep 17 00:00:00 2001 From: luhuichun Date: Wed, 1 Apr 2015 03:01:05 +0800 Subject: [PATCH] Adding config hints for CDH plugin This change adds an implementation of the config hints for both versions of the CDH plugin. Change-Id: I5f2f9f99aae49d53606fcd3ff11c940f17f99691 Partial-Implements: bp: edp-job-types-endpoint --- sahara/plugins/cdh/confighints_helper.py | 46 +++++ sahara/plugins/cdh/v5/edp_engine.py | 17 ++ sahara/plugins/cdh/v5/resources/hive-site.xml | 61 +++++++ .../plugins/cdh/v5/resources/mapred-site.xml | 157 ++++++++++++++++++ sahara/plugins/cdh/v5_3_0/edp_engine.py | 17 ++ .../cdh/v5_3_0/resources/hive-site.xml | 61 +++++++ .../cdh/v5_3_0/resources/mapred-site.xml | 157 ++++++++++++++++++ .../plugins/cdh/test_confighints_helper.py | 59 +++++++ sahara/tests/unit/plugins/cdh/v5/__init__.py | 0 .../unit/plugins/cdh/v5/test_edp_engine.py | 94 +++++++++++ .../tests/unit/plugins/cdh/v5_3_0/__init__.py | 0 .../plugins/cdh/v5_3_0/test_edp_engine.py | 94 +++++++++++ 12 files changed, 763 insertions(+) create mode 100644 sahara/plugins/cdh/confighints_helper.py create mode 100644 sahara/plugins/cdh/v5/resources/hive-site.xml create mode 100644 sahara/plugins/cdh/v5/resources/mapred-site.xml create mode 100644 sahara/plugins/cdh/v5_3_0/resources/hive-site.xml create mode 100644 sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml create mode 100644 sahara/tests/unit/plugins/cdh/test_confighints_helper.py create mode 100644 sahara/tests/unit/plugins/cdh/v5/__init__.py create mode 100644 sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py create mode 100644 sahara/tests/unit/plugins/cdh/v5_3_0/__init__.py create mode 100644 sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py diff --git a/sahara/plugins/cdh/confighints_helper.py b/sahara/plugins/cdh/confighints_helper.py new file mode 100644 index 00000000..e613a7fe --- /dev/null +++ b/sahara/plugins/cdh/confighints_helper.py @@ -0,0 +1,46 @@ +# Copyright (c) 2015 Intel Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from sahara.service.edp.oozie.workflow_creator import workflow_factory +from sahara.utils import xmlutils + + +def get_possible_hive_config_from(file_name): + '''Return the possible configs, args, params for a Hive job.''' + config = { + 'configs': xmlutils.load_hadoop_xml_defaults(file_name), + 'params': {} + } + return config + + +def get_possible_mapreduce_config_from(file_name): + '''Return the possible configs, args, params for a MapReduce job.''' + config = { + 'configs': get_possible_pig_config_from(file_name).get('configs') + } + config['configs'] += workflow_factory.get_possible_mapreduce_configs() + return config + + +def get_possible_pig_config_from(file_name): + '''Return the possible configs, args, params for a Pig job.''' + config = { + 'configs': xmlutils.load_hadoop_xml_defaults(file_name), + 'args': [], + 'params': {} + } + return config diff --git a/sahara/plugins/cdh/v5/edp_engine.py b/sahara/plugins/cdh/v5/edp_engine.py index c744590b..867b8fb0 100644 --- a/sahara/plugins/cdh/v5/edp_engine.py +++ b/sahara/plugins/cdh/v5/edp_engine.py @@ -13,11 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from sahara.plugins.cdh import confighints_helper as ch_helper from sahara.plugins.cdh.v5 import cloudera_utils as cu from sahara.plugins import exceptions as ex from sahara.plugins import utils as u from sahara.service.edp import hdfs_helper from sahara.service.edp.oozie import engine as edp_engine +from sahara.utils import edp CU = cu.ClouderaUtilsV5() @@ -52,3 +54,18 @@ class EdpOozieEngine(edp_engine.OozieJobEngine): 'OOZIE_SERVER', '1', oo_count) super(EdpOozieEngine, self).validate_job_execution(cluster, job, data) + + @staticmethod + def get_possible_job_config(job_type): + if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): + return {'job_config': ch_helper.get_possible_hive_config_from( + 'plugins/cdh/v5/resources/hive-site.xml')} + if edp.compare_job_type(job_type, + edp.JOB_TYPE_MAPREDUCE, + edp.JOB_TYPE_MAPREDUCE_STREAMING): + return {'job_config': ch_helper.get_possible_mapreduce_config_from( + 'plugins/cdh/v5/resources/mapred-site.xml')} + if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): + return {'job_config': ch_helper.get_possible_pig_config_from( + 'plugins/cdh/v5/resources/mapred-site.xml')} + return edp_engine.OozieJobEngine.get_possible_job_config(job_type) diff --git a/sahara/plugins/cdh/v5/resources/hive-site.xml b/sahara/plugins/cdh/v5/resources/hive-site.xml new file mode 100644 index 00000000..7e70626f --- /dev/null +++ b/sahara/plugins/cdh/v5/resources/hive-site.xml @@ -0,0 +1,61 @@ + + + + + + hive.metastore.local + false + + + hive.metastore.uris + thrift://test-master-001.novalocal:9083 + + + hive.metastore.client.socket.timeout + 300 + + + hive.metastore.warehouse.dir + /user/hive/warehouse + + + hive.warehouse.subdir.inherit.perms + true + + + mapred.reduce.tasks + -1 + + + hive.exec.reducers.bytes.per.reducer + 1073741824 + + + hive.exec.copyfile.maxsize + 33554432 + + + hive.exec.reducers.max + 999 + + + hive.metastore.execute.setugi + true + + + hive.cluster.delegation.token.store.class + org.apache.hadoop.hive.thrift.MemoryTokenStore + + + hive.server2.enable.doAs + true + + + fs.hdfs.impl.disable.cache + true + + + hive.server2.use.SSL + false + + diff --git a/sahara/plugins/cdh/v5/resources/mapred-site.xml b/sahara/plugins/cdh/v5/resources/mapred-site.xml new file mode 100644 index 00000000..5228ed17 --- /dev/null +++ b/sahara/plugins/cdh/v5/resources/mapred-site.xml @@ -0,0 +1,157 @@ + + + + + + mapreduce.job.split.metainfo.maxsize + 10000000 + + + mapreduce.job.counters.max + 120 + + + mapreduce.output.fileoutputformat.compress + false + + + mapreduce.output.fileoutputformat.compress.type + BLOCK + + + mapreduce.output.fileoutputformat.compress.codec + org.apache.hadoop.io.compress.DefaultCodec + + + mapreduce.map.output.compress.codec + org.apache.hadoop.io.compress.SnappyCodec + + + mapreduce.map.output.compress + true + + + zlib.compress.level + DEFAULT_COMPRESSION + + + mapreduce.task.io.sort.factor + 64 + + + mapreduce.map.sort.spill.percent + 0.8 + + + mapreduce.reduce.shuffle.parallelcopies + 10 + + + mapreduce.task.timeout + 600000 + + + mapreduce.client.submit.file.replication + 10 + + + mapreduce.job.reduces + 1 + + + mapreduce.task.io.sort.mb + 256 + + + mapreduce.map.speculative + false + + + mapreduce.reduce.speculative + false + + + mapreduce.job.reduce.slowstart.completedmaps + 0.8 + + + mapreduce.jobhistory.address + test-master-001.novalocal:10020 + + + mapreduce.jobhistory.webapp.address + test-master-001.novalocal:19888 + + + mapreduce.jobhistory.webapp.https.address + test-master-001.novalocal:19890 + + + mapreduce.jobhistory.admin.address + test-master-001.novalocal:10033 + + + mapreduce.framework.name + yarn + + + yarn.app.mapreduce.am.staging-dir + /user + + + yarn.app.mapreduce.am.resource.mb + 1024 + + + yarn.app.mapreduce.am.resource.cpu-vcores + 1 + + + mapreduce.job.ubertask.enable + false + + + yarn.app.mapreduce.am.command-opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + mapreduce.map.java.opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + mapreduce.reduce.java.opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + yarn.app.mapreduce.am.admin.user.env + LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH + + + mapreduce.map.memory.mb + 1024 + + + mapreduce.map.cpu.vcores + 1 + + + mapreduce.reduce.memory.mb + 1024 + + + mapreduce.reduce.cpu.vcores + 1 + + + mapreduce.application.classpath + $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH + + + mapreduce.admin.user.env + LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH + + + mapreduce.shuffle.max.connections + 80 + + diff --git a/sahara/plugins/cdh/v5_3_0/edp_engine.py b/sahara/plugins/cdh/v5_3_0/edp_engine.py index 2ffc29ed..7cc4d9a0 100644 --- a/sahara/plugins/cdh/v5_3_0/edp_engine.py +++ b/sahara/plugins/cdh/v5_3_0/edp_engine.py @@ -13,11 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from sahara.plugins.cdh import confighints_helper as ch_helper from sahara.plugins.cdh.v5_3_0 import cloudera_utils as cu from sahara.plugins import exceptions as ex from sahara.plugins import utils as u from sahara.service.edp import hdfs_helper from sahara.service.edp.oozie import engine as edp_engine +from sahara.utils import edp CU = cu.ClouderaUtilsV530() @@ -52,3 +54,18 @@ class EdpOozieEngine(edp_engine.OozieJobEngine): 'OOZIE_SERVER', '1', oo_count) super(EdpOozieEngine, self).validate_job_execution(cluster, job, data) + + @staticmethod + def get_possible_job_config(job_type): + if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): + return {'job_config': ch_helper.get_possible_hive_config_from( + 'plugins/cdh/v5_3_0/resources/hive-site.xml')} + if edp.compare_job_type(job_type, + edp.JOB_TYPE_MAPREDUCE, + edp.JOB_TYPE_MAPREDUCE_STREAMING): + return {'job_config': ch_helper.get_possible_mapreduce_config_from( + 'plugins/cdh/v5_3_0/resources/mapred-site.xml')} + if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): + return {'job_config': ch_helper.get_possible_pig_config_from( + 'plugins/cdh/v5_3_0/resources/mapred-site.xml')} + return edp_engine.OozieJobEngine.get_possible_job_config(job_type) diff --git a/sahara/plugins/cdh/v5_3_0/resources/hive-site.xml b/sahara/plugins/cdh/v5_3_0/resources/hive-site.xml new file mode 100644 index 00000000..7e70626f --- /dev/null +++ b/sahara/plugins/cdh/v5_3_0/resources/hive-site.xml @@ -0,0 +1,61 @@ + + + + + + hive.metastore.local + false + + + hive.metastore.uris + thrift://test-master-001.novalocal:9083 + + + hive.metastore.client.socket.timeout + 300 + + + hive.metastore.warehouse.dir + /user/hive/warehouse + + + hive.warehouse.subdir.inherit.perms + true + + + mapred.reduce.tasks + -1 + + + hive.exec.reducers.bytes.per.reducer + 1073741824 + + + hive.exec.copyfile.maxsize + 33554432 + + + hive.exec.reducers.max + 999 + + + hive.metastore.execute.setugi + true + + + hive.cluster.delegation.token.store.class + org.apache.hadoop.hive.thrift.MemoryTokenStore + + + hive.server2.enable.doAs + true + + + fs.hdfs.impl.disable.cache + true + + + hive.server2.use.SSL + false + + diff --git a/sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml b/sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml new file mode 100644 index 00000000..5228ed17 --- /dev/null +++ b/sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml @@ -0,0 +1,157 @@ + + + + + + mapreduce.job.split.metainfo.maxsize + 10000000 + + + mapreduce.job.counters.max + 120 + + + mapreduce.output.fileoutputformat.compress + false + + + mapreduce.output.fileoutputformat.compress.type + BLOCK + + + mapreduce.output.fileoutputformat.compress.codec + org.apache.hadoop.io.compress.DefaultCodec + + + mapreduce.map.output.compress.codec + org.apache.hadoop.io.compress.SnappyCodec + + + mapreduce.map.output.compress + true + + + zlib.compress.level + DEFAULT_COMPRESSION + + + mapreduce.task.io.sort.factor + 64 + + + mapreduce.map.sort.spill.percent + 0.8 + + + mapreduce.reduce.shuffle.parallelcopies + 10 + + + mapreduce.task.timeout + 600000 + + + mapreduce.client.submit.file.replication + 10 + + + mapreduce.job.reduces + 1 + + + mapreduce.task.io.sort.mb + 256 + + + mapreduce.map.speculative + false + + + mapreduce.reduce.speculative + false + + + mapreduce.job.reduce.slowstart.completedmaps + 0.8 + + + mapreduce.jobhistory.address + test-master-001.novalocal:10020 + + + mapreduce.jobhistory.webapp.address + test-master-001.novalocal:19888 + + + mapreduce.jobhistory.webapp.https.address + test-master-001.novalocal:19890 + + + mapreduce.jobhistory.admin.address + test-master-001.novalocal:10033 + + + mapreduce.framework.name + yarn + + + yarn.app.mapreduce.am.staging-dir + /user + + + yarn.app.mapreduce.am.resource.mb + 1024 + + + yarn.app.mapreduce.am.resource.cpu-vcores + 1 + + + mapreduce.job.ubertask.enable + false + + + yarn.app.mapreduce.am.command-opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + mapreduce.map.java.opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + mapreduce.reduce.java.opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + yarn.app.mapreduce.am.admin.user.env + LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH + + + mapreduce.map.memory.mb + 1024 + + + mapreduce.map.cpu.vcores + 1 + + + mapreduce.reduce.memory.mb + 1024 + + + mapreduce.reduce.cpu.vcores + 1 + + + mapreduce.application.classpath + $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH + + + mapreduce.admin.user.env + LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH + + + mapreduce.shuffle.max.connections + 80 + + diff --git a/sahara/tests/unit/plugins/cdh/test_confighints_helper.py b/sahara/tests/unit/plugins/cdh/test_confighints_helper.py new file mode 100644 index 00000000..e3acdc7a --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/test_confighints_helper.py @@ -0,0 +1,59 @@ +# Copyright (c) 2015 Intel Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.cdh import confighints_helper as ch_helper +from sahara.tests.unit import base as sahara_base + + +class ConfigHintsHelperTest(sahara_base.SaharaTestCase): + @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + return_value=[]) + def test_get_possible_hive_config_from(self, load_hadoop_xml_defaults): + expected_config = { + 'configs': [], + 'params': {} + } + actual_config = ch_helper.get_possible_hive_config_from( + 'sample-config.xml') + load_hadoop_xml_defaults.called_once_with('sample-config.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + return_value=[]) + def test_get_possible_mapreduce_config_from( + self, load_hadoop_xml_defaults): + expected_config = { + 'configs': [], + } + actual_config = ch_helper.get_possible_mapreduce_config_from( + 'sample-config.xml') + load_hadoop_xml_defaults.called_once_with('sample-config.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + return_value=[]) + def test_get_possible_pig_config_from( + self, load_hadoop_xml_defaults): + expected_config = { + 'configs': [], + 'args': [], + 'params': {} + } + actual_config = ch_helper.get_possible_pig_config_from( + 'sample-config.xml') + load_hadoop_xml_defaults.called_once_with('sample-config.xml') + self.assertEqual(expected_config, actual_config) diff --git a/sahara/tests/unit/plugins/cdh/v5/__init__.py b/sahara/tests/unit/plugins/cdh/v5/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py b/sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py new file mode 100644 index 00000000..a86a5d08 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py @@ -0,0 +1,94 @@ +# Copyright (c) 2015 Intel Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.cdh.v5 import edp_engine +from sahara.tests.unit import base as sahara_base +from sahara.utils import edp + + +class CDH5ConfigHintsTest(sahara_base.SaharaTestCase): + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.get_possible_hive_config_from', + return_value={}) + def test_get_possible_job_config_hive(self, + get_possible_hive_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_HIVE) + get_possible_hive_config_from.assert_called_once_with( + 'plugins/cdh/v5/resources/hive-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.cdh.v5.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_java(self, BaseCDHEdpOozieEngine): + expected_config = {'job_config': {}} + BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_JAVA) + BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( + edp.JOB_TYPE_JAVA) + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/cdh/v5/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce_streaming( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE_STREAMING) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/cdh/v5/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.get_possible_pig_config_from', + return_value={}) + def test_get_possible_job_config_pig(self, + get_possible_pig_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_PIG) + get_possible_pig_config_from.assert_called_once_with( + 'plugins/cdh/v5/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.cdh.v5.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_shell(self, BaseCDHEdpOozieEngine): + expected_config = {'job_config': {}} + BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_SHELL) + BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( + edp.JOB_TYPE_SHELL) + self.assertEqual(expected_config, actual_config) diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/__init__.py b/sahara/tests/unit/plugins/cdh/v5_3_0/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py b/sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py new file mode 100644 index 00000000..5f75b162 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py @@ -0,0 +1,94 @@ +# Copyright (c) 2015 Intel Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.cdh.v5_3_0 import edp_engine +from sahara.tests.unit import base as sahara_base +from sahara.utils import edp + + +class CDH53ConfigHintsTest(sahara_base.SaharaTestCase): + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.get_possible_hive_config_from', + return_value={}) + def test_get_possible_job_config_hive(self, + get_possible_hive_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_HIVE) + get_possible_hive_config_from.assert_called_once_with( + 'plugins/cdh/v5_3_0/resources/hive-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.cdh.v5_3_0.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_java(self, BaseCDHEdpOozieEngine): + expected_config = {'job_config': {}} + BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_JAVA) + BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( + edp.JOB_TYPE_JAVA) + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/cdh/v5_3_0/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce_streaming( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE_STREAMING) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/cdh/v5_3_0/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.get_possible_pig_config_from', + return_value={}) + def test_get_possible_job_config_pig(self, + get_possible_pig_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_PIG) + get_possible_pig_config_from.assert_called_once_with( + 'plugins/cdh/v5_3_0/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.cdh.v5_3_0.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_shell(self, BaseCDHEdpOozieEngine): + expected_config = {'job_config': {}} + BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_SHELL) + BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( + edp.JOB_TYPE_SHELL) + self.assertEqual(expected_config, actual_config)