From 2d0952cd4037f16aee5409ce6b9a8ef8ac787f93 Mon Sep 17 00:00:00 2001 From: Michael McCune Date: Mon, 23 Mar 2015 18:52:05 -0400 Subject: [PATCH] Adding config hints for vanilla plugin This change adds an implementation of the config hints for both versions of the vanilla plugin. Changes * adding confighints_helper module to hold utility functions for creating the config hints * adding edp engine for v2_6_0 * adding specific config hints functions for both vanilla edp_engine versions * adding tests for vanilla config hints and hints helper Change-Id: I3ae0eb94d380195708b1d4ca8f479dacf345212b Partial-Implements: bp edp-job-types-endpoint --- sahara/plugins/vanilla/confighints_helper.py | 45 +++++++++ sahara/plugins/vanilla/v1_2_1/edp_engine.py | 17 ++++ sahara/plugins/vanilla/v2_6_0/edp_engine.py | 35 +++++++ .../plugins/vanilla/v2_6_0/versionhandler.py | 2 +- .../vanilla/test_confighints_helper.py | 59 ++++++++++++ .../plugins/vanilla/v1_2_1/test_edp_engine.py | 96 +++++++++++++++++++ .../unit/plugins/vanilla/v2_6_0/__init__.py | 0 .../plugins/vanilla/v2_6_0/test_edp_engine.py | 96 +++++++++++++++++++ 8 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 sahara/plugins/vanilla/confighints_helper.py create mode 100644 sahara/plugins/vanilla/v2_6_0/edp_engine.py create mode 100644 sahara/tests/unit/plugins/vanilla/test_confighints_helper.py create mode 100644 sahara/tests/unit/plugins/vanilla/v1_2_1/test_edp_engine.py create mode 100644 sahara/tests/unit/plugins/vanilla/v2_6_0/__init__.py create mode 100644 sahara/tests/unit/plugins/vanilla/v2_6_0/test_edp_engine.py diff --git a/sahara/plugins/vanilla/confighints_helper.py b/sahara/plugins/vanilla/confighints_helper.py new file mode 100644 index 00000000..17a900c7 --- /dev/null +++ b/sahara/plugins/vanilla/confighints_helper.py @@ -0,0 +1,45 @@ +# Copyright (c) 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.service.edp.oozie.workflow_creator import workflow_factory +from sahara.utils import xmlutils + + +def get_possible_hive_config_from(file_name): + '''Return the possible configs, args, params for a Hive job.''' + config = { + 'configs': xmlutils.load_hadoop_xml_defaults(file_name), + 'params': {} + } + return config + + +def get_possible_mapreduce_config_from(file_name): + '''Return the possible configs, args, params for a MapReduce job.''' + config = { + 'configs': get_possible_pig_config_from(file_name).get('configs') + } + config['configs'] += workflow_factory.get_possible_mapreduce_configs() + return config + + +def get_possible_pig_config_from(file_name): + '''Return the possible configs, args, params for a Pig job.''' + config = { + 'configs': xmlutils.load_hadoop_xml_defaults(file_name), + 'args': [], + 'params': {} + } + return config diff --git a/sahara/plugins/vanilla/v1_2_1/edp_engine.py b/sahara/plugins/vanilla/v1_2_1/edp_engine.py index d77cc1e0..c371af7d 100644 --- a/sahara/plugins/vanilla/v1_2_1/edp_engine.py +++ b/sahara/plugins/vanilla/v1_2_1/edp_engine.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from sahara.plugins.vanilla import confighints_helper as ch_helper from sahara.plugins.vanilla import edp_engine from sahara.service.edp import hdfs_helper +from sahara.utils import edp class EdpOozieEngine(edp_engine.EdpOozieEngine): @@ -22,5 +24,20 @@ class EdpOozieEngine(edp_engine.EdpOozieEngine): def create_hdfs_dir(self, remote, dir_name): hdfs_helper.create_dir_hadoop1(remote, dir_name, self.get_hdfs_user()) + @staticmethod + def get_possible_job_config(job_type): + if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): + return {'job_config': ch_helper.get_possible_hive_config_from( + 'plugins/vanilla/v1_2_1/resources/hive-default.xml')} + if edp.compare_job_type(job_type, + edp.JOB_TYPE_MAPREDUCE, + edp.JOB_TYPE_MAPREDUCE_STREAMING): + return {'job_config': ch_helper.get_possible_mapreduce_config_from( + 'plugins/vanilla/v1_2_1/resources/mapred-default.xml')} + if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): + return {'job_config': ch_helper.get_possible_pig_config_from( + 'plugins/vanilla/v1_2_1/resources/mapred-default.xml')} + return edp_engine.EdpOozieEngine.get_possible_job_config(job_type) + def get_resource_manager_uri(self, cluster): return cluster['info']['MapReduce']['JobTracker'] diff --git a/sahara/plugins/vanilla/v2_6_0/edp_engine.py b/sahara/plugins/vanilla/v2_6_0/edp_engine.py new file mode 100644 index 00000000..5d7c208c --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/edp_engine.py @@ -0,0 +1,35 @@ +# Copyright (c) 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.vanilla import confighints_helper as ch_helper +from sahara.plugins.vanilla.hadoop2 import edp_engine +from sahara.utils import edp + + +class EdpOozieEngine(edp_engine.EdpOozieEngine): + @staticmethod + def get_possible_job_config(job_type): + if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): + return {'job_config': ch_helper.get_possible_hive_config_from( + 'plugins/vanilla/v2_6_0/resources/hive-default.xml')} + if edp.compare_job_type(job_type, + edp.JOB_TYPE_MAPREDUCE, + edp.JOB_TYPE_MAPREDUCE_STREAMING): + return {'job_config': ch_helper.get_possible_mapreduce_config_from( + 'plugins/vanilla/v2_6_0/resources/mapred-default.xml')} + if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): + return {'job_config': ch_helper.get_possible_pig_config_from( + 'plugins/vanilla/v2_6_0/resources/mapred-default.xml')} + return edp_engine.EdpOozieEngine.get_possible_job_config(job_type) diff --git a/sahara/plugins/vanilla/v2_6_0/versionhandler.py b/sahara/plugins/vanilla/v2_6_0/versionhandler.py index d0e186ed..f17068c6 100644 --- a/sahara/plugins/vanilla/v2_6_0/versionhandler.py +++ b/sahara/plugins/vanilla/v2_6_0/versionhandler.py @@ -21,12 +21,12 @@ from sahara import context from sahara.plugins import utils from sahara.plugins.vanilla import abstractversionhandler as avm from sahara.plugins.vanilla.hadoop2 import config as c -from sahara.plugins.vanilla.hadoop2 import edp_engine from sahara.plugins.vanilla.hadoop2 import run_scripts as run from sahara.plugins.vanilla.hadoop2 import scaling as sc from sahara.plugins.vanilla.hadoop2 import validation as vl from sahara.plugins.vanilla import utils as vu from sahara.plugins.vanilla.v2_4_1 import config_helper as c_helper +from sahara.plugins.vanilla.v2_6_0 import edp_engine from sahara.utils import cluster_progress_ops as cpo diff --git a/sahara/tests/unit/plugins/vanilla/test_confighints_helper.py b/sahara/tests/unit/plugins/vanilla/test_confighints_helper.py new file mode 100644 index 00000000..103464b0 --- /dev/null +++ b/sahara/tests/unit/plugins/vanilla/test_confighints_helper.py @@ -0,0 +1,59 @@ +# Copyright (c) 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.vanilla import confighints_helper as ch_helper +from sahara.tests.unit import base as sahara_base + + +class ConfigHintsHelperTest(sahara_base.SaharaTestCase): + @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + return_value=[]) + def test_get_possible_hive_config_from(self, load_hadoop_xml_defaults): + expected_config = { + 'configs': [], + 'params': {} + } + actual_config = ch_helper.get_possible_hive_config_from( + 'sample-config.xml') + load_hadoop_xml_defaults.called_once_with('sample-config.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + return_value=[]) + def test_get_possible_mapreduce_config_from( + self, load_hadoop_xml_defaults): + expected_config = { + 'configs': [], + } + actual_config = ch_helper.get_possible_mapreduce_config_from( + 'sample-config.xml') + load_hadoop_xml_defaults.called_once_with('sample-config.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + return_value=[]) + def test_get_possible_pig_config_from( + self, load_hadoop_xml_defaults): + expected_config = { + 'configs': [], + 'args': [], + 'params': {} + } + actual_config = ch_helper.get_possible_pig_config_from( + 'sample-config.xml') + load_hadoop_xml_defaults.called_once_with('sample-config.xml') + self.assertEqual(expected_config, actual_config) diff --git a/sahara/tests/unit/plugins/vanilla/v1_2_1/test_edp_engine.py b/sahara/tests/unit/plugins/vanilla/v1_2_1/test_edp_engine.py new file mode 100644 index 00000000..ba8d2bf9 --- /dev/null +++ b/sahara/tests/unit/plugins/vanilla/v1_2_1/test_edp_engine.py @@ -0,0 +1,96 @@ +# Copyright (c) 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.vanilla.v1_2_1 import edp_engine +from sahara.tests.unit import base as sahara_base +from sahara.utils import edp + + +class VanillaConfigHintsTest(sahara_base.SaharaTestCase): + @mock.patch( + 'sahara.plugins.vanilla.confighints_helper.' + 'get_possible_hive_config_from', + return_value={}) + def test_get_possible_job_config_hive( + self, get_possible_hive_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_HIVE) + get_possible_hive_config_from.assert_called_once_with( + 'plugins/vanilla/v1_2_1/resources/hive-default.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.vanilla.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine): + expected_config = {'job_config': {}} + BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_JAVA) + (BaseVanillaEdpOozieEngine.get_possible_job_config. + assert_called_once_with(edp.JOB_TYPE_JAVA)) + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.vanilla.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.vanilla.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce_streaming( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE_STREAMING) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.vanilla.confighints_helper.' + 'get_possible_pig_config_from', + return_value={}) + def test_get_possible_job_config_pig( + self, get_possible_pig_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_PIG) + get_possible_pig_config_from.assert_called_once_with( + 'plugins/vanilla/v1_2_1/resources/mapred-default.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.vanilla.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine): + expected_config = {'job_config': {}} + BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_SHELL) + (BaseVanillaEdpOozieEngine.get_possible_job_config. + assert_called_once_with(edp.JOB_TYPE_SHELL)) + self.assertEqual(expected_config, actual_config) diff --git a/sahara/tests/unit/plugins/vanilla/v2_6_0/__init__.py b/sahara/tests/unit/plugins/vanilla/v2_6_0/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/tests/unit/plugins/vanilla/v2_6_0/test_edp_engine.py b/sahara/tests/unit/plugins/vanilla/v2_6_0/test_edp_engine.py new file mode 100644 index 00000000..abfe7382 --- /dev/null +++ b/sahara/tests/unit/plugins/vanilla/v2_6_0/test_edp_engine.py @@ -0,0 +1,96 @@ +# Copyright (c) 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.vanilla.v2_6_0 import edp_engine +from sahara.tests.unit import base as sahara_base +from sahara.utils import edp + + +class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): + @mock.patch( + 'sahara.plugins.vanilla.confighints_helper.' + 'get_possible_hive_config_from', + return_value={}) + def test_get_possible_job_config_hive( + self, get_possible_hive_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_HIVE) + get_possible_hive_config_from.assert_called_once_with( + 'plugins/vanilla/v2_6_0/resources/hive-default.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine): + expected_config = {'job_config': {}} + BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_JAVA) + (BaseVanillaEdpOozieEngine.get_possible_job_config. + assert_called_once_with(edp.JOB_TYPE_JAVA)) + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.vanilla.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/vanilla/v2_6_0/resources/mapred-default.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.vanilla.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce_streaming( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE_STREAMING) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/vanilla/v2_6_0/resources/mapred-default.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.vanilla.confighints_helper.' + 'get_possible_pig_config_from', + return_value={}) + def test_get_possible_job_config_pig( + self, get_possible_pig_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_PIG) + get_possible_pig_config_from.assert_called_once_with( + 'plugins/vanilla/v2_6_0/resources/mapred-default.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine): + expected_config = {'job_config': {}} + BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_SHELL) + (BaseVanillaEdpOozieEngine.get_possible_job_config. + assert_called_once_with(edp.JOB_TYPE_SHELL)) + self.assertEqual(expected_config, actual_config)