Adding config hints for vanilla plugin

This change adds an implementation of the config hints for both versions
of the vanilla plugin.

Changes
* adding confighints_helper module to hold utility functions for
  creating the config hints
* adding edp engine for v2_6_0
* adding specific config hints functions for both vanilla edp_engine
  versions
* adding tests for vanilla config hints and hints helper

Change-Id: I3ae0eb94d380195708b1d4ca8f479dacf345212b
Partial-Implements: bp edp-job-types-endpoint

sahara/plugins/vanilla/confighints_helper.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sahara.service.edp.oozie.workflow_creator import workflow_factory
+from sahara.utils import xmlutils
+
+
+def get_possible_hive_config_from(file_name):
+    '''Return the possible configs, args, params for a Hive job.'''
+    config = {
+        'configs': xmlutils.load_hadoop_xml_defaults(file_name),
+        'params': {}
+        }
+    return config
+
+
+def get_possible_mapreduce_config_from(file_name):
+    '''Return the possible configs, args, params for a MapReduce job.'''
+    config = {
+        'configs': get_possible_pig_config_from(file_name).get('configs')
+        }
+    config['configs'] += workflow_factory.get_possible_mapreduce_configs()
+    return config
+
+
+def get_possible_pig_config_from(file_name):
+    '''Return the possible configs, args, params for a Pig job.'''
+    config = {
+        'configs': xmlutils.load_hadoop_xml_defaults(file_name),
+        'args': [],
+        'params': {}
+        }
+    return config

sahara/plugins/vanilla/v1_2_1/edp_engine.py

@@ -13,8 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from sahara.plugins.vanilla import confighints_helper as ch_helper
 from sahara.plugins.vanilla import edp_engine
 from sahara.service.edp import hdfs_helper
+from sahara.utils import edp
 
 
 class EdpOozieEngine(edp_engine.EdpOozieEngine):
@@ -22,5 +24,20 @@ class EdpOozieEngine(edp_engine.EdpOozieEngine):
     def create_hdfs_dir(self, remote, dir_name):
         hdfs_helper.create_dir_hadoop1(remote, dir_name, self.get_hdfs_user())
 
+    @staticmethod
+    def get_possible_job_config(job_type):
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
+            return {'job_config': ch_helper.get_possible_hive_config_from(
+                    'plugins/vanilla/v1_2_1/resources/hive-default.xml')}
+        if edp.compare_job_type(job_type,
+                                edp.JOB_TYPE_MAPREDUCE,
+                                edp.JOB_TYPE_MAPREDUCE_STREAMING):
+            return {'job_config': ch_helper.get_possible_mapreduce_config_from(
+                    'plugins/vanilla/v1_2_1/resources/mapred-default.xml')}
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
+            return {'job_config': ch_helper.get_possible_pig_config_from(
+                    'plugins/vanilla/v1_2_1/resources/mapred-default.xml')}
+        return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
+
     def get_resource_manager_uri(self, cluster):
         return cluster['info']['MapReduce']['JobTracker']

sahara/plugins/vanilla/v2_6_0/edp_engine.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sahara.plugins.vanilla import confighints_helper as ch_helper
+from sahara.plugins.vanilla.hadoop2 import edp_engine
+from sahara.utils import edp
+
+
+class EdpOozieEngine(edp_engine.EdpOozieEngine):
+    @staticmethod
+    def get_possible_job_config(job_type):
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
+            return {'job_config': ch_helper.get_possible_hive_config_from(
+                    'plugins/vanilla/v2_6_0/resources/hive-default.xml')}
+        if edp.compare_job_type(job_type,
+                                edp.JOB_TYPE_MAPREDUCE,
+                                edp.JOB_TYPE_MAPREDUCE_STREAMING):
+            return {'job_config': ch_helper.get_possible_mapreduce_config_from(
+                    'plugins/vanilla/v2_6_0/resources/mapred-default.xml')}
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
+            return {'job_config': ch_helper.get_possible_pig_config_from(
+                    'plugins/vanilla/v2_6_0/resources/mapred-default.xml')}
+        return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)

sahara/plugins/vanilla/v2_6_0/versionhandler.py

@@ -21,12 +21,12 @@ from sahara import context
 from sahara.plugins import utils
 from sahara.plugins.vanilla import abstractversionhandler as avm
 from sahara.plugins.vanilla.hadoop2 import config as c
-from sahara.plugins.vanilla.hadoop2 import edp_engine
 from sahara.plugins.vanilla.hadoop2 import run_scripts as run
 from sahara.plugins.vanilla.hadoop2 import scaling as sc
 from sahara.plugins.vanilla.hadoop2 import validation as vl
 from sahara.plugins.vanilla import utils as vu
 from sahara.plugins.vanilla.v2_4_1 import config_helper as c_helper
+from sahara.plugins.vanilla.v2_6_0 import edp_engine
 from sahara.utils import cluster_progress_ops as cpo
 
 

sahara/tests/unit/plugins/vanilla/test_confighints_helper.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from sahara.plugins.vanilla import confighints_helper as ch_helper
+from sahara.tests.unit import base as sahara_base
+
+
+class ConfigHintsHelperTest(sahara_base.SaharaTestCase):
+    @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
+                return_value=[])
+    def test_get_possible_hive_config_from(self, load_hadoop_xml_defaults):
+        expected_config = {
+            'configs': [],
+            'params': {}
+        }
+        actual_config = ch_helper.get_possible_hive_config_from(
+            'sample-config.xml')
+        load_hadoop_xml_defaults.called_once_with('sample-config.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
+                return_value=[])
+    def test_get_possible_mapreduce_config_from(
+            self, load_hadoop_xml_defaults):
+        expected_config = {
+            'configs': [],
+        }
+        actual_config = ch_helper.get_possible_mapreduce_config_from(
+            'sample-config.xml')
+        load_hadoop_xml_defaults.called_once_with('sample-config.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
+                return_value=[])
+    def test_get_possible_pig_config_from(
+            self, load_hadoop_xml_defaults):
+        expected_config = {
+            'configs': [],
+            'args': [],
+            'params': {}
+        }
+        actual_config = ch_helper.get_possible_pig_config_from(
+            'sample-config.xml')
+        load_hadoop_xml_defaults.called_once_with('sample-config.xml')
+        self.assertEqual(expected_config, actual_config)

sahara/tests/unit/plugins/vanilla/v1_2_1/test_edp_engine.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from sahara.plugins.vanilla.v1_2_1 import edp_engine
+from sahara.tests.unit import base as sahara_base
+from sahara.utils import edp
+
+
+class VanillaConfigHintsTest(sahara_base.SaharaTestCase):
+    @mock.patch(
+        'sahara.plugins.vanilla.confighints_helper.'
+        'get_possible_hive_config_from',
+        return_value={})
+    def test_get_possible_job_config_hive(
+            self, get_possible_hive_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_HIVE)
+        get_possible_hive_config_from.assert_called_once_with(
+            'plugins/vanilla/v1_2_1/resources/hive-default.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.plugins.vanilla.edp_engine.EdpOozieEngine')
+    def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine):
+        expected_config = {'job_config': {}}
+        BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = (
+            expected_config)
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_JAVA)
+        (BaseVanillaEdpOozieEngine.get_possible_job_config.
+            assert_called_once_with(edp.JOB_TYPE_JAVA))
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.vanilla.confighints_helper.'
+        'get_possible_mapreduce_config_from',
+        return_value={})
+    def test_get_possible_job_config_mapreduce(
+            self, get_possible_mapreduce_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_MAPREDUCE)
+        get_possible_mapreduce_config_from.assert_called_once_with(
+            'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.vanilla.confighints_helper.'
+        'get_possible_mapreduce_config_from',
+        return_value={})
+    def test_get_possible_job_config_mapreduce_streaming(
+            self, get_possible_mapreduce_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_MAPREDUCE_STREAMING)
+        get_possible_mapreduce_config_from.assert_called_once_with(
+            'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.vanilla.confighints_helper.'
+        'get_possible_pig_config_from',
+        return_value={})
+    def test_get_possible_job_config_pig(
+            self, get_possible_pig_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_PIG)
+        get_possible_pig_config_from.assert_called_once_with(
+            'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.plugins.vanilla.edp_engine.EdpOozieEngine')
+    def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine):
+        expected_config = {'job_config': {}}
+        BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = (
+            expected_config)
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_SHELL)
+        (BaseVanillaEdpOozieEngine.get_possible_job_config.
+            assert_called_once_with(edp.JOB_TYPE_SHELL))
+        self.assertEqual(expected_config, actual_config)

sahara/tests/unit/plugins/vanilla/v2_6_0/test_edp_engine.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from sahara.plugins.vanilla.v2_6_0 import edp_engine
+from sahara.tests.unit import base as sahara_base
+from sahara.utils import edp
+
+
+class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase):
+    @mock.patch(
+        'sahara.plugins.vanilla.confighints_helper.'
+        'get_possible_hive_config_from',
+        return_value={})
+    def test_get_possible_job_config_hive(
+            self, get_possible_hive_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_HIVE)
+        get_possible_hive_config_from.assert_called_once_with(
+            'plugins/vanilla/v2_6_0/resources/hive-default.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine')
+    def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine):
+        expected_config = {'job_config': {}}
+        BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = (
+            expected_config)
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_JAVA)
+        (BaseVanillaEdpOozieEngine.get_possible_job_config.
+            assert_called_once_with(edp.JOB_TYPE_JAVA))
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.vanilla.confighints_helper.'
+        'get_possible_mapreduce_config_from',
+        return_value={})
+    def test_get_possible_job_config_mapreduce(
+            self, get_possible_mapreduce_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_MAPREDUCE)
+        get_possible_mapreduce_config_from.assert_called_once_with(
+            'plugins/vanilla/v2_6_0/resources/mapred-default.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.vanilla.confighints_helper.'
+        'get_possible_mapreduce_config_from',
+        return_value={})
+    def test_get_possible_job_config_mapreduce_streaming(
+            self, get_possible_mapreduce_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_MAPREDUCE_STREAMING)
+        get_possible_mapreduce_config_from.assert_called_once_with(
+            'plugins/vanilla/v2_6_0/resources/mapred-default.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.vanilla.confighints_helper.'
+        'get_possible_pig_config_from',
+        return_value={})
+    def test_get_possible_job_config_pig(
+            self, get_possible_pig_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_PIG)
+        get_possible_pig_config_from.assert_called_once_with(
+            'plugins/vanilla/v2_6_0/resources/mapred-default.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine')
+    def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine):
+        expected_config = {'job_config': {}}
+        BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = (
+            expected_config)
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_SHELL)
+        (BaseVanillaEdpOozieEngine.get_possible_job_config.
+            assert_called_once_with(edp.JOB_TYPE_SHELL))
+        self.assertEqual(expected_config, actual_config)