Adding config hints for HDP plugin

This changes adds HDP plugin configuration hints for both versions
(1.3.2 and 2.0.6).

Changes
* adding confighints_helper module to hold utility functions for
  creating the config hints
* adding specific config hints functions for both HDP edp_engine
  versions
* fixing inconsistencies in the oozie workflow_factory possible hints
  function
* adding tests for hdp config hints and hints helper

Change-Id: I7f85e47a4f9dfc7ccba0a5678701e5a4fb6742bb
Partial-Implements: bp edp-job-types-endpoint

sahara/plugins/hdp/confighints_helper.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_serialization import jsonutils as json
+
+from sahara.service.edp.oozie.workflow_creator import workflow_factory
+from sahara.utils import files as pkg
+
+
+def get_possible_hive_config_from(file_name):
+    '''Return the possible configs, args, params for a Hive job.'''
+    config = {
+        'configs': load_hadoop_json_for_tag(file_name, 'hive-site.xml'),
+        'params': {}
+        }
+    return config
+
+
+def get_possible_mapreduce_config_from(file_name):
+    '''Return the possible configs, args, params for a MapReduce job.'''
+    config = {
+        'configs': get_possible_pig_config_from(file_name).get('configs')
+        }
+    config['configs'] += workflow_factory.get_possible_mapreduce_configs()
+    return config
+
+
+def get_possible_pig_config_from(file_name):
+    '''Return the possible configs, args, params for a Pig job.'''
+    config = {
+        'configs': load_hadoop_json_for_tag(file_name, 'mapred-site.xml'),
+        'args': [],
+        'params': {}
+        }
+    return config
+
+
+def get_properties_for_tag(configurations, tag_name):
+    '''Get the properties for a tag
+
+    Given a list of configurations, return the properties for the named tag.
+    If the named tag cannot be found returns an empty list.
+
+    '''
+    for obj in configurations:
+        if obj.get('tag') == tag_name:
+            return obj.get('properties')
+    return []
+
+
+def load_hadoop_json_for_tag(file_name, tag_name):
+    '''Given a file name and a tag, return the configs from that tag.'''
+    full_json = load_json_file(file_name)
+    properties = get_properties_for_tag(full_json['configurations'], tag_name)
+    configs = []
+    for prop in properties:
+        configs.append({
+            'name': prop.get('name'),
+            'value': prop.get('default_value'),
+            'description': prop.get('description')
+        })
+    return configs
+
+
+def load_json_file(file_name):
+    '''Given a package relative json file name, return the json.'''
+    ftext = pkg.get_file_text(file_name)
+    loaded_json = json.loads(ftext)
+    return loaded_json

sahara/plugins/hdp/versions/version_1_3_2/edp_engine.py

@@ -13,8 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from sahara.plugins.hdp import confighints_helper as ch_helper
 from sahara.plugins.hdp import edp_engine
 from sahara.service.edp import hdfs_helper
+from sahara.utils import edp
 
 
 class EdpOozieEngine(edp_engine.EdpOozieEngine):
@@ -22,5 +24,23 @@ class EdpOozieEngine(edp_engine.EdpOozieEngine):
     def create_hdfs_dir(self, remote, dir_name):
         hdfs_helper.create_dir_hadoop1(remote, dir_name, self.get_hdfs_user())
 
+    @staticmethod
+    def get_possible_job_config(job_type):
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
+            return {'job_config': ch_helper.get_possible_hive_config_from(
+                    'plugins/hdp/versions/version_1_3_2/resources/'
+                    'ambari-config-resource.json')}
+        if edp.compare_job_type(job_type,
+                                edp.JOB_TYPE_MAPREDUCE,
+                                edp.JOB_TYPE_MAPREDUCE_STREAMING):
+            return {'job_config': ch_helper.get_possible_mapreduce_config_from(
+                    'plugins/hdp/versions/version_1_3_2/resources/'
+                    'ambari-config-resource.json')}
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
+            return {'job_config': ch_helper.get_possible_pig_config_from(
+                    'plugins/hdp/versions/version_1_3_2/resources/'
+                    'ambari-config-resource.json')}
+        return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
+
     def get_resource_manager_uri(self, cluster):
         return cluster['info']['MapReduce']['JobTracker']

sahara/plugins/hdp/versions/version_2_0_6/edp_engine.py

@@ -13,8 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from sahara.plugins.hdp import confighints_helper as ch_helper
 from sahara.plugins.hdp import edp_engine
 from sahara.service.edp import hdfs_helper
+from sahara.utils import edp
 
 
 class EdpOozieEngine(edp_engine.EdpOozieEngine):
@@ -22,5 +24,23 @@ class EdpOozieEngine(edp_engine.EdpOozieEngine):
     def create_hdfs_dir(self, remote, dir_name):
         hdfs_helper.create_dir_hadoop2(remote, dir_name, self.get_hdfs_user())
 
+    @staticmethod
+    def get_possible_job_config(job_type):
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
+            return {'job_config': ch_helper.get_possible_hive_config_from(
+                    'plugins/hdp/versions/version_2_0_6/resources/'
+                    'ambari-config-resource.json')}
+        if edp.compare_job_type(job_type,
+                                edp.JOB_TYPE_MAPREDUCE,
+                                edp.JOB_TYPE_MAPREDUCE_STREAMING):
+            return {'job_config': ch_helper.get_possible_mapreduce_config_from(
+                    'plugins/hdp/versions/version_2_0_6/resources/'
+                    'ambari-config-resource.json')}
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
+            return {'job_config': ch_helper.get_possible_pig_config_from(
+                    'plugins/hdp/versions/version_2_0_6/resources/'
+                    'ambari-config-resource.json')}
+        return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
+
     def get_resource_manager_uri(self, cluster):
         return cluster['info']['Yarn']['ResourceManager']

sahara/service/edp/oozie/workflow_creator/workflow_factory.py

@@ -308,7 +308,7 @@ def get_possible_job_config(job_type):
         return {'job_config': {'configs': [], 'args': []}}
 
     if edp.compare_job_type(job_type, edp.JOB_TYPE_SHELL):
-        return {'job_config': {'configs': [], 'params': [], 'args': []}}
+        return {'job_config': {'configs': [], 'params': {}, 'args': []}}
 
     if edp.compare_job_type(job_type,
                             edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_PIG):
@@ -316,16 +316,22 @@ def get_possible_job_config(job_type):
         cfg = xmlutils.load_hadoop_xml_defaults(
             'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
         if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE):
-            cfg += xmlutils.load_hadoop_xml_defaults(
+            cfg += get_possible_mapreduce_configs()
-                'service/edp/resources/mapred-job-config.xml')
     elif edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
         # TODO(nmakhotkin): Here we need return config based on specific plugin
         cfg = xmlutils.load_hadoop_xml_defaults(
             'plugins/vanilla/v1_2_1/resources/hive-default.xml')
 
-    # TODO(tmckay): args should be a list when bug #269968
+    config = {'configs': cfg}
-    # is fixed on the UI side
+    if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG, edp.JOB_TYPE_HIVE):
-    config = {'configs': cfg, "args": {}}
-    if not edp.compare_job_type(edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_JAVA):
         config.update({'params': {}})
+    if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
+        config.update({'args': []})
     return {'job_config': config}
+
+
+def get_possible_mapreduce_configs():
+    '''return a list of possible configuration values for map reduce jobs.'''
+    cfg = xmlutils.load_hadoop_xml_defaults(
+        'service/edp/resources/mapred-job-config.xml')
+    return cfg

sahara/tests/unit/plugins/hdp/test_confighints_helper.py

@@ -0,0 +1,147 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from sahara.plugins.hdp import confighints_helper as ch_helper
+from sahara.tests.unit import base as sahara_base
+
+
+SAMPLE_CONFIG = {
+    'configurations': [
+        {
+            'tag': 'tag1.xml',
+            'properties': [
+                {
+                    'name': 'prop1',
+                    'default_value': '1234',
+                    'description': 'the first property of tag1'
+                },
+                {
+                    'name': 'prop2',
+                    'default_value': '5678',
+                    'description': 'the second property of tag1'
+                }
+            ]
+        },
+        {
+            'tag': 'tag2.xml',
+            'properties': [
+                {
+                    'name': 'prop3',
+                    'default_value': '0000',
+                    'description': 'the first property of tag2'
+                }
+            ]
+        }
+    ]
+}
+
+
+class ConfigHintsHelperTest(sahara_base.SaharaTestCase):
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.load_hadoop_json_for_tag',
+        wraps=ch_helper.load_hadoop_json_for_tag)
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.load_json_file',
+        return_value=SAMPLE_CONFIG)
+    def test_get_possible_hive_config_from(self,
+                                           load_json_file,
+                                           load_hadoop_json_for_tag):
+        expected_config = {
+            'configs': [],
+            'params': {}
+        }
+        actual_config = ch_helper.get_possible_hive_config_from(
+            'sample-file-name.json')
+        load_hadoop_json_for_tag.assert_called_once_with(
+            'sample-file-name.json', 'hive-site.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.service.edp.oozie.workflow_creator.workflow_factory.'
+        'get_possible_mapreduce_configs',
+        return_value=[])
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.load_hadoop_json_for_tag',
+        wraps=ch_helper.load_hadoop_json_for_tag)
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.load_json_file',
+        return_value=SAMPLE_CONFIG)
+    def test_get_possible_mapreduce_config_from(self,
+                                                load_json_file,
+                                                load_hadoop_json_for_tag,
+                                                get_poss_mr_configs):
+        expected_config = {
+            'configs': []
+        }
+        actual_config = ch_helper.get_possible_mapreduce_config_from(
+            'sample-file-name.json')
+        load_hadoop_json_for_tag.assert_called_once_with(
+            'sample-file-name.json', 'mapred-site.xml')
+        get_poss_mr_configs.assert_called_once_with()
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.load_hadoop_json_for_tag',
+        wraps=ch_helper.load_hadoop_json_for_tag)
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.load_json_file',
+        return_value=SAMPLE_CONFIG)
+    def test_get_possible_pig_config_from(self,
+                                          load_json_file,
+                                          load_hadoop_json_for_tag):
+        expected_config = {
+            'configs': [],
+            'args': [],
+            'params': {}
+        }
+        actual_config = ch_helper.get_possible_pig_config_from(
+            'sample-file-name.json')
+        load_hadoop_json_for_tag.assert_called_once_with(
+            'sample-file-name.json', 'mapred-site.xml')
+        self.assertEqual(expected_config, actual_config)
+
+    def test_get_properties_for_tag(self):
+        expected_properties = [
+            {
+                'name': 'prop1',
+                'default_value': '1234',
+                'description': 'the first property of tag1'
+            },
+            {
+                'name': 'prop2',
+                'default_value': '5678',
+                'description': 'the second property of tag1'
+            }
+        ]
+        actual_properties = ch_helper.get_properties_for_tag(
+            SAMPLE_CONFIG['configurations'], 'tag1.xml')
+        self.assertEqual(expected_properties, actual_properties)
+
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.load_json_file',
+        return_value=SAMPLE_CONFIG)
+    def test_load_hadoop_json_for_tag(self, load_json_file):
+        expected_configs = [
+            {
+                'name': 'prop3',
+                'value': '0000',
+                'description': 'the first property of tag2'
+            }
+        ]
+        actual_configs = ch_helper.load_hadoop_json_for_tag(
+            'sample-file-name.json', 'tag2.xml')
+        self.assertEqual(expected_configs, actual_configs)

sahara/tests/unit/plugins/hdp/versions/version_1_3_2/test_edp_engine.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from sahara.plugins.hdp.versions.version_1_3_2 import edp_engine
+from sahara.tests.unit import base as sahara_base
+from sahara.utils import edp
+
+
+class HDPConfigHintsTest(sahara_base.SaharaTestCase):
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.get_possible_hive_config_from',
+        return_value={})
+    def test_get_possible_job_config_hive(
+            self, get_possible_hive_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_HIVE)
+        get_possible_hive_config_from.assert_called_once_with(
+            'plugins/hdp/versions/version_1_3_2/resources/'
+            'ambari-config-resource.json')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.plugins.hdp.edp_engine.EdpOozieEngine')
+    def test_get_possible_job_config_java(self, BaseHDPEdpOozieEngine):
+        expected_config = {'job_config': {}}
+        BaseHDPEdpOozieEngine.get_possible_job_config.return_value = (
+            expected_config)
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_JAVA)
+        BaseHDPEdpOozieEngine.get_possible_job_config.assert_called_once_with(
+            edp.JOB_TYPE_JAVA)
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.'
+        'get_possible_mapreduce_config_from',
+        return_value={})
+    def test_get_possible_job_config_mapreduce(
+            self, get_possible_mapreduce_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_MAPREDUCE)
+        get_possible_mapreduce_config_from.assert_called_once_with(
+            'plugins/hdp/versions/version_1_3_2/resources/'
+            'ambari-config-resource.json')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.'
+        'get_possible_mapreduce_config_from',
+        return_value={})
+    def test_get_possible_job_config_mapreduce_streaming(
+            self, get_possible_mapreduce_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_MAPREDUCE_STREAMING)
+        get_possible_mapreduce_config_from.assert_called_once_with(
+            'plugins/hdp/versions/version_1_3_2/resources/'
+            'ambari-config-resource.json')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.get_possible_pig_config_from',
+        return_value={})
+    def test_get_possible_job_config_pig(
+            self, get_possible_pig_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_PIG)
+        get_possible_pig_config_from.assert_called_once_with(
+            'plugins/hdp/versions/version_1_3_2/resources/'
+            'ambari-config-resource.json')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.plugins.hdp.edp_engine.EdpOozieEngine')
+    def test_get_possible_job_config_shell(self, BaseHDPEdpOozieEngine):
+        expected_config = {'job_config': {}}
+        BaseHDPEdpOozieEngine.get_possible_job_config.return_value = (
+            expected_config)
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_SHELL)
+        BaseHDPEdpOozieEngine.get_possible_job_config.assert_called_once_with(
+            edp.JOB_TYPE_SHELL)
+        self.assertEqual(expected_config, actual_config)

sahara/tests/unit/plugins/hdp/versions/version_2_0_6/test_edp_engine.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from sahara.plugins.hdp.versions.version_2_0_6 import edp_engine
+from sahara.tests.unit import base as sahara_base
+from sahara.utils import edp
+
+
+class HDP2ConfigHintsTest(sahara_base.SaharaTestCase):
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.get_possible_hive_config_from',
+        return_value={})
+    def test_get_possible_job_config_hive(self,
+                                          get_possible_hive_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_HIVE)
+        get_possible_hive_config_from.assert_called_once_with(
+            'plugins/hdp/versions/version_2_0_6/resources/'
+            'ambari-config-resource.json')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.plugins.hdp.edp_engine.EdpOozieEngine')
+    def test_get_possible_job_config_java(self, BaseHDPEdpOozieEngine):
+        expected_config = {'job_config': {}}
+        BaseHDPEdpOozieEngine.get_possible_job_config.return_value = (
+            expected_config)
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_JAVA)
+        BaseHDPEdpOozieEngine.get_possible_job_config.assert_called_once_with(
+            edp.JOB_TYPE_JAVA)
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.'
+        'get_possible_mapreduce_config_from',
+        return_value={})
+    def test_get_possible_job_config_mapreduce(
+            self, get_possible_mapreduce_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_MAPREDUCE)
+        get_possible_mapreduce_config_from.assert_called_once_with(
+            'plugins/hdp/versions/version_2_0_6/resources/'
+            'ambari-config-resource.json')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.'
+        'get_possible_mapreduce_config_from',
+        return_value={})
+    def test_get_possible_job_config_mapreduce_streaming(
+            self, get_possible_mapreduce_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_MAPREDUCE_STREAMING)
+        get_possible_mapreduce_config_from.assert_called_once_with(
+            'plugins/hdp/versions/version_2_0_6/resources/'
+            'ambari-config-resource.json')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch(
+        'sahara.plugins.hdp.confighints_helper.get_possible_pig_config_from',
+        return_value={})
+    def test_get_possible_job_config_pig(self,
+                                         get_possible_pig_config_from):
+        expected_config = {'job_config': {}}
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_PIG)
+        get_possible_pig_config_from.assert_called_once_with(
+            'plugins/hdp/versions/version_2_0_6/resources/'
+            'ambari-config-resource.json')
+        self.assertEqual(expected_config, actual_config)
+
+    @mock.patch('sahara.plugins.hdp.edp_engine.EdpOozieEngine')
+    def test_get_possible_job_config_shell(self, BaseHDPEdpOozieEngine):
+        expected_config = {'job_config': {}}
+        BaseHDPEdpOozieEngine.get_possible_job_config.return_value = (
+            expected_config)
+        actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
+            edp.JOB_TYPE_SHELL)
+        BaseHDPEdpOozieEngine.get_possible_job_config.assert_called_once_with(
+            edp.JOB_TYPE_SHELL)
+        self.assertEqual(expected_config, actual_config)