openstack/deb-sahara
Code Issues Proposed changes

Adding config hints for HDP plugin

Browse Source 
This changes adds HDP plugin configuration hints for both versions
(1.3.2 and 2.0.6).

Changes
* adding confighints_helper module to hold utility functions for
  creating the config hints
* adding specific config hints functions for both HDP edp_engine
  versions
* fixing inconsistencies in the oozie workflow_factory possible hints
  function
* adding tests for hdp config hints and hints helper

Change-Id: I7f85e47a4f9dfc7ccba0a5678701e5a4fb6742bb
Partial-Implements: bp edp-job-types-endpoint
This commit is contained in:
Michael McCune 2015-03-20 19:00:16 -04:00
parent 882328204c
commit 4ca232ce72
10 changed files with 477 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
sahara/plugins/hdp/confighints_helper.py Normal file
View File

@ -0,0 +1,81 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_serialization import jsonutils as json
from sahara.service.edp.oozie.workflow_creator import workflow_factory
from sahara.utils import files as pkg
def get_possible_hive_config_from(file_name):
'''Return the possible configs, args, params for a Hive job.'''
config = {
'configs': load_hadoop_json_for_tag(file_name, 'hive-site.xml'),
'params': {}
}
return config
def get_possible_mapreduce_config_from(file_name):
'''Return the possible configs, args, params for a MapReduce job.'''
config = {
'configs': get_possible_pig_config_from(file_name).get('configs')
}
config['configs'] += workflow_factory.get_possible_mapreduce_configs()
return config
def get_possible_pig_config_from(file_name):
'''Return the possible configs, args, params for a Pig job.'''
config = {
'configs': load_hadoop_json_for_tag(file_name, 'mapred-site.xml'),
'args': [],
'params': {}
}
return config
def get_properties_for_tag(configurations, tag_name):
'''Get the properties for a tag
Given a list of configurations, return the properties for the named tag.
If the named tag cannot be found returns an empty list.
'''
for obj in configurations:
if obj.get('tag') == tag_name:
return obj.get('properties')
return []
def load_hadoop_json_for_tag(file_name, tag_name):
'''Given a file name and a tag, return the configs from that tag.'''
full_json = load_json_file(file_name)
properties = get_properties_for_tag(full_json['configurations'], tag_name)
configs = []
for prop in properties:
configs.append({
'name': prop.get('name'),
'value': prop.get('default_value'),
'description': prop.get('description')
})
return configs
def load_json_file(file_name):
'''Given a package relative json file name, return the json.'''
ftext = pkg.get_file_text(file_name)
loaded_json = json.loads(ftext)
return loaded_json

20
sahara/plugins/hdp/versions/version_1_3_2/edp_engine.py
View File

@ -13,8 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.hdp import confighints_helper as ch_helper
from sahara.plugins.hdp import edp_engine
from sahara.service.edp import hdfs_helper
from sahara.utils import edp
class EdpOozieEngine(edp_engine.EdpOozieEngine):
@ -22,5 +24,23 @@ class EdpOozieEngine(edp_engine.EdpOozieEngine):
def create_hdfs_dir(self, remote, dir_name):
hdfs_helper.create_dir_hadoop1(remote, dir_name, self.get_hdfs_user())
@staticmethod
def get_possible_job_config(job_type):
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
return {'job_config': ch_helper.get_possible_hive_config_from(
'plugins/hdp/versions/version_1_3_2/resources/'
'ambari-config-resource.json')}
if edp.compare_job_type(job_type,
edp.JOB_TYPE_MAPREDUCE,
edp.JOB_TYPE_MAPREDUCE_STREAMING):
return {'job_config': ch_helper.get_possible_mapreduce_config_from(
'plugins/hdp/versions/version_1_3_2/resources/'
'ambari-config-resource.json')}
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
return {'job_config': ch_helper.get_possible_pig_config_from(
'plugins/hdp/versions/version_1_3_2/resources/'
'ambari-config-resource.json')}
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
def get_resource_manager_uri(self, cluster):
return cluster['info']['MapReduce']['JobTracker']

20
sahara/plugins/hdp/versions/version_2_0_6/edp_engine.py
View File

@ -13,8 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.hdp import confighints_helper as ch_helper
from sahara.plugins.hdp import edp_engine
from sahara.service.edp import hdfs_helper
from sahara.utils import edp
class EdpOozieEngine(edp_engine.EdpOozieEngine):
@ -22,5 +24,23 @@ class EdpOozieEngine(edp_engine.EdpOozieEngine):
def create_hdfs_dir(self, remote, dir_name):
hdfs_helper.create_dir_hadoop2(remote, dir_name, self.get_hdfs_user())
@staticmethod
def get_possible_job_config(job_type):
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
return {'job_config': ch_helper.get_possible_hive_config_from(
'plugins/hdp/versions/version_2_0_6/resources/'
'ambari-config-resource.json')}
if edp.compare_job_type(job_type,
edp.JOB_TYPE_MAPREDUCE,
edp.JOB_TYPE_MAPREDUCE_STREAMING):
return {'job_config': ch_helper.get_possible_mapreduce_config_from(
'plugins/hdp/versions/version_2_0_6/resources/'
'ambari-config-resource.json')}
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
return {'job_config': ch_helper.get_possible_pig_config_from(
'plugins/hdp/versions/version_2_0_6/resources/'
'ambari-config-resource.json')}
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
def get_resource_manager_uri(self, cluster):
return cluster['info']['Yarn']['ResourceManager']

20
sahara/service/edp/oozie/workflow_creator/workflow_factory.py
View File

@ -308,7 +308,7 @@ def get_possible_job_config(job_type):
return {'job_config': {'configs': [], 'args': []}}
if edp.compare_job_type(job_type, edp.JOB_TYPE_SHELL):
return {'job_config': {'configs': [], 'params': [], 'args': []}}
return {'job_config': {'configs': [], 'params': {}, 'args': []}}
if edp.compare_job_type(job_type,
edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_PIG):
@ -316,16 +316,22 @@ def get_possible_job_config(job_type):
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE):
cfg += xmlutils.load_hadoop_xml_defaults(
'service/edp/resources/mapred-job-config.xml')
cfg += get_possible_mapreduce_configs()
elif edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
# TODO(nmakhotkin): Here we need return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/v1_2_1/resources/hive-default.xml')
# TODO(tmckay): args should be a list when bug #269968
# is fixed on the UI side
config = {'configs': cfg, "args": {}}
if not edp.compare_job_type(edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_JAVA):
config = {'configs': cfg}
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG, edp.JOB_TYPE_HIVE):
config.update({'params': {}})
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
config.update({'args': []})
return {'job_config': config}
def get_possible_mapreduce_configs():
'''return a list of possible configuration values for map reduce jobs.'''
cfg = xmlutils.load_hadoop_xml_defaults(
'service/edp/resources/mapred-job-config.xml')
return cfg

147
sahara/tests/unit/plugins/hdp/test_confighints_helper.py Normal file
View File

@ -0,0 +1,147 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from sahara.plugins.hdp import confighints_helper as ch_helper
from sahara.tests.unit import base as sahara_base
SAMPLE_CONFIG = {
'configurations': [
{
'tag': 'tag1.xml',
'properties': [
{
'name': 'prop1',
'default_value': '1234',
'description': 'the first property of tag1'
},
{
'name': 'prop2',
'default_value': '5678',
'description': 'the second property of tag1'
}
]
},
{
'tag': 'tag2.xml',
'properties': [
{
'name': 'prop3',
'default_value': '0000',
'description': 'the first property of tag2'
}
]
}
]
}
class ConfigHintsHelperTest(sahara_base.SaharaTestCase):
@mock.patch(
'sahara.plugins.hdp.confighints_helper.load_hadoop_json_for_tag',
wraps=ch_helper.load_hadoop_json_for_tag)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.load_json_file',
return_value=SAMPLE_CONFIG)
def test_get_possible_hive_config_from(self,
load_json_file,
load_hadoop_json_for_tag):
expected_config = {
'configs': [],
'params': {}
}
actual_config = ch_helper.get_possible_hive_config_from(
'sample-file-name.json')
load_hadoop_json_for_tag.assert_called_once_with(
'sample-file-name.json', 'hive-site.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.service.edp.oozie.workflow_creator.workflow_factory.'
'get_possible_mapreduce_configs',
return_value=[])
@mock.patch(
'sahara.plugins.hdp.confighints_helper.load_hadoop_json_for_tag',
wraps=ch_helper.load_hadoop_json_for_tag)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.load_json_file',
return_value=SAMPLE_CONFIG)
def test_get_possible_mapreduce_config_from(self,
load_json_file,
load_hadoop_json_for_tag,
get_poss_mr_configs):
expected_config = {
'configs': []
}
actual_config = ch_helper.get_possible_mapreduce_config_from(
'sample-file-name.json')
load_hadoop_json_for_tag.assert_called_once_with(
'sample-file-name.json', 'mapred-site.xml')
get_poss_mr_configs.assert_called_once_with()
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.load_hadoop_json_for_tag',
wraps=ch_helper.load_hadoop_json_for_tag)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.load_json_file',
return_value=SAMPLE_CONFIG)
def test_get_possible_pig_config_from(self,
load_json_file,
load_hadoop_json_for_tag):
expected_config = {
'configs': [],
'args': [],
'params': {}
}
actual_config = ch_helper.get_possible_pig_config_from(
'sample-file-name.json')
load_hadoop_json_for_tag.assert_called_once_with(
'sample-file-name.json', 'mapred-site.xml')
self.assertEqual(expected_config, actual_config)
def test_get_properties_for_tag(self):
expected_properties = [
{
'name': 'prop1',
'default_value': '1234',
'description': 'the first property of tag1'
},
{
'name': 'prop2',
'default_value': '5678',
'description': 'the second property of tag1'
}
]
actual_properties = ch_helper.get_properties_for_tag(
SAMPLE_CONFIG['configurations'], 'tag1.xml')
self.assertEqual(expected_properties, actual_properties)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.load_json_file',
return_value=SAMPLE_CONFIG)
def test_load_hadoop_json_for_tag(self, load_json_file):
expected_configs = [
{
'name': 'prop3',
'value': '0000',
'description': 'the first property of tag2'
}
]
actual_configs = ch_helper.load_hadoop_json_for_tag(
'sample-file-name.json', 'tag2.xml')
self.assertEqual(expected_configs, actual_configs)

0
sahara/tests/unit/plugins/hdp/versions/__init__.py Normal file
View File

0
sahara/tests/unit/plugins/hdp/versions/version_1_3_2/__init__.py Normal file
View File

98
sahara/tests/unit/plugins/hdp/versions/version_1_3_2/test_edp_engine.py Normal file
View File

@ -0,0 +1,98 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from sahara.plugins.hdp.versions.version_1_3_2 import edp_engine
from sahara.tests.unit import base as sahara_base
from sahara.utils import edp
class HDPConfigHintsTest(sahara_base.SaharaTestCase):
@mock.patch(
'sahara.plugins.hdp.confighints_helper.get_possible_hive_config_from',
return_value={})
def test_get_possible_job_config_hive(
self, get_possible_hive_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_HIVE)
get_possible_hive_config_from.assert_called_once_with(
'plugins/hdp/versions/version_1_3_2/resources/'
'ambari-config-resource.json')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.plugins.hdp.edp_engine.EdpOozieEngine')
def test_get_possible_job_config_java(self, BaseHDPEdpOozieEngine):
expected_config = {'job_config': {}}
BaseHDPEdpOozieEngine.get_possible_job_config.return_value = (
expected_config)
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_JAVA)
BaseHDPEdpOozieEngine.get_possible_job_config.assert_called_once_with(
edp.JOB_TYPE_JAVA)
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.'
'get_possible_mapreduce_config_from',
return_value={})
def test_get_possible_job_config_mapreduce(
self, get_possible_mapreduce_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_MAPREDUCE)
get_possible_mapreduce_config_from.assert_called_once_with(
'plugins/hdp/versions/version_1_3_2/resources/'
'ambari-config-resource.json')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.'
'get_possible_mapreduce_config_from',
return_value={})
def test_get_possible_job_config_mapreduce_streaming(
self, get_possible_mapreduce_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_MAPREDUCE_STREAMING)
get_possible_mapreduce_config_from.assert_called_once_with(
'plugins/hdp/versions/version_1_3_2/resources/'
'ambari-config-resource.json')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.get_possible_pig_config_from',
return_value={})
def test_get_possible_job_config_pig(
self, get_possible_pig_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_PIG)
get_possible_pig_config_from.assert_called_once_with(
'plugins/hdp/versions/version_1_3_2/resources/'
'ambari-config-resource.json')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.plugins.hdp.edp_engine.EdpOozieEngine')
def test_get_possible_job_config_shell(self, BaseHDPEdpOozieEngine):
expected_config = {'job_config': {}}
BaseHDPEdpOozieEngine.get_possible_job_config.return_value = (
expected_config)
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_SHELL)
BaseHDPEdpOozieEngine.get_possible_job_config.assert_called_once_with(
edp.JOB_TYPE_SHELL)
self.assertEqual(expected_config, actual_config)

0
sahara/tests/unit/plugins/hdp/versions/version_2_0_6/__init__.py Normal file
View File

98
sahara/tests/unit/plugins/hdp/versions/version_2_0_6/test_edp_engine.py Normal file
View File

@ -0,0 +1,98 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from sahara.plugins.hdp.versions.version_2_0_6 import edp_engine
from sahara.tests.unit import base as sahara_base
from sahara.utils import edp
class HDP2ConfigHintsTest(sahara_base.SaharaTestCase):
@mock.patch(
'sahara.plugins.hdp.confighints_helper.get_possible_hive_config_from',
return_value={})
def test_get_possible_job_config_hive(self,
get_possible_hive_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_HIVE)
get_possible_hive_config_from.assert_called_once_with(
'plugins/hdp/versions/version_2_0_6/resources/'
'ambari-config-resource.json')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.plugins.hdp.edp_engine.EdpOozieEngine')
def test_get_possible_job_config_java(self, BaseHDPEdpOozieEngine):
expected_config = {'job_config': {}}
BaseHDPEdpOozieEngine.get_possible_job_config.return_value = (
expected_config)
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_JAVA)
BaseHDPEdpOozieEngine.get_possible_job_config.assert_called_once_with(
edp.JOB_TYPE_JAVA)
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.'
'get_possible_mapreduce_config_from',
return_value={})
def test_get_possible_job_config_mapreduce(
self, get_possible_mapreduce_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_MAPREDUCE)
get_possible_mapreduce_config_from.assert_called_once_with(
'plugins/hdp/versions/version_2_0_6/resources/'
'ambari-config-resource.json')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.'
'get_possible_mapreduce_config_from',
return_value={})
def test_get_possible_job_config_mapreduce_streaming(
self, get_possible_mapreduce_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_MAPREDUCE_STREAMING)
get_possible_mapreduce_config_from.assert_called_once_with(
'plugins/hdp/versions/version_2_0_6/resources/'
'ambari-config-resource.json')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.hdp.confighints_helper.get_possible_pig_config_from',
return_value={})
def test_get_possible_job_config_pig(self,
get_possible_pig_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_PIG)
get_possible_pig_config_from.assert_called_once_with(
'plugins/hdp/versions/version_2_0_6/resources/'
'ambari-config-resource.json')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.plugins.hdp.edp_engine.EdpOozieEngine')
def test_get_possible_job_config_shell(self, BaseHDPEdpOozieEngine):
expected_config = {'job_config': {}}
BaseHDPEdpOozieEngine.get_possible_job_config.return_value = (
expected_config)
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_SHELL)
BaseHDPEdpOozieEngine.get_possible_job_config.assert_called_once_with(
edp.JOB_TYPE_SHELL)
self.assertEqual(expected_config, actual_config)