Adding config hints for vanilla plugin

This change adds an implementation of the config hints for both versions
of the vanilla plugin.

Changes
* adding confighints_helper module to hold utility functions for
  creating the config hints
* adding edp engine for v2_6_0
* adding specific config hints functions for both vanilla edp_engine
  versions
* adding tests for vanilla config hints and hints helper

Change-Id: I3ae0eb94d380195708b1d4ca8f479dacf345212b
Partial-Implements: bp edp-job-types-endpoint
This commit is contained in:
Michael McCune 2015-03-23 18:52:05 -04:00
parent 4ca232ce72
commit 2d0952cd40
8 changed files with 349 additions and 1 deletions

View File

@ -0,0 +1,45 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.service.edp.oozie.workflow_creator import workflow_factory
from sahara.utils import xmlutils
def get_possible_hive_config_from(file_name):
'''Return the possible configs, args, params for a Hive job.'''
config = {
'configs': xmlutils.load_hadoop_xml_defaults(file_name),
'params': {}
}
return config
def get_possible_mapreduce_config_from(file_name):
'''Return the possible configs, args, params for a MapReduce job.'''
config = {
'configs': get_possible_pig_config_from(file_name).get('configs')
}
config['configs'] += workflow_factory.get_possible_mapreduce_configs()
return config
def get_possible_pig_config_from(file_name):
'''Return the possible configs, args, params for a Pig job.'''
config = {
'configs': xmlutils.load_hadoop_xml_defaults(file_name),
'args': [],
'params': {}
}
return config

View File

@ -13,8 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.vanilla import confighints_helper as ch_helper
from sahara.plugins.vanilla import edp_engine
from sahara.service.edp import hdfs_helper
from sahara.utils import edp
class EdpOozieEngine(edp_engine.EdpOozieEngine):
@ -22,5 +24,20 @@ class EdpOozieEngine(edp_engine.EdpOozieEngine):
def create_hdfs_dir(self, remote, dir_name):
hdfs_helper.create_dir_hadoop1(remote, dir_name, self.get_hdfs_user())
@staticmethod
def get_possible_job_config(job_type):
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
return {'job_config': ch_helper.get_possible_hive_config_from(
'plugins/vanilla/v1_2_1/resources/hive-default.xml')}
if edp.compare_job_type(job_type,
edp.JOB_TYPE_MAPREDUCE,
edp.JOB_TYPE_MAPREDUCE_STREAMING):
return {'job_config': ch_helper.get_possible_mapreduce_config_from(
'plugins/vanilla/v1_2_1/resources/mapred-default.xml')}
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
return {'job_config': ch_helper.get_possible_pig_config_from(
'plugins/vanilla/v1_2_1/resources/mapred-default.xml')}
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
def get_resource_manager_uri(self, cluster):
return cluster['info']['MapReduce']['JobTracker']

View File

@ -0,0 +1,35 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.vanilla import confighints_helper as ch_helper
from sahara.plugins.vanilla.hadoop2 import edp_engine
from sahara.utils import edp
class EdpOozieEngine(edp_engine.EdpOozieEngine):
@staticmethod
def get_possible_job_config(job_type):
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
return {'job_config': ch_helper.get_possible_hive_config_from(
'plugins/vanilla/v2_6_0/resources/hive-default.xml')}
if edp.compare_job_type(job_type,
edp.JOB_TYPE_MAPREDUCE,
edp.JOB_TYPE_MAPREDUCE_STREAMING):
return {'job_config': ch_helper.get_possible_mapreduce_config_from(
'plugins/vanilla/v2_6_0/resources/mapred-default.xml')}
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
return {'job_config': ch_helper.get_possible_pig_config_from(
'plugins/vanilla/v2_6_0/resources/mapred-default.xml')}
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)

View File

@ -21,12 +21,12 @@ from sahara import context
from sahara.plugins import utils
from sahara.plugins.vanilla import abstractversionhandler as avm
from sahara.plugins.vanilla.hadoop2 import config as c
from sahara.plugins.vanilla.hadoop2 import edp_engine
from sahara.plugins.vanilla.hadoop2 import run_scripts as run
from sahara.plugins.vanilla.hadoop2 import scaling as sc
from sahara.plugins.vanilla.hadoop2 import validation as vl
from sahara.plugins.vanilla import utils as vu
from sahara.plugins.vanilla.v2_4_1 import config_helper as c_helper
from sahara.plugins.vanilla.v2_6_0 import edp_engine
from sahara.utils import cluster_progress_ops as cpo

View File

@ -0,0 +1,59 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from sahara.plugins.vanilla import confighints_helper as ch_helper
from sahara.tests.unit import base as sahara_base
class ConfigHintsHelperTest(sahara_base.SaharaTestCase):
@mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
return_value=[])
def test_get_possible_hive_config_from(self, load_hadoop_xml_defaults):
expected_config = {
'configs': [],
'params': {}
}
actual_config = ch_helper.get_possible_hive_config_from(
'sample-config.xml')
load_hadoop_xml_defaults.called_once_with('sample-config.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
return_value=[])
def test_get_possible_mapreduce_config_from(
self, load_hadoop_xml_defaults):
expected_config = {
'configs': [],
}
actual_config = ch_helper.get_possible_mapreduce_config_from(
'sample-config.xml')
load_hadoop_xml_defaults.called_once_with('sample-config.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults',
return_value=[])
def test_get_possible_pig_config_from(
self, load_hadoop_xml_defaults):
expected_config = {
'configs': [],
'args': [],
'params': {}
}
actual_config = ch_helper.get_possible_pig_config_from(
'sample-config.xml')
load_hadoop_xml_defaults.called_once_with('sample-config.xml')
self.assertEqual(expected_config, actual_config)

View File

@ -0,0 +1,96 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from sahara.plugins.vanilla.v1_2_1 import edp_engine
from sahara.tests.unit import base as sahara_base
from sahara.utils import edp
class VanillaConfigHintsTest(sahara_base.SaharaTestCase):
@mock.patch(
'sahara.plugins.vanilla.confighints_helper.'
'get_possible_hive_config_from',
return_value={})
def test_get_possible_job_config_hive(
self, get_possible_hive_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_HIVE)
get_possible_hive_config_from.assert_called_once_with(
'plugins/vanilla/v1_2_1/resources/hive-default.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.plugins.vanilla.edp_engine.EdpOozieEngine')
def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine):
expected_config = {'job_config': {}}
BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = (
expected_config)
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_JAVA)
(BaseVanillaEdpOozieEngine.get_possible_job_config.
assert_called_once_with(edp.JOB_TYPE_JAVA))
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.vanilla.confighints_helper.'
'get_possible_mapreduce_config_from',
return_value={})
def test_get_possible_job_config_mapreduce(
self, get_possible_mapreduce_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_MAPREDUCE)
get_possible_mapreduce_config_from.assert_called_once_with(
'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.vanilla.confighints_helper.'
'get_possible_mapreduce_config_from',
return_value={})
def test_get_possible_job_config_mapreduce_streaming(
self, get_possible_mapreduce_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_MAPREDUCE_STREAMING)
get_possible_mapreduce_config_from.assert_called_once_with(
'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.vanilla.confighints_helper.'
'get_possible_pig_config_from',
return_value={})
def test_get_possible_job_config_pig(
self, get_possible_pig_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_PIG)
get_possible_pig_config_from.assert_called_once_with(
'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.plugins.vanilla.edp_engine.EdpOozieEngine')
def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine):
expected_config = {'job_config': {}}
BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = (
expected_config)
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_SHELL)
(BaseVanillaEdpOozieEngine.get_possible_job_config.
assert_called_once_with(edp.JOB_TYPE_SHELL))
self.assertEqual(expected_config, actual_config)

View File

@ -0,0 +1,96 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from sahara.plugins.vanilla.v2_6_0 import edp_engine
from sahara.tests.unit import base as sahara_base
from sahara.utils import edp
class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase):
@mock.patch(
'sahara.plugins.vanilla.confighints_helper.'
'get_possible_hive_config_from',
return_value={})
def test_get_possible_job_config_hive(
self, get_possible_hive_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_HIVE)
get_possible_hive_config_from.assert_called_once_with(
'plugins/vanilla/v2_6_0/resources/hive-default.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine')
def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine):
expected_config = {'job_config': {}}
BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = (
expected_config)
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_JAVA)
(BaseVanillaEdpOozieEngine.get_possible_job_config.
assert_called_once_with(edp.JOB_TYPE_JAVA))
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.vanilla.confighints_helper.'
'get_possible_mapreduce_config_from',
return_value={})
def test_get_possible_job_config_mapreduce(
self, get_possible_mapreduce_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_MAPREDUCE)
get_possible_mapreduce_config_from.assert_called_once_with(
'plugins/vanilla/v2_6_0/resources/mapred-default.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.vanilla.confighints_helper.'
'get_possible_mapreduce_config_from',
return_value={})
def test_get_possible_job_config_mapreduce_streaming(
self, get_possible_mapreduce_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_MAPREDUCE_STREAMING)
get_possible_mapreduce_config_from.assert_called_once_with(
'plugins/vanilla/v2_6_0/resources/mapred-default.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch(
'sahara.plugins.vanilla.confighints_helper.'
'get_possible_pig_config_from',
return_value={})
def test_get_possible_job_config_pig(
self, get_possible_pig_config_from):
expected_config = {'job_config': {}}
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_PIG)
get_possible_pig_config_from.assert_called_once_with(
'plugins/vanilla/v2_6_0/resources/mapred-default.xml')
self.assertEqual(expected_config, actual_config)
@mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine')
def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine):
expected_config = {'job_config': {}}
BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = (
expected_config)
actual_config = edp_engine.EdpOozieEngine.get_possible_job_config(
edp.JOB_TYPE_SHELL)
(BaseVanillaEdpOozieEngine.get_possible_job_config.
assert_called_once_with(edp.JOB_TYPE_SHELL))
self.assertEqual(expected_config, actual_config)