Merge "Updating Spark versions"
This commit is contained in:
commit
2a6ddfee99
|
@ -0,0 +1,3 @@
|
|||
---
|
||||
upgrade:
|
||||
- Adding Spark 2.3 to supported plugins list.
|
|
@ -26,7 +26,7 @@ from sahara.service.edp.spark import engine as edp_engine
|
|||
|
||||
class EdpEngine(edp_engine.SparkJobEngine):
|
||||
|
||||
edp_base_version = "1.3.1"
|
||||
edp_base_version = "1.6.0"
|
||||
|
||||
def __init__(self, cluster):
|
||||
super(EdpEngine, self).__init__(cluster)
|
||||
|
|
|
@ -63,7 +63,7 @@ class SparkProvider(p.ProvisioningPluginBase):
|
|||
deprecated = {'enabled': {'status': True},
|
||||
'deprecated': {'status': True}}
|
||||
result = {'plugin_labels': copy.deepcopy(default)}
|
||||
stable_versions = ['2.2']
|
||||
stable_versions = ['2.3', '2.2']
|
||||
result['version_labels'] = {
|
||||
version: copy.deepcopy(
|
||||
default if version in stable_versions else deprecated
|
||||
|
@ -72,7 +72,7 @@ class SparkProvider(p.ProvisioningPluginBase):
|
|||
return result
|
||||
|
||||
def get_versions(self):
|
||||
return ['2.2', '2.1.0', '1.6.0']
|
||||
return ['2.3', '2.2', '2.1.0', '1.6.0']
|
||||
|
||||
def get_configs(self, hadoop_version):
|
||||
return c_helper.get_plugin_configs()
|
||||
|
|
|
@ -68,6 +68,14 @@ class SparkPluginTest(base.SaharaWithDbTestCase):
|
|||
self._test_engine('2.2', edp.JOB_TYPE_SHELL,
|
||||
engine.SparkShellJobEngine)
|
||||
|
||||
def test_plugin23_edp_engine(self):
|
||||
self._test_engine('2.3', edp.JOB_TYPE_SPARK,
|
||||
engine.SparkJobEngine)
|
||||
|
||||
def test_plugin23_shell_engine(self):
|
||||
self._test_engine('2.3', edp.JOB_TYPE_SHELL,
|
||||
engine.SparkShellJobEngine)
|
||||
|
||||
def _test_engine(self, version, job_type, eng):
|
||||
cluster_dict = self._init_cluster_dict(version)
|
||||
|
||||
|
@ -172,6 +180,8 @@ class SparkProviderTest(base.SaharaTestCase):
|
|||
res['2.1.0'])
|
||||
self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
|
||||
res['2.2'])
|
||||
self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
|
||||
res['2.3'])
|
||||
|
||||
def test_edp_config_hints(self):
|
||||
provider = pl.SparkProvider()
|
||||
|
@ -199,3 +209,11 @@ class SparkProviderTest(base.SaharaTestCase):
|
|||
res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.2")
|
||||
self.assertEqual({'args': [], 'configs': {}, 'params': {}},
|
||||
res['job_config'])
|
||||
|
||||
res = provider.get_edp_config_hints(edp.JOB_TYPE_SPARK, "2.3")
|
||||
self.assertEqual({'args': [], 'configs': []},
|
||||
res['job_config'])
|
||||
|
||||
res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.3")
|
||||
self.assertEqual({'args': [], 'configs': {}, 'params': {}},
|
||||
res['job_config'])
|
||||
|
|
Loading…
Reference in New Issue