Merge "Upgrading Spark to version 2.2"
This commit is contained in:
commit
cf1a94a2c2
|
@ -0,0 +1,3 @@
|
|||
---
|
||||
features:
|
||||
- Adding Spark version 2.2 to Sahara.
|
|
@ -63,7 +63,7 @@ class SparkProvider(p.ProvisioningPluginBase):
|
|||
deprecated = {'enabled': {'status': True},
|
||||
'deprecated': {'status': True}}
|
||||
result = {'plugin_labels': copy.deepcopy(default)}
|
||||
stable_versions = ['2.1.0', '1.6.0']
|
||||
stable_versions = ['2.2', '2.1.0', '1.6.0']
|
||||
result['version_labels'] = {
|
||||
version: copy.deepcopy(
|
||||
default if version in stable_versions else deprecated
|
||||
|
@ -72,7 +72,7 @@ class SparkProvider(p.ProvisioningPluginBase):
|
|||
return result
|
||||
|
||||
def get_versions(self):
|
||||
return ['2.1.0', '1.6.0', '1.3.1']
|
||||
return ['2.2', '2.1.0', '1.6.0', '1.3.1']
|
||||
|
||||
def get_configs(self, hadoop_version):
|
||||
return c_helper.get_plugin_configs()
|
||||
|
|
|
@ -80,10 +80,18 @@ class SparkPluginTest(base.SaharaWithDbTestCase):
|
|||
self._test_engine('2.1.0', edp.JOB_TYPE_SPARK,
|
||||
engine.SparkJobEngine)
|
||||
|
||||
def test_plugin22_shell_engine(self):
|
||||
def test_plugin21_shell_engine(self):
|
||||
self._test_engine('2.1.0', edp.JOB_TYPE_SHELL,
|
||||
engine.SparkShellJobEngine)
|
||||
|
||||
def test_plugin22_edp_engine(self):
|
||||
self._test_engine('2.2', edp.JOB_TYPE_SPARK,
|
||||
engine.SparkJobEngine)
|
||||
|
||||
def test_plugin22_shell_engine(self):
|
||||
self._test_engine('2.2', edp.JOB_TYPE_SHELL,
|
||||
engine.SparkShellJobEngine)
|
||||
|
||||
def _test_engine(self, version, job_type, eng):
|
||||
cluster_dict = self._init_cluster_dict(version)
|
||||
|
||||
|
@ -201,6 +209,8 @@ class SparkProviderTest(base.SaharaTestCase):
|
|||
res['1.6.0'])
|
||||
self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
|
||||
res['2.1.0'])
|
||||
self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
|
||||
res['2.2'])
|
||||
|
||||
def test_edp_config_hints(self):
|
||||
provider = pl.SparkProvider()
|
||||
|
@ -225,6 +235,14 @@ class SparkProviderTest(base.SaharaTestCase):
|
|||
self.assertEqual({'args': [], 'configs': []},
|
||||
res['job_config'])
|
||||
|
||||
res = provider.get_edp_config_hints(edp.JOB_TYPE_SPARK, "2.1.0")
|
||||
res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.1.0")
|
||||
self.assertEqual({'args': [], 'configs': {}, 'params': {}},
|
||||
res['job_config'])
|
||||
|
||||
res = provider.get_edp_config_hints(edp.JOB_TYPE_SPARK, "2.2")
|
||||
self.assertEqual({'args': [], 'configs': []},
|
||||
res['job_config'])
|
||||
|
||||
res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.2")
|
||||
self.assertEqual({'args': [], 'configs': {}, 'params': {}},
|
||||
res['job_config'])
|
||||
|
|
Loading…
Reference in New Issue