diff --git a/sahara/plugins/cdh/v5_3_0/config_helper.py b/sahara/plugins/cdh/v5_3_0/config_helper.py index 65ba84c8..8de84d46 100644 --- a/sahara/plugins/cdh/v5_3_0/config_helper.py +++ b/sahara/plugins/cdh/v5_3_0/config_helper.py @@ -161,7 +161,7 @@ AWAIT_MANAGER_STARTING_TIMEOUT = p.Config( _default_executor_classpath = ":".join( ['/usr/lib/hadoop/lib/jackson-core-asl-1.8.8.jar', - '/usr/lib/hadoop/hadoop-swift.jar']) + '/usr/lib/hadoop-mapreduce/hadoop-openstack.jar']) EXECUTOR_EXTRA_CLASSPATH = p.Config( 'Executor extra classpath', 'Spark', 'cluster', priority=2, diff --git a/sahara/plugins/cdh/v5_4_0/config_helper.py b/sahara/plugins/cdh/v5_4_0/config_helper.py index 948debc9..a4f66b80 100644 --- a/sahara/plugins/cdh/v5_4_0/config_helper.py +++ b/sahara/plugins/cdh/v5_4_0/config_helper.py @@ -183,7 +183,7 @@ AWAIT_MANAGER_STARTING_TIMEOUT = p.Config( _default_executor_classpath = ":".join( ['/usr/lib/hadoop/lib/jackson-core-asl-1.8.8.jar', - '/usr/lib/hadoop/hadoop-swift.jar']) + '/usr/lib/hadoop-mapreduce/hadoop-openstack.jar']) EXECUTOR_EXTRA_CLASSPATH = p.Config( 'Executor extra classpath', 'Spark', 'cluster', priority=2, diff --git a/sahara/service/edp/spark/engine.py b/sahara/service/edp/spark/engine.py index fb80157f..6a090e11 100644 --- a/sahara/service/edp/spark/engine.py +++ b/sahara/service/edp/spark/engine.py @@ -241,8 +241,10 @@ class SparkJobEngine(base_engine.JobEngine): indep_params["wrapper_args"] = "%s %s" % ( wrapper_xml, indep_params["job_class"]) + indep_params["addnl_files"] = wrapper_xml + indep_params["addnl_jars"] = ",".join( - [indep_params["app_jar"]] + paths + builtin_paths) + [indep_params["wrapper_jar"]] + paths + builtin_paths) else: indep_params["addnl_jars"] = ",".join(paths) @@ -268,10 +270,11 @@ class SparkJobEngine(base_engine.JobEngine): # embedded if they are non-empty cmd = ( '%(spark-user)s%(spark-submit)s%(driver-class-path)s' + ' --files %(addnl_files)s' ' --class %(wrapper_class)s%(addnl_jars)s' ' --master %(master)s' ' --deploy-mode %(deploy-mode)s' - ' %(wrapper_jar)s %(wrapper_args)s%(args)s') % dict( + ' %(app_jar)s %(wrapper_args)s%(args)s') % dict( mutual_dict) else: cmd = ( diff --git a/sahara/tests/unit/service/edp/spark/base.py b/sahara/tests/unit/service/edp/spark/base.py index 5c89a815..8f979162 100644 --- a/sahara/tests/unit/service/edp/spark/base.py +++ b/sahara/tests/unit/service/edp/spark/base.py @@ -556,11 +556,12 @@ class TestSpark(base.SaharaTestCase): 'cd %(workflow_dir)s; ' './launch_command %(spark_user)s%(spark_submit)s ' '--driver-class-path %(driver_cp)s ' + '--files spark.xml ' '--class org.openstack.sahara.edp.SparkWrapper ' - '--jars app.jar,jar1.jar,jar2.jar ' + '--jars wrapper.jar,jar1.jar,jar2.jar ' '--master %(master)s ' '--deploy-mode %(deploy_mode)s ' - 'wrapper.jar spark.xml org.me.myclass input_arg output_arg ' + 'app.jar spark.xml org.me.myclass input_arg output_arg ' '> /dev/null 2>&1 & echo $!' % {"workflow_dir": self.workflow_dir, "spark_user": self.spark_user, "spark_submit": self.spark_submit, @@ -593,11 +594,12 @@ class TestSpark(base.SaharaTestCase): 'cd %(workflow_dir)s; ' './launch_command %(spark_user)s%(spark_submit)s ' '--driver-class-path %(driver_cp)s ' + '--files spark.xml ' '--class org.openstack.sahara.edp.SparkWrapper ' - '--jars app.jar ' + '--jars wrapper.jar ' '--master %(master)s ' '--deploy-mode %(deploy_mode)s ' - 'wrapper.jar spark.xml org.me.myclass input_arg output_arg ' + 'app.jar spark.xml org.me.myclass input_arg output_arg ' '> /dev/null 2>&1 & echo $!' % {"workflow_dir": self.workflow_dir, "spark_user": self.spark_user, "spark_submit": self.spark_submit, @@ -629,11 +631,12 @@ class TestSpark(base.SaharaTestCase): 'cd %(workflow_dir)s; ' './launch_command %(spark_user)s%(spark_submit)s ' '--driver-class-path %(driver_cp)s ' + '--files spark.xml ' '--class org.openstack.sahara.edp.SparkWrapper ' - '--jars app.jar ' + '--jars wrapper.jar ' '--master %(master)s ' '--deploy-mode %(deploy_mode)s ' - 'wrapper.jar spark.xml org.me.myclass ' + 'app.jar spark.xml org.me.myclass ' '> /dev/null 2>&1 & echo $!' % {"workflow_dir": self.workflow_dir, "spark_user": self.spark_user, "spark_submit": self.spark_submit,