Merge "Follow the argument order specified in spark-submit help"
This commit is contained in:
commit
a8f3363777
@ -143,7 +143,7 @@ class SparkJobEngine(base_engine.JobEngine):
|
||||
# The rest of the paths will be passed with --jars
|
||||
additional_jars = ",".join(paths)
|
||||
if additional_jars:
|
||||
additional_jars = "--jars " + additional_jars
|
||||
additional_jars = " --jars " + additional_jars
|
||||
|
||||
# Launch the spark job using spark-submit and deploy_mode = client
|
||||
host = master.hostname()
|
||||
@ -159,15 +159,17 @@ class SparkJobEngine(base_engine.JobEngine):
|
||||
# TODO(tmckay): we need to clean up wf_dirs on long running clusters
|
||||
# TODO(tmckay): probably allow for general options to spark-submit
|
||||
args = " ".join(updated_job_configs.get('args', []))
|
||||
if args:
|
||||
args = " " + args
|
||||
|
||||
# The redirects of stdout and stderr will preserve output in the wf_dir
|
||||
cmd = "%s %s --class %s %s --master spark://%s:%s %s" % (
|
||||
cmd = "%s --class %s%s --master spark://%s:%s %s%s" % (
|
||||
spark_submit,
|
||||
app_jar,
|
||||
job_class,
|
||||
additional_jars,
|
||||
host,
|
||||
port,
|
||||
app_jar,
|
||||
args)
|
||||
|
||||
job_execution = conductor.job_execution_get(ctx, job_execution.id)
|
||||
|
@ -356,9 +356,9 @@ class TestSpark(base.SaharaTestCase):
|
||||
|
||||
# Check the command
|
||||
remote_instance.execute_command.assert_called_with(
|
||||
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit app.jar '
|
||||
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit '
|
||||
'--class org.me.myclass --jars jar1.jar,jar2.jar '
|
||||
'--master spark://master:7077 input_arg output_arg '
|
||||
'--master spark://master:7077 app.jar input_arg output_arg '
|
||||
'> /dev/null 2>&1 & echo $!')
|
||||
|
||||
# Check result here
|
||||
@ -366,14 +366,22 @@ class TestSpark(base.SaharaTestCase):
|
||||
edp.JOB_STATUS_RUNNING,
|
||||
{"spark-path": "/wfdir"}))
|
||||
|
||||
# Run again without support jars. Note the extra space
|
||||
# after 'myclass', this is from a %s with empty string
|
||||
# Run again without arguments
|
||||
job_exec.job_configs['args'] = []
|
||||
status = eng.run_job(job_exec)
|
||||
remote_instance.execute_command.assert_called_with(
|
||||
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit '
|
||||
'--class org.me.myclass --jars jar1.jar,jar2.jar '
|
||||
'--master spark://master:7077 app.jar '
|
||||
'> /dev/null 2>&1 & echo $!')
|
||||
|
||||
# Run again without support jars.
|
||||
upload_job_files.return_value = ["/wfdir/app.jar"]
|
||||
status = eng.run_job(job_exec)
|
||||
remote_instance.execute_command.assert_called_with(
|
||||
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit app.jar '
|
||||
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit '
|
||||
'--class org.me.myclass '
|
||||
'--master spark://master:7077 input_arg output_arg '
|
||||
'--master spark://master:7077 app.jar '
|
||||
'> /dev/null 2>&1 & echo $!')
|
||||
|
||||
# run again with non-zero result, should raise EDPError
|
||||
|
Loading…
Reference in New Issue
Block a user