openstack/deb-sahara
Code Issues Proposed changes

Merge "Follow the argument order specified in spark-submit help"

Browse Source
This commit is contained in:
Jenkins 2015-01-27 11:15:53 +00:00 committed by Gerrit Code Review
commit a8f3363777
2 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
sahara/service/edp/spark/engine.py
View File

@ -143,7 +143,7 @@ class SparkJobEngine(base_engine.JobEngine):
# The rest of the paths will be passed with --jars
additional_jars = ",".join(paths)
if additional_jars:
additional_jars = "--jars " + additional_jars
additional_jars = " --jars " + additional_jars
# Launch the spark job using spark-submit and deploy_mode = client
host = master.hostname()
@ -159,15 +159,17 @@ class SparkJobEngine(base_engine.JobEngine):
# TODO(tmckay): we need to clean up wf_dirs on long running clusters
# TODO(tmckay): probably allow for general options to spark-submit
args = " ".join(updated_job_configs.get('args', []))
if args:
args = " " + args
# The redirects of stdout and stderr will preserve output in the wf_dir
cmd = "%s %s --class %s %s --master spark://%s:%s %s" % (
cmd = "%s --class %s%s --master spark://%s:%s %s%s" % (
spark_submit,
app_jar,
job_class,
additional_jars,
host,
port,
app_jar,
args)
job_execution = conductor.job_execution_get(ctx, job_execution.id)

22
sahara/tests/unit/service/edp/spark/test_spark.py
View File

@ -356,9 +356,9 @@ class TestSpark(base.SaharaTestCase):
# Check the command
remote_instance.execute_command.assert_called_with(
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit app.jar '
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit '
'--class org.me.myclass --jars jar1.jar,jar2.jar '
'--master spark://master:7077 input_arg output_arg '
'--master spark://master:7077 app.jar input_arg output_arg '
'> /dev/null 2>&1 & echo $!')
# Check result here
@ -366,14 +366,22 @@ class TestSpark(base.SaharaTestCase):
edp.JOB_STATUS_RUNNING,
{"spark-path": "/wfdir"}))
# Run again without support jars. Note the extra space
# after 'myclass', this is from a %s with empty string
# Run again without arguments
job_exec.job_configs['args'] = []
status = eng.run_job(job_exec)
remote_instance.execute_command.assert_called_with(
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit '
'--class org.me.myclass --jars jar1.jar,jar2.jar '
'--master spark://master:7077 app.jar '
'> /dev/null 2>&1 & echo $!')
# Run again without support jars.
upload_job_files.return_value = ["/wfdir/app.jar"]
status = eng.run_job(job_exec)
remote_instance.execute_command.assert_called_with(
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit app.jar '
'--class org.me.myclass '
'--master spark://master:7077 input_arg output_arg '
'cd /wfdir; ./launch_command /opt/spark/bin/spark-submit '
'--class org.me.myclass '
'--master spark://master:7077 app.jar '
'> /dev/null 2>&1 & echo $!')
# run again with non-zero result, should raise EDPError