openstack
/
sahara-image-elements
Code Issues Proposed changes

Add the swift_hadoop element to Spark images 

The Spark image needs the hadoop-swift.jar in /usr/lib/hadoop
in order for Spark jobs to access Swift URLs.

Change-Id: I1c47cbc877e6c6628dc3fd2181152b2c4d4cd3f9
Partial-Implements: blueprint edp-spark-swift-integration
This commit is contained in:
Trevor McKay 2015-01-16 15:43:29 -05:00
parent cea6bb10c0
commit 4560d069f2
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
diskimage-create/diskimage-create.sh
View File

@ -365,6 +365,7 @@ fi
##########################
if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
export DIB_HDFS_LIB_DIR="/usr/lib/hadoop"
export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
# Ignoring image type and hadoop version options
@ -374,7 +375,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
export DIB_HADOOP_VERSION="CDH4"
export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"}
ubuntu_elements_sequence="base vm ubuntu java hadoop-cdh spark"
ubuntu_elements_sequence="base vm ubuntu java hadoop-cdh swift_hadoop spark"
if [ -n "$USE_MIRRORS" ]; then
[ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror"
@ -384,6 +385,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
disk-image-create $ubuntu_elements_sequence -o $ubuntu_image_name
mv $ubuntu_image_name.qcow2 ../
unset DIB_CLOUD_INIT_DATASOURCES
unset DIB_HDFS_LIB_DIR
fi

2
elements/swift_hadoop/post-install.d/81-add-jar
View File

@ -2,7 +2,7 @@
install-packages wget
HDFS_LIB_DIR="/usr/share/hadoop/lib"
HDFS_LIB_DIR=${DIB_HDFS_LIB_DIR:-"/usr/share/hadoop/lib"}
SWIFT_LIB_URI="http://sahara-files.mirantis.com/hadoop-swift/hadoop-swift-latest.jar"
if [ -z "$swift_url" ]; then