Merge "Add S3 jar to Hadoop classpath"

diskimage-create/diskimage-create.sh

@@ -477,7 +477,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
     export HIVE_VERSION=${HIVE_VERSION:-"0.11.0"}
     export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"}
     export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"http://sahara-files.mirantis.com/oozie-4.2.0-hadoop-2.7.1.tar.gz"}
-    export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib"
+    export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/common/lib"
     export plugin_type="vanilla"
 
     if [ "$DIB_SPARK_VERSION" = "1.3.1" ]; then
@@ -493,10 +493,10 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
         export SPARK_HADOOP_DL=hadoop2.7
     fi
 
-    ubuntu_elements_sequence="hadoop oozie mysql hive $JAVA_ELEMENT swift_hadoop spark"
+    ubuntu_elements_sequence="hadoop oozie mysql hive $JAVA_ELEMENT swift_hadoop spark s3_hadoop"
-    fedora_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark"
+    fedora_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark s3_hadoop"
-    centos_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark nc"
+    centos_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark nc s3_hadoop"
-    centos7_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark nc"
+    centos7_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark nc s3_hadoop"
 
     # Workaround for https://bugs.launchpad.net/diskimage-builder/+bug/1204824
     # https://bugs.launchpad.net/sahara/+bug/1252684
@@ -571,7 +571,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
     # Tell the cloudera element to install only hdfs
     export DIB_CDH_HDFS_ONLY=1
 
-    ubuntu_elements_sequence="$JAVA_ELEMENT swift_hadoop spark hadoop-cloudera"
+    ubuntu_elements_sequence="$JAVA_ELEMENT swift_hadoop spark hadoop-cloudera s3_hadoop"
     export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"}
 
     # Creating Ubuntu cloud image
@@ -619,19 +619,19 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "ambari" ]; then
 
     if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
         ambari_ubuntu_image_name=${ambari_ubuntu_image_name:-ubuntu_sahara_ambari}
-        ambari_element_sequence="ambari $JAVA_ELEMENT swift_hadoop kdc"
+        ambari_element_sequence="ambari $JAVA_ELEMENT swift_hadoop kdc s3_hadoop"
         export DIB_RELEASE="trusty"
         image_create ubuntu $ambari_ubuntu_image_name $ambari_element_sequence
         unset DIB_RELEASE
     fi
     if [ "$BASE_IMAGE_OS" = "centos" ]; then
         ambari_centos_image_name=${ambari_centos_image_name:-centos_sahara_ambari}
-        ambari_element_sequence="ambari $JAVA_ELEMENT disable-firewall swift_hadoop kdc nc"
+        ambari_element_sequence="ambari $JAVA_ELEMENT disable-firewall swift_hadoop kdc nc s3_hadoop"
         image_create centos $ambari_centos_image_name $ambari_element_sequence
     fi
     if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then
         ambari_centos7_image_name=${ambari_centos7_image_name:-"centos7-sahara-ambari"}
-        ambari_element_sequence="disable-selinux ambari $JAVA_ELEMENT disable-firewall swift_hadoop kdc nc"
+        ambari_element_sequence="disable-selinux ambari $JAVA_ELEMENT disable-firewall swift_hadoop kdc nc s3_hadoop"
         image_create centos7 $ambari_centos7_image_name $ambari_element_sequence
     fi
 
@@ -657,7 +657,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
         HADOOP_VERSION=${DIB_CDH_MINOR_VERSION%.*}
     fi
 
-    cloudera_elements_sequence="hadoop-cloudera swift_hadoop kdc"
+    cloudera_elements_sequence="hadoop-cloudera swift_hadoop kdc s3_hadoop"
     if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
         if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then
             export DIB_CDH_VERSION="5.5"
@@ -698,7 +698,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
     fi
 
     if [ "$BASE_IMAGE_OS" = "centos" ]; then
-        centos_cloudera_elements_sequence="selinux-permissive disable-firewall nc"
+        centos_cloudera_elements_sequence="selinux-permissive disable-firewall nc s3_hadoop"
         if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then
             export DIB_CDH_VERSION="5.5"
 
@@ -710,7 +710,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
     fi
 
     if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then
-        centos7_cloudera_elements_sequence="selinux-permissive disable-firewall nc"
+        centos7_cloudera_elements_sequence="selinux-permissive disable-firewall nc s3_hadoop"
         if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then
             export DIB_CDH_VERSION="5.5"
 
@@ -759,6 +759,7 @@ fi
 ##########################
 if [ -z "$PLUGIN" -o "$PLUGIN" = "mapr" ]; then
     export DIB_MAPR_VERSION=${DIB_MAPR_VERSION:-${DIB_DEFAULT_MAPR_VERSION}}
+    export plugin_type="mapr"
 
     export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
 
@@ -766,8 +767,8 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "mapr" ]; then
     #MapR repository requires additional space
     export DIB_MIN_TMPFS=10
 
-    mapr_ubuntu_elements_sequence="ssh hadoop-mapr $JAVA_ELEMENT"
+    mapr_ubuntu_elements_sequence="ssh hadoop-mapr $JAVA_ELEMENT s3_hadoop"
-    mapr_centos_elements_sequence="ssh hadoop-mapr selinux-permissive $JAVA_ELEMENT disable-firewall nc"
+    mapr_centos_elements_sequence="ssh hadoop-mapr selinux-permissive $JAVA_ELEMENT disable-firewall nc s3_hadoop"
 
     if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
         export DIB_RELEASE=${DIB_RELEASE:-trusty}
@@ -794,6 +795,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "mapr" ]; then
 
         unset DIB_CLOUD_INIT_DATASOURCES
     fi
+    unset plugin_type
 
 fi
 

elements/hadoop/install.d/40-setup-hadoop

@@ -61,8 +61,9 @@ EOF
         $HADOOP_HOME/etc/hadoop/yarn-env.sh
     echo "source $JAVA_RC" >> $HADOOP_HOME/etc/hadoop/yarn-env.sh
 
-    # enable swiftfs
+    # remove apache-built swiftfs
-    ln -s ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-openstack-${DIB_HADOOP_VERSION}.jar ${HADOOP_HOME}/share/hadoop/common/lib/
+    rm ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-openstack-${DIB_HADOOP_VERSION}.jar
+
 }
 
 case "$DISTRO_NAME" in

elements/s3_hadoop/README.rst

@@ -0,0 +1,23 @@
+=========
+s3_hadoop
+=========
+
+Copy the Hadoop S3 connector jar file into the Hadoop classpath.
+
+Environment Variables
+---------------------
+
+HADOOP_S3_JAR_ORIGIN
+  :Required: No
+  :Default: Depends on plugin.
+  :Description: Path to where the S3 jar is (already) located.
+
+HADOOP_S3_JAR_DOWNLOAD
+  :Required: No
+  :Default: None.
+  :Description: If set, use a download a specific S3 jar instead of one already available on the image.
+
+DIB_HDFS_LIB_DIR
+  :Required: No
+  :Default: /usr/share/hadoop/lib
+  :Description: Directory in the guest where to save the S3 jar. Shared with swift_hadoop.

elements/s3_hadoop/element-deps

@@ -0,0 +1 @@
+package-installs

elements/s3_hadoop/package-installs.yaml

@@ -0,0 +1,2 @@
+wget:
+  phase: post-install.d

elements/s3_hadoop/post-install.d/89-add-amazon-jar

@@ -0,0 +1,36 @@
+#!/bin/bash
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [ -z "${HADOOP_S3_JAR_ORIGIN:-}" ]; then
+    # The jar is not locally available during image-gen on Ambari/MapR plugins: relevant packages are installed later.
+    # The jar is not appropriate for the Storm plugin: you cannot stream data from an object store.
+    # For plugins not found in the switch statement below, a user-specified jar can still be downloaded.
+    case "$plugin_type" in
+        "vanilla")
+            HADOOP_S3_JAR_ORIGIN="/opt/hadoop/share/hadoop/tools/lib/hadoop-aws-$DIB_HADOOP_VERSION.jar"
+            ;;
+        "cloudera" | "spark" )
+            HADOOP_S3_JAR_ORIGIN="/usr/lib/hadoop/hadoop-aws.jar"
+            ;;
+    esac
+fi
+
+HDFS_LIB_DIR=${DIB_HDFS_LIB_DIR:-"/usr/share/hadoop/lib"} # matches swift_hadoop default
+
+if [ -z "${HADOOP_S3_JAR_DOWNLOAD:-}" ]; then
+    if [ "${HADOOP_S3_JAR_ORIGIN:-}" ]; then
+        cp $HADOOP_S3_JAR_ORIGIN $HDFS_LIB_DIR/hadoop-aws.jar
+    fi
+else
+    wget -O $HDFS_LIB_DIR/hadoop-aws.jar $HADOOP_S3_JAR_DOWNLOAD
+fi
+
+path=$HDFS_LIB_DIR/hadoop-aws.jar
+
+if [ -f $path ]; then
+    chmod 0644 $path
+fi

elements/swift_hadoop/README.rst

@@ -17,7 +17,7 @@ swift_url
 DIB_HDFS_LIB_DIR
   :Required: No
   :Default: /usr/share/hadoop/lib
-  :Description: Directory in the guest where to save the swift jar.
+  :Description: Directory in the guest where to save the swift jar. Shared with s3_hadoop.
 
 DIB_HADOOP_SWIFT_JAR_NAME
   :Required: No