a77a9a978a
As prereq of support for S3 datasource, the hadoop-aws jar needs to be in the Hadoop classpath. The jar is copied into the proper folder when possible on the appropriate plugins, and otherwise can be provided from a download URL by the user. Additionally, set the correct value of DIB_HDFS_LIB_DIR on the Vanilla plugin to avoid any unnecessary simlinking. Partially-Implements: bp sahara-support-s3 Change-Id: I94c5b0055b87f6a4e1382118d0718e588fccfe87
115 lines
3.2 KiB
Bash
Executable File
115 lines
3.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
|
set -x
|
|
fi
|
|
set -eu
|
|
set -o pipefail
|
|
|
|
function install_hadoop_v2 {
|
|
case "$DIB_HADOOP_VERSION" in
|
|
"2.7.1")
|
|
hadoop_native_libs_url="${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL}"
|
|
;;
|
|
*)
|
|
echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
package="hadoop-$DIB_HADOOP_VERSION.tar.gz"
|
|
|
|
echo "Installing hadoop"
|
|
|
|
INSTALL_DIR="/opt"
|
|
HADOOP_HOME="/opt/hadoop"
|
|
mkdir -p "$INSTALL_DIR"
|
|
tar xvf "$tmp_dir/$package" -C "$INSTALL_DIR"
|
|
ln -s "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION" "$HADOOP_HOME"
|
|
chown -R hadoop:hadoop "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION"
|
|
chown -R hadoop:hadoop "$HADOOP_HOME"
|
|
rm -r $tmp_dir
|
|
|
|
echo "Inject Hadoop native libs"
|
|
rm -r "$HADOOP_HOME/lib/native"
|
|
wget "$hadoop_native_libs_url"
|
|
native_libs_filename=$(basename "$hadoop_native_libs_url")
|
|
tar xvf "$native_libs_filename" -C "$HADOOP_HOME/lib"
|
|
rm "$native_libs_filename"
|
|
|
|
echo "Pre-configuring Hadoop"
|
|
|
|
HADOOP_PID_DIR="/var/run/hadoop"
|
|
|
|
cat >> /etc/profile.d/hadoop.sh <<EOF
|
|
export HADOOP_COMMON_HOME=$HADOOP_HOME
|
|
export PATH=\$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
|
|
export HADOOP_HDFS_HOME=\$HADOOP_COMMON_HOME
|
|
export HADOOP_YARN_HOME=\$HADOOP_COMMON_HOME
|
|
export HADOOP_MAPRED_HOME=\$HADOOP_COMMON_HOME
|
|
export HADOOP_PID_DIR=$HADOOP_PID_DIR
|
|
export YARN_PID_DIR=$HADOOP_PID_DIR
|
|
export HADOOP_MAPRED_PID_DIR=$HADOOP_PID_DIR
|
|
EOF
|
|
|
|
sed -i -e "s,\#export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=/mnt/log/hadoop/\$USER," \
|
|
-e "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=/mnt/log/hadoop/hdfs," \
|
|
$HADOOP_HOME/etc/hadoop/hadoop-env.sh
|
|
echo "source $JAVA_RC" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh
|
|
|
|
sed -i -e "s,YARN_LOG_DIR=.*,YARN_LOG_DIR=/mnt/log/hadoop/yarn," \
|
|
$HADOOP_HOME/etc/hadoop/yarn-env.sh
|
|
echo "source $JAVA_RC" >> $HADOOP_HOME/etc/hadoop/yarn-env.sh
|
|
|
|
# remove apache-built swiftfs
|
|
rm ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-openstack-${DIB_HADOOP_VERSION}.jar
|
|
|
|
}
|
|
|
|
case "$DISTRO_NAME" in
|
|
fedora | ubuntu | rhel | rhel7 | centos | centos7 )
|
|
;;
|
|
*)
|
|
echo "Unknown distro: $DISTRO_NAME. Exiting."
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "Hadoop setup begins for $DISTRO_NAME"
|
|
tmp_dir=/tmp/hadoop
|
|
|
|
echo "Creating hadoop user & group"
|
|
case "$DISTRO_NAME" in
|
|
ubuntu )
|
|
addgroup hadoop
|
|
adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
|
|
adduser hadoop sudo
|
|
;;
|
|
fedora | rhel | rhel7 | centos | centos7 )
|
|
adduser -G adm,wheel hadoop
|
|
;;
|
|
esac
|
|
|
|
install_hadoop_v2
|
|
|
|
echo "Applying firstboot script"
|
|
|
|
RC_SCRIPT_DIR=""
|
|
if [ "$DISTRO_NAME" == "ubuntu" ]; then
|
|
# File '/etc/rc.local' may not exist
|
|
if [ -f "/etc/rc.local" ]; then
|
|
mv /etc/rc.local /etc/rc.local.old
|
|
fi
|
|
RC_SCRIPT_DIR="/etc"
|
|
else
|
|
# File '/etc/rc.d/rc.local' may not exist
|
|
if [ -f "/etc/rc.d/rc.local" ]; then
|
|
mv /etc/rc.d/rc.local /etc/rc.d/rc.local.old
|
|
fi
|
|
RC_SCRIPT_DIR="/etc/rc.d"
|
|
fi
|
|
|
|
install -D -g root -o root -m 0755 $(dirname $0)/firstboot $RC_SCRIPT_DIR/rc.local
|
|
# make sure it is run, be it on SysV, upstart, or systemd
|
|
chmod +x $RC_SCRIPT_DIR/rc.local
|