f2a0837046
partially implements bp: add-vanilla-2-hadoop-2-6-0 Change-Id: I042a55697a36a9b4dbf1da0a29cf357dbab54dee
199 lines
6.1 KiB
Bash
Executable File
199 lines
6.1 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# XXX: This is in post-install.d, instead of install.d, because the
|
|
# hadoop RPM claims ownership of files owned by the filesystem RPM,
|
|
# such as /usr and /bin, and installing hadoop then updating
|
|
# filesystem results in a failure. This can be moved to install.d when
|
|
# HADOOP-9777 is resolved.
|
|
# https://issues.apache.org/jira/browse/HADOOP-9777
|
|
|
|
function download_hadoop_package {
|
|
echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
|
|
install-packages wget
|
|
wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-$DIB_HADOOP_VERSION/$package"
|
|
if [ $? -ne 0 ]; then
|
|
echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
function find_java_home {
|
|
JAVA_HOME=$(find $TARGET_ROOT$JAVA_TARGET_LOCATION/ -maxdepth 1 -name "jdk*")
|
|
|
|
if [ -z "$JAVA_HOME" ]; then
|
|
case "$distro" in
|
|
Ubuntu )
|
|
JAVA_HOME=$(readlink -e /usr/bin/java | sed "s:bin/java::")
|
|
;;
|
|
Fedora | RedHatEnterpriseServer | CentOS )
|
|
JAVA_HOME=$(rpm --eval '%{?java_home}')
|
|
;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
function install_hadoop_v1 {
|
|
case "$distro" in
|
|
Ubuntu )
|
|
package="hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb"
|
|
;;
|
|
Fedora | RedHatEnterpriseServer | CentOS )
|
|
package="hadoop-$DIB_HADOOP_VERSION-1.x86_64.rpm"
|
|
;;
|
|
esac
|
|
|
|
download_hadoop_package
|
|
|
|
case "$distro" in
|
|
Ubuntu )
|
|
dpkg -i $tmp_dir/$package
|
|
;;
|
|
Fedora | RedHatEnterpriseServer | CentOS )
|
|
if [ $(lsb_release -rs) -ge '19' ]; then
|
|
rpm -i $tmp_dir/$package --relocate /usr=/usr --replacefiles
|
|
else
|
|
rpm -ivh --replacefiles $tmp_dir/$package
|
|
fi
|
|
chmod 755 /usr/sbin/start-*
|
|
chmod 755 /usr/sbin/stop-*
|
|
chmod 755 /usr/sbin/slaves.sh
|
|
chmod 755 /usr/sbin/update-hadoop-env.sh
|
|
;;
|
|
esac
|
|
rm -r $tmp_dir
|
|
|
|
echo "Pre-configuring Hadoop"
|
|
find_java_home
|
|
|
|
cat >> /home/hadoop/.bashrc <<EOF
|
|
PATH=\$PATH:/usr/sbin:$JAVA_HOME/bin
|
|
JAVA_HOME=$JAVA_HOME
|
|
HADOOP_HOME=/usr/share/hadoop/
|
|
EOF
|
|
|
|
sed -i -e "s,export JAVA_HOME=.*,export JAVA_HOME=$JAVA_HOME," \
|
|
-e "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=/mnt/log/hadoop/\$USER," \
|
|
-e "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=/mnt/log/hadoop/hdfs," \
|
|
/etc/hadoop/hadoop-env.sh
|
|
}
|
|
|
|
|
|
function install_hadoop_v2 {
|
|
case "$DIB_HADOOP_VERSION" in
|
|
"2.3.0")
|
|
hadoop_native_libs_url="${HADOOP_V2_3_NATIVE_LIBS_DOWNLOAD_URL}"
|
|
;;
|
|
"2.4.1")
|
|
hadoop_native_libs_url="${HADOOP_V2_4_NATIVE_LIBS_DOWNLOAD_URL}"
|
|
;;
|
|
"2.6.0")
|
|
hadoop_native_libs_url="${HADOOP_V2_6_NATIVE_LIBS_DOWNLOAD_URL}"
|
|
;;
|
|
*)
|
|
echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
package="hadoop-$DIB_HADOOP_VERSION.tar.gz"
|
|
|
|
download_hadoop_package
|
|
|
|
echo "Installing hadoop"
|
|
|
|
INSTALL_DIR="/opt"
|
|
HADOOP_HOME="/opt/hadoop"
|
|
mkdir -p "$INSTALL_DIR"
|
|
tar xvf "$tmp_dir/$package" -C "$INSTALL_DIR"
|
|
ln -s "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION" "$HADOOP_HOME"
|
|
chown -R hadoop:hadoop "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION"
|
|
chown -R hadoop:hadoop "$HADOOP_HOME"
|
|
rm -r $tmp_dir
|
|
|
|
echo "Inject Hadoop native libs"
|
|
rm -r "$HADOOP_HOME/lib/native"
|
|
wget "$hadoop_native_libs_url"
|
|
native_libs_filename=$(basename "$hadoop_native_libs_url")
|
|
tar xvf "$native_libs_filename" -C "$HADOOP_HOME/lib"
|
|
rm "$native_libs_filename"
|
|
|
|
echo "Pre-configuring Hadoop"
|
|
find_java_home
|
|
|
|
HADOOP_PID_DIR="/var/run/hadoop"
|
|
mkdir $HADOOP_PID_DIR
|
|
chown hadoop:hadoop $HADOOP_PID_DIR
|
|
|
|
cat >> /etc/profile.d/hadoop.sh <<EOF
|
|
export JAVA_HOME=$JAVA_HOME
|
|
export HADOOP_COMMON_HOME=$HADOOP_HOME
|
|
export PATH=\$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
|
|
export HADOOP_HDFS_HOME=\$HADOOP_COMMON_HOME
|
|
export HADOOP_YARN_HOME=\$HADOOP_COMMON_HOME
|
|
export HADOOP_MAPRED_HOME=\$HADOOP_COMMON_HOME
|
|
export HADOOP_PID_DIR=$HADOOP_PID_DIR
|
|
export YARN_PID_DIR=$HADOOP_PID_DIR
|
|
export HADOOP_MAPRED_PID_DIR=$HADOOP_PID_DIR
|
|
EOF
|
|
|
|
sed -i -e "s,export JAVA_HOME=.*,export JAVA_HOME=$JAVA_HOME," \
|
|
-e "s,\#export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=/mnt/log/hadoop/\$USER," \
|
|
-e "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=/mnt/log/hadoop/hdfs," \
|
|
$HADOOP_HOME/etc/hadoop/hadoop-env.sh
|
|
|
|
sed -i -e "s,\# export JAVA_HOME=.*,export JAVA_HOME=$JAVA_HOME," \
|
|
-e "s,YARN_LOG_DIR=.*,YARN_LOG_DIR=/mnt/log/hadoop/yarn," \
|
|
$HADOOP_HOME/etc/hadoop/yarn-env.sh
|
|
|
|
# enable swiftfs
|
|
ln -s ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-openstack-${DIB_HADOOP_VERSION}.jar ${HADOOP_HOME}/share/hadoop/common/lib/
|
|
}
|
|
|
|
distro=$(lsb_release -is || :)
|
|
if [ ! "$distro" == "Fedora" -a ! "$distro" == "Ubuntu" -a ! "$distro" == "RedHatEnterpriseServer" -a ! "$distro" == "CentOS" ]; then
|
|
echo "Unknown distro: $distro. Exiting."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Hadoop setup begins for $distro"
|
|
tmp_dir=/tmp/hadoop
|
|
|
|
echo "Creating hadoop user & group"
|
|
case "$distro" in
|
|
Ubuntu )
|
|
addgroup hadoop
|
|
adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
|
|
adduser hadoop sudo
|
|
;;
|
|
Fedora | RedHatEnterpriseServer | CentOS )
|
|
adduser -G adm,wheel hadoop
|
|
;;
|
|
esac
|
|
|
|
: ${JAVA_TARGET_LOCATION:="/usr/java"}
|
|
|
|
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
|
|
install_hadoop_v1
|
|
else
|
|
install_hadoop_v2
|
|
fi
|
|
|
|
echo "Applying firstboot script"
|
|
|
|
RC_SCRIPT_DIR=""
|
|
if [ "$distro" == "Ubuntu" ]; then
|
|
# File '/etc/rc.local' may not exist
|
|
if [ -f "/etc/rc.local" ]; then
|
|
mv /etc/rc.local /etc/rc.local.old
|
|
fi
|
|
RC_SCRIPT_DIR="/etc"
|
|
else
|
|
# File '/etc/rc.d/rc.local' may not exist
|
|
if [ -f "/etc/rc.d/rc.local" ]; then
|
|
mv /etc/rc.d/rc.local /etc/rc.d/rc.local.old
|
|
fi
|
|
RC_SCRIPT_DIR="/etc/rc.d"
|
|
fi
|
|
|
|
install -D -g root -o root -m 0755 $(dirname $0)/firstboot $RC_SCRIPT_DIR/rc.local
|