Merge first-boot, setup-hadoop and setup-ssh with distro switching based on lsb_release

Implements: blueprint merge-dib-hadoop-elements
Change-Id: I5090a556471414cfd663fa07a0efec2ff94c3581
This commit is contained in:
Matthew Farrellee 2013-07-11 17:46:26 -04:00
parent 60c36eb0b1
commit c21cba3d7b
3 changed files with 74 additions and 32 deletions

View File

@ -5,18 +5,26 @@ chown root:root /mnt
mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
# Unique to Ubuntu distro=$(lsb_release -is || :)
#mkdir /run/hadoop case "$distro" in
#chown hadoop:hadoop /run/hadoop/ Ubuntu )
#mkdir -p /home/ubuntu/.ssh mkdir /run/hadoop
#touch /home/ubuntu/.ssh/authorized_keys chown hadoop:hadoop /run/hadoop/
#chown -R ubuntu:ubuntu /home/ubuntu mkdir -p /home/ubuntu/.ssh
touch /home/ubuntu/.ssh/authorized_keys
# Unique to Fedora chown -R ubuntu:ubuntu /home/ubuntu
sleep 10 ;;
rm /etc/resolv.conf Fedora )
service network restart sleep 10
chown -R ec2-user:ec2-user /home/ec2-user rm /etc/resolv.conf
chown -R ec2-user:ec2-user /etc/hadoop service network restart
#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744) chown -R ec2-user:ec2-user /home/ec2-user
iptables -F chown -R ec2-user:ec2-user /etc/hadoop
#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
iptables -F
;;
* )
echo "Unknown distro: $distro. Exiting."
exit 1
;;
esac

View File

@ -1,32 +1,65 @@
#!/bin/bash #!/bin/bash
echo "Hadoop setup begins"
distro=$(lsb_release -is || :)
if [ ! "$distro" == "Fedora" -a ! "$distro" == "Ubuntu" ]; then
echo "Unknown distro: $distro. Exiting."
exit 1
fi
echo "Hadoop setup begins for $distro"
tmp_dir=/tmp/hadoop tmp_dir=/tmp/hadoop
install-packages wget
echo "Creating hadoop user & group" echo "Creating hadoop user & group"
adduser -G adm,wheel hadoop case "$distro" in
Ubuntu )
addgroup hadoop
adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
adduser hadoop sudo
;;
Fedora )
adduser -G adm,wheel hadoop
;;
esac
echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download" echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
wget -P $tmp_dir/ "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm" case "$distro" in
Ubuntu )
package="hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb"
;;
Fedora )
package="hadoop-$DIB_HADOOP_VERSION-1.x86_64.rpm"
;;
esac
install-packages wget
wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-$DIB_HADOOP_VERSION/$package"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting" echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
exit 1 exit 1
fi fi
rpm -ivh --replacefiles $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm case "$distro" in
chmod 755 /usr/sbin/start-* Ubuntu )
chmod 755 /usr/sbin/stop-* dpkg -i $tmp_dir/$package
chmod 755 /usr/sbin/slaves.sh ;;
chmod 755 /usr/sbin/update-hadoop-env.sh Fedora )
rm $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm rpm -ivh --replacefiles $tmp_dir/$package
chmod 755 /usr/sbin/start-*
chmod 755 /usr/sbin/stop-*
chmod 755 /usr/sbin/slaves.sh
chmod 755 /usr/sbin/update-hadoop-env.sh
;;
esac
rm -r $tmp_dir rm -r $tmp_dir
echo "Pre-configuring Hadoop" echo "Pre-configuring Hadoop"
filename=$(find $TARGET_ROOT/usr/java/ -maxdepth 1 -name "jdk*") filename=$(find $TARGET_ROOT/usr/java/ -maxdepth 1 -name "jdk*")
echo -e "PATH=\$PATH:/usr/sbin:$filename/bin\nJAVA_HOME=$filename" >> /home/hadoop/.bashrc cat >> /home/hadoop/.bashrc <<EOF
echo -e "HADOOP_HOME=/usr/share/hadoop/" >> /home/hadoop/.bashrc PATH=\$PATH:/usr/sbin:$filename/bin
sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh JAVA_HOME=$filename
log_dir=/mnt/log/hadoop/\$USER/ HADOOP_HOME=/usr/share/hadoop/
sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh EOF
log_dir=/mnt/log/hadoop/hdfs sed -i -e "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," \
sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh -e "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=/mnt/log/hadoop/\$USER," \
-e "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=/mnt/log/hadoop/hdfs," \
/etc/hadoop/hadoop-env.sh

View File

@ -6,6 +6,7 @@ echo "Adjusting ssh configuration"
# /etc/ssh/ssh_config is provided by openssh-clients # /etc/ssh/ssh_config is provided by openssh-clients
# Note0: augtool is provided by augeas-tools on Ubuntu # Note0: augtool is provided by augeas-tools on Ubuntu
# Note1: augtool on Ubuntu does not auto-save, pass -s # Note1: augtool on Ubuntu does not auto-save, pass -s
# Note2: openssh-clients is openssh-client on Ubuntu
install-packages augeas openssh-server openssh-clients install-packages augeas openssh-server openssh-clients