diff --git a/elements/hadoop/README.md b/elements/hadoop/README.md index 48c7f9aa..6b82de1f 100644 --- a/elements/hadoop/README.md +++ b/elements/hadoop/README.md @@ -1 +1 @@ -Installs Java 6 and Hadoop on Ubuntu +Installs Java and Hadoop on Ubuntu diff --git a/elements/hadoop/first-boot.d/99-setup b/elements/hadoop/first-boot.d/99-setup new file mode 100755 index 00000000..0b59a1bb --- /dev/null +++ b/elements/hadoop/first-boot.d/99-setup @@ -0,0 +1,10 @@ +#!/bin/bash +echo -e "Creating hadoop_pid_dir" +mkdir /var/run/hadoop +mkdir /run/hadoop +chown hadoop:hadoop /var/run/hadoop/ +chown hadoop:hadoop /run/hadoop/ +echo "Setup ubuntu's directory" +mkdir -p /home/ubuntu/.ssh +touch /home/ubuntu/.ssh/authorized_keys +chown -R ubuntu:ubuntu /home/ubuntu diff --git a/elements/hadoop/install.d/70-setup-java b/elements/hadoop/install.d/70-setup-java index cce97e7a..b4efb65a 100755 --- a/elements/hadoop/install.d/70-setup-java +++ b/elements/hadoop/install.d/70-setup-java @@ -1,13 +1,15 @@ #!/bin/bash +tmp_dir=/tmp/java +mkdir -p $TARGET_ROOT/$tmp_dir echo "Java setup begins" set -e if [ -z "$JAVA_DOWNLOAD_URL" ]; then - mkdir -p $TARGET_ROOT/home/ubuntu + mkdir -p $TARGET_ROOT/$tmp_dir script_dir=$(dirname $0) - install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/home/ubuntu/ + install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir echo "Java file moved" else - wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/home/ubuntu/ $JAVA_DOWNLOAD_URL + wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL if [ $? -eq 0 ]; then echo "Java is downloading" else @@ -15,31 +17,27 @@ else exit 1 fi fi -filename=$(find $TARGET_ROOT/home/ubuntu/ -maxdepth 1 -name "jdk*") +filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*") filename=$(basename $filename) if echo $filename | grep -q -s -F .tar.gz ; then path=$(pwd) - cd $TARGET_ROOT/home/ubuntu/ - echo $TARGET_ROOT/home/ubuntu/$filename | xargs -n 1 tar -zxvf + cd $TARGET_ROOT/$tmp_dir + echo $TARGET_ROOT/$tmp_dir/$filename | xargs -n 1 tar -zxvf cd $path else if echo $filename | grep -q -s -F .bin ; then path=$(pwd) - cd $TARGET_ROOT/home/ubuntu/ + cd $TARGET_ROOT/$tmp_dir echo -e "\n" | sh $filename rm $filename cd $path fi fi -filename=$(find $TARGET_ROOT/home/ubuntu/ -maxdepth 1 -type d -name "jdk*") +filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*") filename=$(basename $filename) javaPath=/usr/lib/ mkdir -p $TARGET_ROOT/$javaPath -mv $TARGET_ROOT/home/ubuntu/$filename $TARGET_ROOT/$javaPath -cat $TARGET_ROOT/root/.bashrc >> $TARGET_ROOT/home/ubuntu/.bashrc -echo -e "export JAVA_HOME=$javaPath$filename\nexport PATH=\$PATH:$javaPath$filename/bin" >> $TARGET_ROOT/home/ubuntu/.bashrc -echo -e "export JAVA_HOME=$javaPath$filename\nexport PATH=\$PATH:$javaPath$filename/bin" >> $TARGET_ROOT/root/.bashrc -source $TARGET_ROOT/home/ubuntu/.bashrc +mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath echo "Java was installed" diff --git a/elements/hadoop/install.d/80-setup-hadoop b/elements/hadoop/install.d/80-setup-hadoop index a848ddc9..55532106 100755 --- a/elements/hadoop/install.d/80-setup-hadoop +++ b/elements/hadoop/install.d/80-setup-hadoop @@ -1,7 +1,9 @@ #!/bin/bash - +tmp_dir=/tmp/hadoop +mkdir -p $TARGET_ROOT/$tmp_dir echo "Installing SSH server" +mkdir /root/.ssh install-packages openssh-server echo "Creating hadoop user & group" @@ -9,11 +11,10 @@ echo "Creating hadoop user & group" addgroup hadoop adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop adduser hadoop sudo -echo -e "swordfish\nswordfish\n" | passwd hadoop echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download" -wget -P /home/ubuntu/ "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop_"$DIB_HADOOP_VERSION"-1_x86_64.deb" +wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop_"$DIB_HADOOP_VERSION"-1_x86_64.deb" if [ $? -ne 0 ]; then echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nExit" exit 1 @@ -21,15 +22,19 @@ fi echo "Installing Hadoop" -dpkg -i /home/ubuntu/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb -rm /home/ubuntu/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb +dpkg -i $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb +rm $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb echo "Pre-configuring Hadoop" -cat $TARGET_ROOT/home/ubuntu/.bashrc > $TARGET_ROOT/home/hadoop/.bashrc filename=$(find $TARGET_ROOT/usr/lib/ -maxdepth 1 -name "jdk*") echo -e "HADOOP_HOME=/usr/share/hadoop/\nPATH=\$PATH:/usr/sbin/" >> $TARGET_ROOT/home/hadoop/.bashrc +echo -e "JAVA_HOME=$filename\nPATH=\$PATH:$filename/bin/" >> $TARGET_ROOT/home/hadoop/.bashrc sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh +log_dir=/mnt/log/hadoop/\$USER/ +sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh +log_dir=/mnt/log/hadoop/hdfs +sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh echo "Generating keypair for 'hadoop' user" @@ -37,16 +42,5 @@ su -c "mkdir -m 700 /home/hadoop/.ssh" hadoop su -c "ssh-keygen -t rsa -P \"\" -N \"\" -f /home/hadoop/.ssh/id_rsa" hadoop su -c "cat /home/hadoop/.ssh/id_rsa.pub >> /home/hadoop/.ssh/authorized_keys" hadoop chmod 600 /home/hadoop/.ssh/authorized_keys -chown hadoop:hadoop /var/run/ -mkdir /root/.ssh cp /home/hadoop/.ssh/authorized_keys /root/.ssh/ - -echo "Adjusting ssh configuration" - -sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config -echo -e "AuthorizedKeysFile .ssh/authorized_keys\nUseDNS no\nPermitTunnel yes" >> /etc/ssh/sshd_config - -sed -i -e 's/# StrictHostKeyChecking ask/StrictHostKeyChecking no/' \ - -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' \ - /etc/ssh/ssh_config diff --git a/elements/hadoop/install.d/90-setup-ssh b/elements/hadoop/install.d/90-setup-ssh new file mode 100755 index 00000000..33e8f297 --- /dev/null +++ b/elements/hadoop/install.d/90-setup-ssh @@ -0,0 +1,8 @@ +#!/bin/bash +sed -i -e 's/PasswordAuthentication no/PasswordAuthentication yes/' \ + -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' \ + /etc/ssh/sshd_config +echo -e "AuthorizedKeysFile .ssh/authorized_keys\nUseDNS no\nPermitTunnel yes\nGSSAPICleanupCredentials yes" >> /etc/ssh/sshd_config + +sed -i -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config +echo -e "StrictHostKeyChecking no" >> /etc/ssh/ssh_config diff --git a/elements/hadoop/root.d/0-check b/elements/hadoop/root.d/0-check index 3367ebbc..61a79e75 100755 --- a/elements/hadoop/root.d/0-check +++ b/elements/hadoop/root.d/0-check @@ -1,16 +1,11 @@ #!/bin/bash set -e - +source $_LIB/die if [ -z "$JAVA_DOWNLOAD_URL" ]; then - if [ -z "$JAVA_FILE" ]; then - echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit" - exit 1 - fi -fi -if [ -z "$DIB_HADOOP_VERSION" ]; then - echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit" - exit 1 + [ -n "$JAVA_FILE" ] || die "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit" fi +[ -n "$DIB_HADOOP_VERSION" ] || die "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit" + version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d') if [[ -z $version_check ]]; then echo "All variables are set, continue."