Browse Source

Replace elements/hadoop/ with Fedora&Ubuntu supporting elements/hadoop_fedora

Note: this requires a DIB version that includes change I7a42409a (SHA
82eacdec) from 11 July 2013

Implements: blueprint merge-dib-hadoop-elements
Change-Id: I23d873adfc7c16e36cf26af2cd9156c2a8ec90fc
tags/0.2.1.rc1^0
Matthew Farrellee 5 years ago
parent
commit
40931bdb5f

+ 1
- 1
elements/README.rst View File

@@ -38,7 +38,7 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
38 38
 
39 39
 .. sourcecode:: bash
40 40
 
41
-    DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2
41
+    JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop root-passwd -o fedora_hadoop_1_1_2
42 42
 
43 43
 In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
44 44
 You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).

+ 1
- 1
elements/hadoop/README.md View File

@@ -1 +1 @@
1
-Installs Java and Hadoop on Ubuntu
1
+Installs Java and Hadoop, configures SSH

+ 28
- 10
elements/hadoop/first-boot.d/99-setup View File

@@ -1,12 +1,30 @@
1 1
 #!/bin/bash
2
+
3
+# Common
2 4
 chown root:root /mnt
3
-mkdir -p /mnt/log/hadoop
4
-chown hadoop:hadoop /mnt/log/hadoop
5
-mkdir /var/run/hadoop
6
-mkdir /run/hadoop
7
-chown hadoop:hadoop /var/run/hadoop/
8
-chown hadoop:hadoop /run/hadoop/
9
-echo "Setup ubuntu's directory"
10
-mkdir -p /home/ubuntu/.ssh
11
-touch /home/ubuntu/.ssh/authorized_keys
12
-chown -R ubuntu:ubuntu /home/ubuntu
5
+mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
6
+mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
7
+
8
+distro=$(lsb_release -is || :)
9
+case "$distro" in
10
+   Ubuntu )
11
+      mkdir /run/hadoop
12
+      chown hadoop:hadoop /run/hadoop/
13
+      mkdir -p /home/ubuntu/.ssh
14
+      touch /home/ubuntu/.ssh/authorized_keys
15
+      chown -R ubuntu:ubuntu /home/ubuntu
16
+   ;;
17
+   Fedora )
18
+      sleep 10
19
+      rm /etc/resolv.conf
20
+      service network restart
21
+      chown -R ec2-user:ec2-user /home/ec2-user
22
+      chown -R ec2-user:ec2-user /etc/hadoop
23
+      #TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
24
+      iptables -F
25
+   ;;
26
+   * )
27
+      echo "Unknown distro: $distro. Exiting."
28
+      exit 1
29
+   ;;
30
+esac

elements/hadoop_fedora/install.d/20-setup-java → elements/hadoop/install.d/20-setup-java View File


elements/hadoop_fedora/install.d/30-setup-hadoop → elements/hadoop/install.d/30-setup-hadoop View File


elements/hadoop_fedora/install.d/40-setup-ssh → elements/hadoop/install.d/40-setup-ssh View File


+ 0
- 43
elements/hadoop/install.d/70-setup-java View File

@@ -1,43 +0,0 @@
1
-#!/bin/bash
2
-tmp_dir=/tmp/java
3
-mkdir -p $TARGET_ROOT/$tmp_dir
4
-echo "Java setup begins"
5
-set -e
6
-if [ -z "$JAVA_DOWNLOAD_URL" ]; then
7
-   mkdir -p $TARGET_ROOT/$tmp_dir
8
-   script_dir=$(dirname $0)
9
-   install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
10
-   echo "Java file moved"
11
-else
12
-   wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
13
-   if [ $? -eq 0 ]; then
14
-      echo "Java is downloading"
15
-      else
16
-      echo "Url error. Exit"
17
-      exit 1
18
-   fi
19
-fi
20
-filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
21
-filename=$(basename $filename)
22
-
23
-if echo $filename | grep -q -s -F .tar.gz ; then
24
-   path=$(pwd)
25
-   cd $TARGET_ROOT/$tmp_dir
26
-   echo $TARGET_ROOT/$tmp_dir/$filename | xargs -n 1 tar -zxvf
27
-   cd $path
28
-   else
29
-      if echo $filename | grep -q -s -F .bin ; then
30
-         path=$(pwd)
31
-         cd $TARGET_ROOT/$tmp_dir
32
-         echo -e "\n" | sh $filename
33
-         rm $filename
34
-         cd $path
35
-      fi
36
-fi
37
-
38
-filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
39
-filename=$(basename $filename)
40
-javaPath=/usr/lib/
41
-mkdir -p $TARGET_ROOT/$javaPath
42
-mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
43
-echo "Java was installed"

+ 0
- 37
elements/hadoop/install.d/80-setup-hadoop View File

@@ -1,37 +0,0 @@
1
-#!/bin/bash
2
-tmp_dir=/tmp/hadoop
3
-mkdir -p $TARGET_ROOT/$tmp_dir
4
-echo "Installing SSH server"
5
-
6
-mkdir /root/.ssh
7
-install-packages openssh-server
8
-
9
-echo "Creating hadoop user & group"
10
-
11
-addgroup hadoop
12
-adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
13
-adduser hadoop sudo
14
-
15
-echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
16
-
17
-wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop_"$DIB_HADOOP_VERSION"-1_x86_64.deb"
18
-if [ $? -ne 0 ]; then
19
-   echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nExit"
20
-   exit 1
21
-fi
22
-
23
-echo "Installing Hadoop"
24
-
25
-dpkg -i $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
26
-rm $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
27
-
28
-echo "Pre-configuring Hadoop"
29
-
30
-filename=$(find $TARGET_ROOT/usr/lib/ -maxdepth 1 -name "jdk*")
31
-echo -e "HADOOP_HOME=/usr/share/hadoop/\nPATH=\$PATH:/usr/sbin/" >> $TARGET_ROOT/home/hadoop/.bashrc
32
-echo -e "JAVA_HOME=$filename\nPATH=\$PATH:$filename/bin/" >> $TARGET_ROOT/home/hadoop/.bashrc
33
-sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
34
-log_dir=/mnt/log/hadoop/\$USER/
35
-sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
36
-log_dir=/mnt/log/hadoop/hdfs
37
-sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh

+ 0
- 8
elements/hadoop/install.d/90-setup-ssh View File

@@ -1,8 +0,0 @@
1
-#!/bin/bash
2
-sed -i -e 's/PasswordAuthentication no/PasswordAuthentication yes/' \
3
-       -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' \
4
-    /etc/ssh/sshd_config
5
-echo -e "AuthorizedKeysFile .ssh/authorized_keys\nUseDNS no\nPermitTunnel yes\nGSSAPICleanupCredentials yes" >> /etc/ssh/sshd_config
6
-
7
-sed -i -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
8
-echo -e "StrictHostKeyChecking no" >> /etc/ssh/ssh_config

+ 9
- 5
elements/hadoop/root.d/0-check View File

@@ -1,11 +1,16 @@
1 1
 #!/bin/bash
2 2
 set -e
3
-source $_LIB/die
3
+
4 4
 if [ -z "$JAVA_DOWNLOAD_URL" ]; then
5
-   [ -n "$JAVA_FILE" ] || die "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
5
+   if [ -z "$JAVA_FILE" ]; then
6
+      echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
7
+      exit 1
8
+   fi
9
+fi
10
+if [ -z "$DIB_HADOOP_VERSION" ]; then
11
+   echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
12
+   exit 1
6 13
 fi
7
-[ -n "$DIB_HADOOP_VERSION" ] || die "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
8
-
9 14
 version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
10 15
 if [[ -z $version_check ]]; then
11 16
    echo "All variables are set, continue."
@@ -13,4 +18,3 @@ else
13 18
    echo "Version error. Exit"
14 19
    exit 1
15 20
 fi
16
-

+ 0
- 1
elements/hadoop_fedora/README.md View File

@@ -1 +0,0 @@
1
-Installs Java and Hadoop on Fedora

+ 0
- 30
elements/hadoop_fedora/first-boot.d/99-setup View File

@@ -1,30 +0,0 @@
1
-#!/bin/bash
2
-
3
-# Common
4
-chown root:root /mnt
5
-mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
6
-mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
7
-
8
-distro=$(lsb_release -is || :)
9
-case "$distro" in
10
-   Ubuntu )
11
-      mkdir /run/hadoop
12
-      chown hadoop:hadoop /run/hadoop/
13
-      mkdir -p /home/ubuntu/.ssh
14
-      touch /home/ubuntu/.ssh/authorized_keys
15
-      chown -R ubuntu:ubuntu /home/ubuntu
16
-   ;;
17
-   Fedora )
18
-      sleep 10
19
-      rm /etc/resolv.conf
20
-      service network restart
21
-      chown -R ec2-user:ec2-user /home/ec2-user
22
-      chown -R ec2-user:ec2-user /etc/hadoop
23
-      #TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
24
-      iptables -F
25
-   ;;
26
-   * )
27
-      echo "Unknown distro: $distro. Exiting."
28
-      exit 1
29
-   ;;
30
-esac

+ 0
- 20
elements/hadoop_fedora/root.d/0-check View File

@@ -1,20 +0,0 @@
1
-#!/bin/bash
2
-set -e
3
-
4
-if [ -z "$JAVA_DOWNLOAD_URL" ]; then
5
-   if [ -z "$JAVA_FILE" ]; then
6
-      echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
7
-      exit 1
8
-   fi
9
-fi
10
-if [ -z "$DIB_HADOOP_VERSION" ]; then
11
-   echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
12
-   exit 1
13
-fi
14
-version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
15
-if [[ -z $version_check ]]; then
16
-   echo "All variables are set, continue."
17
-else
18
-   echo "Version error. Exit"
19
-   exit 1
20
-fi

Loading…
Cancel
Save