Browse Source

Add Hadoop element for creating Fedora cloud image for Savanna v0.2.

* ssh configuration
* java and hadoop installation from tmp directory
* setup network connection

Implements blueprint savanna-image-elements
Fixes bug: https://bugs.launchpad.net/savanna/+bug/1188442

Change-Id: I5b6c6ef3b78e29967b7869e3499f6c146d309dae
Ivan Berezovskiy 5 years ago
parent
commit
17b87b9876

+ 8
- 1
elements/README.rst View File

@@ -32,8 +32,15 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
32 32
 
33 33
 .. sourcecode:: bash
34 34
 
35
-    JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-password -o hadoop_1_1_2
35
+    JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-passwd -o hadoop_1_1_2
36
+
37
+6.2. Fedora cloud image
38
+
39
+.. sourcecode:: bash
40
+
41
+    DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2
36 42
 
37 43
 In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
38 44
 You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).
45
+'DIB_IMAGE_SIZE' is parameter that specifes a volume of hard disk of instance. You need to specify it because Fedora doesn't use all available volume.
39 46
 In case if you have already downloaded jdk package, move it to "elements/hadoop/install.d/" and use its filename as 'JAVA_FILE' parameter.

+ 1
- 0
elements/hadoop_fedora/README.md View File

@@ -0,0 +1 @@
1
+Installs Java and Hadoop on Fedora

+ 13
- 0
elements/hadoop_fedora/first-boot.d/99-setup View File

@@ -0,0 +1,13 @@
1
+#!/bin/bash
2
+sleep 10
3
+rm /etc/resolv.conf
4
+service network restart
5
+chown -R ec2-user:ec2-user /home/ec2-user
6
+chown -R ec2-user:ec2-user /etc/hadoop
7
+mkdir -p /var/run/hadoop
8
+chown root:root /mnt
9
+mkdir -p /mnt/log/hadoop
10
+chown hadoop:hadoop /var/run/hadoop
11
+chown hadoop:hadoop /mnt/log/hadoop
12
+#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
13
+iptables -F

+ 44
- 0
elements/hadoop_fedora/post-install.d/11-setup-java View File

@@ -0,0 +1,44 @@
1
+#!/bin/bash
2
+echo "Java setup begins"
3
+install-packages wget
4
+tmp_dir=/tmp/java/
5
+mkdir -p $TARGET_ROOT/$tmp_dir
6
+set -e
7
+if [ -z "$JAVA_DOWNLOAD_URL" ]; then
8
+   script_dir=$(dirname $0)
9
+   install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
10
+   filename=$JAVA_FILE
11
+   echo "Java file moved"
12
+else
13
+   wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
14
+   if [ $? -eq 0 ]; then
15
+      echo "Java is downloading"
16
+      else
17
+      echo "Url error. Exit"
18
+      exit 1
19
+   fi
20
+   filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
21
+   filename=$(basename $filename)
22
+fi
23
+
24
+if echo $tmp_dir/$filename | grep -q -s -F .tar.gz ; then
25
+   pushd $TARGET_ROOT/$tmp_dir
26
+   echo -e "\n" | tar -zxvf $TARGET_ROOT/$tmp_dir/$filename
27
+   popd
28
+   else
29
+      if echo $filename | grep -q -s -F .bin ; then
30
+         pushd $TARGET_ROOT/$tmp_dir
31
+         echo -e "\n" | sh $filename
32
+         rm $filename
33
+         popd
34
+      fi
35
+fi
36
+
37
+filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
38
+filename=$(basename $filename)
39
+javaPath=/usr/java/
40
+mkdir -p $TARGET_ROOT/$javaPath
41
+mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
42
+rm -r $tmp_dir
43
+echo "Java was installed"
44
+

+ 32
- 0
elements/hadoop_fedora/post-install.d/12-setup-hadoop View File

@@ -0,0 +1,32 @@
1
+#!/bin/bash
2
+echo "Hadoop setup begins"
3
+tmp_dir=/tmp/hadoop
4
+
5
+install-packages openssh-server
6
+echo "Creating hadoop user & group"
7
+adduser -G adm,wheel hadoop
8
+
9
+echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
10
+wget -P $tmp_dir/ "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm"
11
+if [ $? -ne 0 ]; then
12
+   echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
13
+   exit 1
14
+fi
15
+
16
+rpm -ivh --replacefiles $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
17
+chmod 755 /usr/sbin/start-*
18
+chmod 755 /usr/sbin/stop-*
19
+chmod 755 /usr/sbin/slaves.sh
20
+chmod 755 /usr/sbin/update-hadoop-env.sh
21
+rm $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
22
+rm -r $tmp_dir
23
+
24
+echo "Pre-configuring Hadoop"
25
+filename=$(find $TARGET_ROOT/usr/java/ -maxdepth 1 -name "jdk*")
26
+echo -e "PATH=\$PATH:/usr/sbin:$filename/bin\nJAVA_HOME=$filename" >> /home/hadoop/.bashrc
27
+echo -e "HADOOP_HOME=/usr/share/hadoop/" >> /home/hadoop/.bashrc
28
+sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
29
+log_dir=/mnt/log/hadoop/\$USER/
30
+sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
31
+log_dir=/mnt/log/hadoop/hdfs
32
+sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh

+ 21
- 0
elements/hadoop_fedora/post-install.d/13-connection-setup View File

@@ -0,0 +1,21 @@
1
+#!/bin/bash
2
+echo "Adjusting ssh configuration"
3
+
4
+sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
5
+echo "UseDNS no" >> /etc/ssh/sshd_config
6
+echo "PermitTunnel yes" >> /etc/ssh/sshd_config
7
+echo "SyslogFacility AUTH" >> /etc/ssh/sshd_config
8
+echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
9
+echo "StrictModes yes" >> /etc/ssh/sshd_config
10
+echo "RSAAuthentication yes" >> /etc/ssh/sshd_config
11
+echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
12
+echo "IgnoreRhosts yes" >> /etc/ssh/sshd_config
13
+
14
+echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
15
+echo "GSSAPIDelegateCredentials no" >> /etc/ssh/ssh_config
16
+sed -i 's/        GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
17
+
18
+sed -i 's/ssh_pwauth:   0/ssh_pwauth:   1/' /etc/cloud/cloud.cfg
19
+chmod 640 /etc/sudoers
20
+sed -i 's/Defaults    requiretty/#Defaults    requiretty/' /etc/sudoers
21
+chmod 0440 /etc/sudoers

+ 20
- 0
elements/hadoop_fedora/root.d/0-check View File

@@ -0,0 +1,20 @@
1
+#!/bin/bash
2
+set -e
3
+
4
+if [ -z "$JAVA_DOWNLOAD_URL" ]; then
5
+   if [ -z "$JAVA_FILE" ]; then
6
+      echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
7
+      exit 1
8
+   fi
9
+fi
10
+if [ -z "$DIB_HADOOP_VERSION" ]; then
11
+   echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
12
+   exit 1
13
+fi
14
+version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
15
+if [[ -z $version_check ]]; then
16
+   echo "All variables are set, continue."
17
+else
18
+   echo "Version error. Exit"
19
+   exit 1
20
+fi

Loading…
Cancel
Save