Adding Storm and Zookeeper elements
This patch implements storm element and zookeeper. They are both needed by storm plugin. Change-Id: If279457b8f2b0fc18dca031afa979d0485bbbd81 Partial-implements: blueprint storm-integration
This commit is contained in:
parent
8727996fbc
commit
43d18bf020
0
classdist_noinst.stamp
Normal file
0
classdist_noinst.stamp
Normal file
@ -37,7 +37,7 @@ while getopts "p:i:v:d:m" opt; do
|
||||
*)
|
||||
echo
|
||||
echo "Usage: $(basename $0)"
|
||||
echo " [-p vanilla|spark|hdp|cloudera]"
|
||||
echo " [-p vanilla|spark|hdp|cloudera|storm]"
|
||||
echo " [-i ubuntu|fedora|centos]"
|
||||
echo " [-v 1|2|2.3|2.4|plain]"
|
||||
echo " [-d]"
|
||||
@ -81,7 +81,7 @@ if [ "$DEBUG_MODE" = "true" -a "$platform" != 'NAME="Ubuntu"' ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" ]; then
|
||||
if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" -a "$PLUGIN" != "storm" ]; then
|
||||
echo -e "Unknown plugin selected.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
@ -321,6 +321,30 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
|
||||
mv $ubuntu_image_name.qcow2 ../
|
||||
fi
|
||||
|
||||
|
||||
##########################
|
||||
# Image for Storm plugin #
|
||||
##########################
|
||||
|
||||
if [ -z "$PLUGIN" -o "$PLUGIN" = "storm" ]; then
|
||||
# Ignoring image type and hadoop version options
|
||||
echo "For storm plugin options -i and -v are ignored"
|
||||
|
||||
export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"}
|
||||
export DIB_STORM_VERSION=${DIB_STORM_VERSION:-0.9.1}
|
||||
export ubuntu_image_name=${ubuntu_storm_image_name:-"ubuntu_sahara_storm_latest"}
|
||||
|
||||
ubuntu_elements_sequence="base vm ubuntu java zookeeper storm"
|
||||
|
||||
if [ -n "$USE_MIRRORS" ]; then
|
||||
mirror_element=" apt-mirror"
|
||||
ubuntu_elements_sequence=$ubuntu_elements_sequence$mirror_element
|
||||
fi
|
||||
|
||||
# Creating Ubuntu cloud image
|
||||
disk-image-create $ubuntu_elements_sequence -o $ubuntu_image_name
|
||||
mv $ubuntu_image_name.qcow2 ../
|
||||
fi
|
||||
#########################
|
||||
# Images for HDP plugin #
|
||||
#########################
|
||||
|
1
elements/storm/element-deps
Normal file
1
elements/storm/element-deps
Normal file
@ -0,0 +1 @@
|
||||
zookeeper
|
76
elements/storm/install.d/60-storm
Executable file
76
elements/storm/install.d/60-storm
Executable file
@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
# This script installs storm and its dependencies
|
||||
# More documentation on the README.md file
|
||||
|
||||
install-packages wget unzip build-essential uuid-dev git libtool autoconf pkg-config tar
|
||||
|
||||
tmp_dir=/tmp
|
||||
|
||||
cd $tmp_dir
|
||||
echo "Installing ZeroMQ"
|
||||
wget http://download.zeromq.org/zeromq-2.1.7.tar.gz
|
||||
tar -xzf zeromq-2.1.7.tar.gz
|
||||
cd zeromq-2.1.7
|
||||
./configure
|
||||
make
|
||||
make install
|
||||
cd ..
|
||||
|
||||
echo "Installing JZMQ"
|
||||
git clone https://github.com/nathanmarz/jzmq.git
|
||||
|
||||
cd jzmq/src
|
||||
touch classdist_noinst.stamp
|
||||
CLASSPATH=.:./.:$CLASSPATH javac -d . org/zeromq/ZMQ.java org/zeromq/ZMQException.java org/zeromq/ZMQQueue.java org/zeromq/ZMQForwarder.java org/zeromq/ZMQStreamer.java
|
||||
cd ..
|
||||
|
||||
cd $tmp_dir/jzmq
|
||||
./autogen.sh
|
||||
./configure
|
||||
make
|
||||
make install
|
||||
|
||||
echo "Creating Storm User"
|
||||
groupadd -g 53001 storm
|
||||
mkdir -p /app/home/storm
|
||||
useradd -u 53001 -g 53001 -d /app/home/storm -s /bin/bash storm -c "Storm service account"
|
||||
chmod 700 /app/home/storm
|
||||
chage -I -1 -E -1 -m -1 -M -1 -W -1 -E -1 storm
|
||||
|
||||
cd $tmp_dir
|
||||
echo "Downloading Storm"
|
||||
# The user is not providing his own Spark distribution package
|
||||
if [ -z "$STORM_DOWNLOAD_URL" ]; then
|
||||
# Check storm version
|
||||
case "$DIB_STORM_VERSION" in
|
||||
0.9.*)
|
||||
STORM_DOWNLOAD_URL="http://archive.apache.org/dist/incubator/storm/apache-storm-$DIB_STORM_VERSION-incubating/apache-storm-$DIB_STORM_VERSION-incubating.tar.gz"
|
||||
;;
|
||||
*)
|
||||
echo -e "WARNING: Storm version $DIB_STORM_VERSION not supported."
|
||||
echo -e "WARNING: make sure STORM_DOWNLOAD_URL points to a compatible Storm version."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
wget $STORM_DOWNLOAD_URL
|
||||
cd /usr/local
|
||||
tar -xvf /tmp/apache-storm-$DIB_STORM_VERSION-incubating.tar.gz
|
||||
chown -R storm:storm apache-storm-$DIB_STORM_VERSION-incubating
|
||||
ln -s apache-storm-$DIB_STORM_VERSION-incubating storm
|
||||
|
||||
|
||||
echo "Configuring Storm home"
|
||||
mkdir -p /app/storm
|
||||
chown -R storm:storm /app/storm
|
||||
chmod 750 /app/storm
|
||||
|
||||
echo "Installing Supervisor Deamon"
|
||||
|
||||
install-packages supervisor
|
||||
update-rc.d supervisor defaults
|
||||
chmod 600 /etc/supervisord.conf
|
||||
|
||||
mkdir -p /var/log/storm
|
||||
chown -R storm:storm /var/log/storm
|
5
elements/storm/root.d/0-check
Executable file
5
elements/storm/root.d/0-check
Executable file
@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
if [ -z "$STORM_DOWNLOAD_URL" -a -z "$DIB_STORM_VERSION"]; then
|
||||
echo -e "Neither DIB_STORM_VERSION nor STORM_DOWNLOAD_URL are set. Impossible to install Storm.\nAborting"
|
||||
exit 1
|
||||
fi
|
19
elements/zookeeper/install.d/40-zookeeper
Executable file
19
elements/zookeeper/install.d/40-zookeeper
Executable file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# This scrip installs zookeeper
|
||||
# More documentation on the README.md file
|
||||
|
||||
install-packages wget
|
||||
|
||||
echo "Downloading Zookeeper"
|
||||
|
||||
mkdir -p /opt/zookeeper
|
||||
|
||||
cd /opt/zookeeper
|
||||
|
||||
wget http://archive.apache.org/dist/zookeeper/stable/zookeeper-3.4.6.tar.gz
|
||||
tar -xzf zookeeper-3.4.6.tar.gz
|
||||
rm -rf zookeeper-3.4.6.tar.gz
|
||||
|
||||
PID_PATH="/var/zookeeper"
|
||||
|
||||
mkdir -p $PID_PATH
|
15
elements/zookeeper/post-install.d/50-zookeeper
Executable file
15
elements/zookeeper/post-install.d/50-zookeeper
Executable file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
# We have to chown the Zookeeper directories to make it usable for the default user
|
||||
# Unfortunately the Ubuntu user does not exists when DIB is creating the image
|
||||
# and we need to execute this code the first time the VM boots.
|
||||
|
||||
sed -i -e 's,^exit 0$,[ -f /opt/zookeeper/firstboot.sh ] \&\& sh /opt/zookeeper/firstboot.sh; exit 0,' /etc/rc.local
|
||||
|
||||
cat >> /opt/zookeeper/firstboot.sh <<EOF
|
||||
#!/bin/sh
|
||||
chown -R ubuntu:ubuntu /opt/zookeeper
|
||||
chown -R ubuntu:ubuntu /var/zookeeper
|
||||
rm \$0
|
||||
EOF
|
||||
|
Loading…
Reference in New Issue
Block a user