openstack/sahara-image-elements
Code Issues Proposed changes

Drop hadoop/vanilla v1 images

Browse Source 
Drop support for generating vanilla/hadoop v1 images.  This version is
deprecated in Sahara, and the support for going to be removed soon.

Partially-implements bp: drop-hadoop-1

Change-Id: I7b0d36d255a783057c7c5876017653e0f28f83de
This commit is contained in:
Pino Toscano 2015-09-14 17:54:20 +02:00
parent e48ecca752
commit 5e936d63d1
8 changed files with 9 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
diskimage-create/diskimage-create.sh
View File

@ -26,7 +26,7 @@ usage() {
echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|hdp|cloudera|storm|mapr|plain]"
echo " [-i ubuntu|fedora|centos|centos7]"
echo " [-v 1|2|2.6|2.7.1|4|5.0|5.3|5.4]"
echo " [-v 2|2.6|2.7.1|4|5.0|5.3|5.4]"
echo " [-r 3.1.1|4.0.1|4.0.2|5.0.0]"
echo " [-s <Spark version>]"
echo " [-d]"
@ -127,7 +127,7 @@ case "$PLUGIN" in
"");;
"vanilla")
case "$HADOOP_VERSION" in
"" | "1" | "2.6" | "2.7.1");;
"" | "2.6" | "2.7.1");;
*)
echo -e "Unknown hadoop version selected.\nAborting"
exit 1
@ -377,7 +377,6 @@ fi
#############################
if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
export OOZIE_HADOOP_V1_DOWNLOAD_URL=${OOZIE_HADOOP_V1_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0.tar.gz"}
export OOZIE_HADOOP_V2_6_DOWNLOAD_URL=${OOZIE_HADOOP_V2_6_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.1-hadoop-2.6.0.tar.gz"}
export HADOOP_V2_6_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_6_NATIVE_LIBS_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/hadoop-native-libs-2.6.0.tar.gz"}
export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://dev.sencha.com/deploy/ext-2.2.zip"}
@ -422,12 +421,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_1:-"1.2.1"}
export ubuntu_image_name=${ubuntu_vanilla_hadoop_1_image_name:-"ubuntu_sahara_vanilla_hadoop_1_latest"}
elements_sequence="$ubuntu_elements_sequence swift_hadoop"
disk-image-create $TRACING $elements_sequence -o $ubuntu_image_name
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.6" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_6:-"2.6.0"}
export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_6_image_name:-"ubuntu_sahara_vanilla_hadoop_2_6_latest"}
@ -443,12 +436,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
# Fedora cloud image
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "fedora" ]; then
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_1:-"1.2.1"}
export fedora_image_name=${fedora_vanilla_hadoop_1_image_name:-"fedora_sahara_vanilla_hadoop_1_latest$suffix"}
elements_sequence="$fedora_elements_sequence swift_hadoop"
disk-image-create $TRACING $elements_sequence -o $fedora_image_name
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.6" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_6:-"2.6.0"}
export fedora_image_name=${fedora_vanilla_hadoop_2_6_image_name:-"fedora_sahara_vanilla_hadoop_2_6_latest$suffix"}
@ -468,12 +455,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
# Read Create_CentOS_cloud_image.rst to know how to create CentOS image in qcow2 format
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20150821.qcow2"
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_1:-"1.2.1"}
export centos_image_name=${centos_vanilla_hadoop_1_image_name:-"centos_sahara_vanilla_hadoop_1_latest$suffix"}
elements_sequence="$centos_elements_sequence swift_hadoop"
disk-image-create $TRACING $elements_sequence -o $centos_image_name
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.6" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_6:-"2.6.0"}
export centos_image_name=${centos_vanilla_hadoop_2_6_image_name:-"centos_sahara_vanilla_hadoop_2_6_latest$suffix"}

50
elements/hadoop/post-install.d/40-setup-hadoop
View File

@ -13,50 +13,6 @@ fi
set -eu
set -o pipefail
function install_hadoop_v1 {
case "$DISTRO_NAME" in
ubuntu )
package="hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb"
;;
fedora | rhel | centos )
package="hadoop-$DIB_HADOOP_VERSION-1.x86_64.rpm"
;;
esac
case "$DISTRO_NAME" in
ubuntu )
dpkg -i $tmp_dir/$package
;;
fedora | rhel | centos )
if [ "$DISTRO_NAME" = "fedora" ] && [ $(lsb_release -rs) -ge '19' ]; then
rpm -i $tmp_dir/$package --relocate /usr=/usr --replacefiles
else
rpm -ivh --replacefiles $tmp_dir/$package
fi
chmod 755 /usr/sbin/start-*
chmod 755 /usr/sbin/stop-*
chmod 755 /usr/sbin/slaves.sh
chmod 755 /usr/sbin/update-hadoop-env.sh
;;
esac
rm -r $tmp_dir
echo "Pre-configuring Hadoop"
cat >> /home/hadoop/.bashrc <<EOF
PATH=\$PATH:/usr/sbin
HADOOP_HOME=/usr/share/hadoop/
EOF
sed -i -e "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=/mnt/log/hadoop/\$USER," \
-e "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=/mnt/log/hadoop/hdfs," \
-e "s,HADOOP_DATANODE_OPTS=\"-Dhadoop.security.logger=ERROR\,DRFAS \$HADOOP_DATANODE_OPTS\",HADOOP_DATANODE_OPTS=\"-Djava.security.egd=file:/dev/./urandom -Dhadoop.security.logger=ERROR\,DRFAS \$HADOOP_DATANODE_OPTS\"," \
/etc/hadoop/hadoop-env.sh
echo "source $JAVA_RC" >> /etc/hadoop/hadoop-env.sh
}
function install_hadoop_v2 {
case "$DIB_HADOOP_VERSION" in
"2.6.0")
@ -143,11 +99,7 @@ case "$DISTRO_NAME" in
;;
esac
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
install_hadoop_v1
else
install_hadoop_v2
fi
install_hadoop_v2
echo "Applying firstboot script"

28
elements/hadoop/post-install.d/firstboot
View File

@ -53,38 +53,10 @@ function firstboot_common {
chown hadoop:hadoop $RUNDIR/hadoop
}
function firstboot_hadoop_v1 {
distro=$(lsb_release -is || :)
case "$distro" in
Ubuntu )
chown -R ubuntu:ubuntu /etc/hadoop
;;
Fedora )
if [ $(lsb_release -rs) -ge '19' ]; then
user=fedora
else
user=ec2-user
fi
chown -R $user:$user /etc/hadoop
;;
CentOS )
chown -R cloud-user:cloud-user /etc/hadoop
;;
* )
echo "Unknown distro: $distro. Exiting."
exit 1
;;
esac
}
DIB_HADOOP_VERSION=$(su - hadoop hadoop version | head -1 | awk '{print $2}')
firstboot_common
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
firstboot_hadoop_v1
fi
# Clean
if [ "$distro" == "Ubuntu" ]; then
if [ -f /etc/rc.local.old ]; then

19
elements/hadoop/root.d/50-download-hadoop
View File

@ -14,19 +14,6 @@ function download_hadoop_package {
sudo install -D -g root -o root -m 0644 $cached_package $tmp_dir
}
function download_hadoop_v1 {
case "$DISTRO_NAME" in
ubuntu )
package="hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb"
;;
fedora | rhel | centos )
package="hadoop-$DIB_HADOOP_VERSION-1.x86_64.rpm"
;;
esac
download_hadoop_package $package
}
function download_hadoop_v2 {
download_hadoop_package "hadoop-$DIB_HADOOP_VERSION.tar.gz"
}
@ -43,8 +30,4 @@ esac
tmp_dir=$TARGET_ROOT/tmp/hadoop
mkdir -p $tmp_dir
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
download_hadoop_v1
else
download_hadoop_v2
fi
download_hadoop_v2

8
elements/oozie/install.d/50-setup-oozie
View File

@ -10,13 +10,7 @@ echo "Oozie setup"
tmp_dir=/tmp/oozie
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
if [ -z "${OOZIE_HADOOP_V1_DOWNLOAD_URL:-}" ]; then
OOZIE_FILE=$(basename $OOZIE_HADOOP_V1_FILE)
else
OOZIE_FILE=$(basename $OOZIE_HADOOP_V1_DOWNLOAD_URL)
fi
elif [[ "$DIB_HADOOP_VERSION" == "2.6.0" ]]; then
if [[ "$DIB_HADOOP_VERSION" == "2.6.0" ]]; then
if [ -z "${OOZIE_HADOOP_V2_6_DOWNLOAD_URL:-}" ]; then
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_6_FILE)
else

9
elements/oozie/post-install.d/90-oozie-war
View File

@ -9,12 +9,7 @@ set -o pipefail
ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/libtools/mysql.jar
ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
cp -r /usr/share/hadoop/*.jar /opt/oozie/libext
cp -r /usr/share/hadoop/lib/*.jar /opt/oozie/libext
else
wget -P /opt/oozie/libtools/ "$EXTJS_DOWNLOAD_URL"
/opt/oozie/bin/oozie-setup.sh prepare-war -d /opt/oozie/libtools
fi
wget -P /opt/oozie/libtools/ "$EXTJS_DOWNLOAD_URL"
/opt/oozie/bin/oozie-setup.sh prepare-war -d /opt/oozie/libtools
chown -R hadoop:hadoop /opt/oozie

7
elements/oozie/root.d/0-check-oozie
View File

@ -6,12 +6,7 @@ fi
set -eu
set -o pipefail
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
if [ -z "${OOZIE_HADOOP_V1_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V1_FILE:-}" ]; then
echo "OOZIE_HADOOP_V1_FILE and OOZIE_HADOOP_V1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
exit 1
fi
elif [ "$DIB_HADOOP_VERSION" == "2.6.0" ]; then
if [ "$DIB_HADOOP_VERSION" == "2.6.0" ]; then
if [ -z "${OOZIE_HADOOP_V2_6_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_6_FILE:-}" ]; then
echo "OOZIE_HADOOP_V2_6_FILE and OOZIE_HADOOP_V2_6_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
exit 1

5
elements/oozie/root.d/50-download-oozie
View File

@ -9,10 +9,7 @@ set -o pipefail
tmp_dir=$TARGET_ROOT/tmp/oozie
mkdir -p $tmp_dir
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V1_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V1_FILE:-}"
elif [[ "$DIB_HADOOP_VERSION" == "2.6.0" ]]; then
if [[ "$DIB_HADOOP_VERSION" == "2.6.0" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_6_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V2_6_FILE:-}"
elif [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then