openstack/sahara-image-elements
Code Issues Proposed changes

Add support to build 3.0.1 image

Browse Source 
Change-Id: I1d90099b22ed9a0ede08df45059e6d7848c237b8
This commit is contained in:
Zhuang Changkun 2018-06-08 14:11:37 +08:00
parent 5d05a3db2f
commit 83224a6c5e
8 changed files with 55 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
diskimage-create/diskimage-create.sh
View File

@ -32,7 +32,7 @@ usage() {
echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
echo " [-i ubuntu|fedora|centos7]"
echo " [-v 2.7.1|2.7.5|2.8.2|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-v 2.7.1|2.7.5|2.8.2|3.0.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-r 5.1.0|5.2.0]"
echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
@ -170,7 +170,7 @@ case "$PLUGIN" in
"");;
"vanilla")
case "$HADOOP_VERSION" in
"" | "2.7.1" | "2.7.5" | "2.8.2");;
"" | "2.7.1" | "2.7.5" | "2.8.2" | "3.0.1");;
*)
echo -e "Unknown hadoop version selected.\nAborting"
exit 1
@ -447,6 +447,9 @@ set_hive_version() {
"2.8.2" )
export HIVE_VERSION="2.3.2"
;;
"3.0.1" )
export HIVE_VERSION="3.0.0"
;;
*)
echo -e "Unknown Hadoop version, therefore cannot choose Hive version.\nAborting."
exit 1
@ -464,9 +467,11 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"}
export HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.5.tar.gz"}
export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"}
export HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-3.0.1.tar.gz"}
export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"}
export OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_5_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.7.5.tar.gz"}
export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"}
export OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL=${OOZIE_HADOOP_V3_0_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-5.0.0-hadoop-3.0.1.tar.gz"}
export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib"
export plugin_type="vanilla"
@ -527,6 +532,14 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
unset DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"}
export ubuntu_image_name=${ubuntu_vanilla_hadoop_3_0_1_image_name:-"ubuntu_sahara_vanilla_hadoop_3_0_1_latest"}
export DIB_RELEASE=${DIB_RELEASE:-xenial}
set_hive_version
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
unset DIB_RELEASE
fi
unset DIB_CLOUD_INIT_DATASOURCES
fi
@ -550,6 +563,12 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
set_hive_version
image_create fedora $fedora_image_name $fedora_elements_sequence
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"}
export fedora_image_name=${fedora_vanilla_hadoop_3_0_1_image_name:-"fedora_sahara_vanilla_hadoop_3_0_1_latest$suffix"}
set_hive_version
image_create fedora $fedora_image_name $fedora_elements_sequence
fi
fi
# CentOS 7 cloud image
@ -572,6 +591,12 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
set_hive_version
image_create centos7 $centos7_image_name $centos7_elements_sequence
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"}
export centos7_image_name=${centos7_vanilla_hadoop_3_0_1_image_name:-"centos7_sahara_vanilla_hadoop_3_0_1_latest$suffix"}
set_hive_version
image_create centos7 $centos7_image_name $centos7_elements_sequence
fi
fi
unset plugin_type

3
elements/hadoop/install.d/40-setup-hadoop
View File

@ -17,6 +17,9 @@ function install_hadoop_v2 {
"2.8.2")
hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}"
;;
"3.0.1")
hadoop_native_libs_url="${HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL}"
;;
*)
echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION"
exit 1

6
elements/oozie/install.d/50-setup-oozie
View File

@ -28,6 +28,12 @@ elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
else
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL)
fi
elif [[ "$DIB_HADOOP_VERSION" == "3.0.1" ]]; then
if [ -z "${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" ]; then
OOZIE_FILE=$(basename $OOZIE_HADOOP_V3_0_1_FILE)
else
OOZIE_FILE=$(basename $OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL)
fi
else
echo "Unknown Hadoop version selected. Aborting."
exit 1

10
elements/oozie/post-install.d/90-oozie-war
View File

@ -7,8 +7,12 @@ set -eu
set -o pipefail
ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/libtools/mysql.jar
ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar
/opt/oozie/bin/oozie-setup.sh prepare-war -d /opt/oozie/libtools
if [ "$DIB_HADOOP_VERSION" = "3.0.1" ]; then
ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/embedded-oozie-server/dependency/mysql.jar
/opt/oozie/bin/oozie-setup.sh -d /opt/oozie/libtools
else
ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar
/opt/oozie/bin/oozie-setup.sh prepare-war -d /opt/oozie/libtools
fi
chown -R hadoop:hadoop /opt/oozie

5
elements/oozie/root.d/0-check-oozie
View File

@ -21,6 +21,11 @@ elif [ "$DIB_HADOOP_VERSION" == "2.8.2" ]; then
echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
exit 1
fi
elif [ "$DIB_HADOOP_VERSION" == "3.0.1" ]; then
if [ -z "${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V3_0_1_FILE:-}" ]; then
echo "OOZIE_HADOOP_V3_0_1_FILE and OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
exit 1
fi
else
echo "Unknown Hadoop version selected. Aborting"
exit 1

3
elements/oozie/root.d/50-download-oozie
View File

@ -18,6 +18,9 @@ elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then
elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}"
elif [[ "$DIB_HADOOP_VERSION" == "3.0.1" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V3_0_1_FILE:-}"
else
echo "Unknown Hadoop version selected. Aborting."
exit 1

2
elements/swift_hadoop/post-install.d/81-add-jar
View File

@ -16,6 +16,8 @@ case "$plugin_type" in
hadoop="2.7.5";;
"2.8.2")
hadoop="2.8.2";;
"3.0.1")
hadoop="3.0.1";;
esac
DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop.jar"
;;

4
tools/gate/build-images
View File

@ -34,8 +34,8 @@ case "$PLUGIN" in
"vanilla")
# temporary don't test building fedora image
for distro in ubuntu centos7; do
# limit to 3 releases
for version in 2.7.1 2.7.5 2.8.2; do
# limit to 4 releases
for version in 2.7.1 2.7.5 2.8.2 3.0.1; do
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version
done
done