openstack
/
sahara-image-elements
Code Issues Proposed changes

Merge "Add support to build 2.7.5 image"

This commit is contained in:
Zuul 2018-05-10 14:41:00 +00:00 committed by Gerrit Code Review
parent 30717f2516 5707f5cc5a
commit aa3ff84f2b
7 changed files with 48 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
diskimage-create/diskimage-create.sh
View File

@ -32,7 +32,7 @@ usage() {
echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
echo " [-i ubuntu|fedora|centos|centos7]"
echo " [-v 2.7.1|2.8.2|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-v 2.7.1|2.7.5|2.8.2|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-r 5.1.0|5.2.0]"
echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
@ -184,7 +184,7 @@ case "$PLUGIN" in
"");;
"vanilla")
case "$HADOOP_VERSION" in
"" | "2.7.1" | "2.8.2");;
"" | "2.7.1" | "2.7.5" | "2.8.2");;
*)
echo -e "Unknown hadoop version selected.\nAborting"
exit 1
@ -480,6 +480,9 @@ set_hive_version() {
"2.7.1" )
export HIVE_VERSION="0.11.0"
;;
"2.7.5" )
export HIVE_VERSION="2.3.2"
;;
"2.8.2" )
export HIVE_VERSION="2.3.2"
;;
@ -498,8 +501,10 @@ set_hive_version() {
if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"}
export HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.5.tar.gz"}
export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"}
export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"}
export OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_5_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.7.5.tar.gz"}
export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"}
export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib"
export plugin_type="vanilla"
@ -547,6 +552,14 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
unset DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"}
export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_5_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_5_latest"}
export DIB_RELEASE=${DIB_RELEASE:-xenial}
set_hive_version
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
unset DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_8_2_image_name:-"ubuntu_sahara_vanilla_hadoop_2_8_2_latest"}
@ -566,6 +579,12 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
set_hive_version
image_create fedora $fedora_image_name $fedora_elements_sequence
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"}
export fedora_image_name=${fedora_vanilla_hadoop_2_7_5_image_name:-"fedora_sahara_vanilla_hadoop_2_7_5_latest$suffix"}
set_hive_version
image_create fedora $fedora_image_name $fedora_elements_sequence
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
export fedora_image_name=${fedora_vanilla_hadoop_2_8_2_image_name:-"fedora_sahara_vanilla_hadoop_2_8_2_latest$suffix"}
@ -592,6 +611,12 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
set_hive_version
image_create centos7 $centos7_image_name $centos7_elements_sequence
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"}
export centos7_image_name=${centos7_vanilla_hadoop_2_7_5_image_name:-"centos7_sahara_vanilla_hadoop_2_7_5_latest$suffix"}
set_hive_version
image_create centos7 $centos7_image_name $centos7_elements_sequence
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
export centos7_image_name=${centos7_vanilla_hadoop_2_8_2_image_name:-"centos7_sahara_vanilla_hadoop_2_8_2_latest$suffix"}

3
elements/hadoop/install.d/40-setup-hadoop
View File

@ -11,6 +11,9 @@ function install_hadoop_v2 {
"2.7.1")
hadoop_native_libs_url="${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL}"
;;
"2.7.5")
hadoop_native_libs_url="${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL}"
;;
"2.8.2")
hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}"
;;

6
elements/oozie/install.d/50-setup-oozie
View File

@ -16,6 +16,12 @@ if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then
else
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL)
fi
elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then
if [ -z "${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" ]; then
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_5_FILE)
else
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL)
fi
elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" ]; then
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_FILE)

5
elements/oozie/root.d/0-check-oozie
View File

@ -11,6 +11,11 @@ if [ "$DIB_HADOOP_VERSION" == "2.7.1" ]; then
echo "OOZIE_HADOOP_V2_7_1_FILE and OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
exit 1
fi
elif [ "$DIB_HADOOP_VERSION" == "2.7.5" ]; then
if [ -z "${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_7_5_FILE:-}" ]; then
echo "OOZIE_HADOOP_V2_7_5_FILE and OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
exit 1
fi
elif [ "$DIB_HADOOP_VERSION" == "2.8.2" ]; then
if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_8_2_FILE:-}" ]; then
echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"

3
elements/oozie/root.d/50-download-oozie
View File

@ -12,6 +12,9 @@ mkdir -p $tmp_dir
if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V2_7_1_FILE:-}"
elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V2_7_5_FILE:-}"
elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}"

2
elements/swift_hadoop/post-install.d/81-add-jar
View File

@ -12,6 +12,8 @@ case "$plugin_type" in
case "$DIB_HADOOP_VERSION" in
"2.7.1")
hadoop="2.7.1";;
"2.7.5")
hadoop="2.7.5";;
"2.8.2")
hadoop="2.8.2";;
esac

4
tools/gate/build-images
View File

@ -34,8 +34,8 @@ case "$PLUGIN" in
"vanilla")
# temporary don't test building fedora image
for distro in ubuntu centos7; do
# limit to 2 releases
for version in 2.7.1 2.8.2; do
# limit to 3 releases
for version in 2.7.1 2.7.5 2.8.2; do
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version
done
done