From 600811ab126c3ee3b2b3200269910417799232b5 Mon Sep 17 00:00:00 2001 From: Telles Nobrega Date: Thu, 28 Dec 2017 18:48:53 +0100 Subject: [PATCH] Image generation for MapR Adds image generation and validation for MapR Change-Id: Ib2d3bf2fa43db96437682d7479df1e897b997674 --- .../plugins/mapr/base/base_version_handler.py | 19 ++ sahara/plugins/mapr/images.py | 43 ++++ sahara/plugins/mapr/plugin.py | 13 + .../resources/images/centos/disable_firewall | 20 ++ .../centos/install_mapr_core_repository | 27 +++ .../images/centos/install_mapr_eco_repository | 27 +++ .../resources/images/centos/install_openjdk | 14 ++ .../resources/images/centos/install_scala | 34 +++ .../mapr/resources/images/centos/requiretty | 6 + .../images/centos/selinux_permissive | 12 + .../resources/images/centos/update_repository | 5 + .../resources/images/common/configure_extjs | 23 ++ .../mapr/resources/images/common/oracle_java | 42 ++++ .../images/common/resources/package_utils.sh | 226 ++++++++++++++++++ .../images/common/resources/packages.json | 140 +++++++++++ .../images/common/resources/spec_5.1.0.json | 46 ++++ .../images/common/resources/spec_5.2.0.json | 50 ++++ .../common/resources/spec_5.2.0_ubuntu.json | 47 ++++ .../images/common/resources/versions.py | 83 +++++++ .../plugins/mapr/resources/images/image.yaml | 195 +++++++++++++++ .../ubuntu/install_mapr_core_repository | 27 +++ .../images/ubuntu/install_mapr_dependencies | 22 ++ .../images/ubuntu/install_mapr_eco_repository | 32 +++ .../resources/images/ubuntu/install_openjdk | 16 ++ .../resources/images/ubuntu/install_scala | 35 +++ .../resources/images/ubuntu/update_repository | 5 + 26 files changed, 1209 insertions(+) create mode 100644 sahara/plugins/mapr/images.py create mode 100644 sahara/plugins/mapr/resources/images/centos/disable_firewall create mode 100644 sahara/plugins/mapr/resources/images/centos/install_mapr_core_repository create mode 100644 sahara/plugins/mapr/resources/images/centos/install_mapr_eco_repository create mode 100644 sahara/plugins/mapr/resources/images/centos/install_openjdk create mode 100644 sahara/plugins/mapr/resources/images/centos/install_scala create mode 100644 sahara/plugins/mapr/resources/images/centos/requiretty create mode 100644 sahara/plugins/mapr/resources/images/centos/selinux_permissive create mode 100644 sahara/plugins/mapr/resources/images/centos/update_repository create mode 100644 sahara/plugins/mapr/resources/images/common/configure_extjs create mode 100644 sahara/plugins/mapr/resources/images/common/oracle_java create mode 100644 sahara/plugins/mapr/resources/images/common/resources/package_utils.sh create mode 100644 sahara/plugins/mapr/resources/images/common/resources/packages.json create mode 100644 sahara/plugins/mapr/resources/images/common/resources/spec_5.1.0.json create mode 100644 sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0.json create mode 100644 sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json create mode 100644 sahara/plugins/mapr/resources/images/common/resources/versions.py create mode 100644 sahara/plugins/mapr/resources/images/image.yaml create mode 100644 sahara/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository create mode 100644 sahara/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies create mode 100644 sahara/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository create mode 100644 sahara/plugins/mapr/resources/images/ubuntu/install_openjdk create mode 100644 sahara/plugins/mapr/resources/images/ubuntu/install_scala create mode 100644 sahara/plugins/mapr/resources/images/ubuntu/update_repository diff --git a/sahara/plugins/mapr/base/base_version_handler.py b/sahara/plugins/mapr/base/base_version_handler.py index 612874f..b070c96 100644 --- a/sahara/plugins/mapr/base/base_version_handler.py +++ b/sahara/plugins/mapr/base/base_version_handler.py @@ -22,6 +22,7 @@ import sahara.plugins.mapr.base.base_cluster_validator as bv import sahara.plugins.mapr.base.base_edp_engine as edp import sahara.plugins.mapr.base.base_health_checker as health import sahara.plugins.mapr.base.base_node_manager as bs +from sahara.plugins.mapr import images import sahara.plugins.mapr.util.general as util import sahara.plugins.provisioning as p import sahara.plugins.utils as u @@ -38,6 +39,7 @@ class BaseVersionHandler(vh.AbstractVersionHandler): self._services = [] self._node_processes = {} self._configs = [] + self.images = images def get_edp_engine(self, cluster, job_type): if job_type in edp.MapROozieJobEngine.get_supported_job_types(): @@ -177,3 +179,20 @@ class BaseVersionHandler(vh.AbstractVersionHandler): def get_cluster_checks(self, cluster): cluster_context = self.get_context(cluster) return self._health_checker.get_checks(cluster_context) + + def get_image_arguments(self): + if hasattr(self, 'images'): + return self.images.get_image_arguments() + else: + return NotImplemented + + def pack_image(self, hadoop_version, remote, test_only=False, + image_arguments=None): + if hasattr(self, 'images'): + self.images.pack_image( + remote, test_only=test_only, image_arguments=image_arguments) + + def validate_images(self, cluster, test_only=False, image_arguments=None): + if hasattr(self, 'images'): + self.images.validate_images( + cluster, test_only=test_only, image_arguments=image_arguments) diff --git a/sahara/plugins/mapr/images.py b/sahara/plugins/mapr/images.py new file mode 100644 index 0000000..aacf9ca --- /dev/null +++ b/sahara/plugins/mapr/images.py @@ -0,0 +1,43 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins import images +from sahara.plugins import utils as plugin_utils + + +_validator = images.SaharaImageValidator.from_yaml( + 'plugins/mapr/resources/images/image.yaml', + resource_roots=['plugins/mapr/resources/images']) + + +def get_image_arguments(): + return _validator.get_argument_list() + + +def pack_image(remote, test_only=False, image_arguments=None): + _validator.validate(remote, test_only=test_only, + image_arguments=image_arguments) + + +def validate_images(cluster, test_only=False, image_arguments=None): + image_arguments = get_image_arguments() + if not test_only: + instances = plugin_utils.get_instances(cluster) + else: + instances = plugin_utils.get_instances(cluster)[0] + for instance in instances: + with instance.remote() as r: + _validator.validate(r, test_only=test_only, + image_arguments=image_arguments) diff --git a/sahara/plugins/mapr/plugin.py b/sahara/plugins/mapr/plugin.py index 059d9dd..3f7f11d 100644 --- a/sahara/plugins/mapr/plugin.py +++ b/sahara/plugins/mapr/plugin.py @@ -97,3 +97,16 @@ class MapRPlugin(p.ProvisioningPluginBase): def get_health_checks(self, cluster): v_handler = self._get_handler(cluster.hadoop_version) return v_handler.get_cluster_checks(cluster) + + def get_image_arguments(self, hadoop_version): + return self._get_handler(hadoop_version).get_image_arguments() + + def pack_image(self, hadoop_version, remote, + test_only=False, image_arguments=None): + version = self._get_handler(hadoop_version) + version.pack_image(hadoop_version, remote, test_only=test_only, + image_arguments=image_arguments) + + def validate_images(self, cluster, test_only=False, image_arguments=None): + self._get_handler(cluster.hadoop_version).validate_images( + cluster, test_only=test_only, image_arguments=image_arguments) diff --git a/sahara/plugins/mapr/resources/images/centos/disable_firewall b/sahara/plugins/mapr/resources/images/centos/disable_firewall new file mode 100644 index 0000000..e82c456 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/centos/disable_firewall @@ -0,0 +1,20 @@ +#!/bin/bash + +check=$(systemctl --no-pager list-unit-files iptables.service | grep 'enabled' | wc -l) + +if [ $check -eq 1 ]; then + if [ $test_only -eq 0 ]; then + if type -p systemctl && [[ "$(systemctl --no-pager list-unit-files firewalld)" =~ 'enabled' ]]; then + systemctl disable firewalld + fi + + if type -p service; then + service ip6tables save + service iptables save + chkconfig ip6tables off + chkconfig iptables off + fi + else + exit 0 + fi +fi diff --git a/sahara/plugins/mapr/resources/images/centos/install_mapr_core_repository b/sahara/plugins/mapr/resources/images/centos/install_mapr_core_repository new file mode 100644 index 0000000..4c0e5f3 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/centos/install_mapr_core_repository @@ -0,0 +1,27 @@ +#!/bin/bash + +DISTRO_NAME=$distro +source "/tmp/package_utils.sh" + +echo "START: installing MapR core repository" + +MAPR_REPO_URL="http://package.mapr.com/releases/v${plugin_version}/redhat/mapr-v${plugin_version}GA.rpm.tgz" +MAPR_REPO_DIR="/opt/mapr-repository/core" + +if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then + if [ $test_only -eq 0 ]; then + + MAPR_REPO_NAME="mapr_core" + + echo "Downloading MapR repository archive" + mkdir -p "$MAPR_REPO_DIR" && curl "$MAPR_REPO_URL" | tar -xz -C "$MAPR_REPO_DIR" + + echo "Creating local repository" + create_repo "$MAPR_REPO_DIR" + + echo "Adding MapR repository" + add_local_repo "$MAPR_REPO_NAME" "$MAPR_REPO_DIR" + fi +fi + +echo "END: installing MapR core repository" diff --git a/sahara/plugins/mapr/resources/images/centos/install_mapr_eco_repository b/sahara/plugins/mapr/resources/images/centos/install_mapr_eco_repository new file mode 100644 index 0000000..c66de71 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/centos/install_mapr_eco_repository @@ -0,0 +1,27 @@ +#!/bin/bash + + +VERSIONS_PY="/tmp/versions.py" +DISTRO_NAME=$distro + +source "/tmp/package_utils.sh" + +echo "START: installing MapR ecosystem repository" + +MAPR_REPO_URL="http://package.mapr.com/releases/MEP/MEP-2.0.0/redhat/" +MAPR_REPO_DIR="/opt/mapr-repository/ecosystem" + +if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then + if [ $test_only -eq 0 ]; then + + MAPR_REPO_NAME="mapr_ecosystem" + MAPR_PKG_GROUPS="/tmp/packages.json" + + MAPR_SPEC="/tmp/spec_$plugin_version.json" + + echo "Creating local MapR ecosystem repository" + localize_repo "$MAPR_REPO_NAME" "$MAPR_REPO_URL" "$MAPR_PKG_GROUPS" "$MAPR_SPEC" "$MAPR_REPO_DIR" + echo $MAPR_SPEC + fi +fi +echo "END: installing MapR ecosystem repository" diff --git a/sahara/plugins/mapr/resources/images/centos/install_openjdk b/sahara/plugins/mapr/resources/images/centos/install_openjdk new file mode 100644 index 0000000..6e483e6 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/centos/install_openjdk @@ -0,0 +1,14 @@ +#!/bin/bash + +echo "Installing OpenJDK" + +if [ $test_only -eq 0 ]; then + yum install -y java-1.8.0-openjdk-devel + + JRE_HOME="/usr/lib/jvm/java-openjdk/jre" + JDK_HOME="/usr/lib/jvm/java-openjdk" + + echo "OpenJDK has been installed" +else + exit 0 +fi diff --git a/sahara/plugins/mapr/resources/images/centos/install_scala b/sahara/plugins/mapr/resources/images/centos/install_scala new file mode 100644 index 0000000..1546a0b --- /dev/null +++ b/sahara/plugins/mapr/resources/images/centos/install_scala @@ -0,0 +1,34 @@ +#!/bin/bash + +echo "START: installing Scala" +sudo yum -y update +exit 0 + +if [ $test_only -eq 0 ]; then + + RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://www.scala-lang.org/)" + + if [ "$RETURN_CODE" != "200" ]; then + echo "http://www.scala-lang.org is unreachable" && exit 1 + fi + + if [ -n "${scala_version:-}" ]; then + VERSION=$scala_version + else + VERSION="$(curl -s --fail http://www.scala-lang.org| tr -d '\n' | sed 's/^.*[^0-9]\+\([0-9\.\?]\+\)<.\+$/\1/')" + + if [ $? != 0 -o -z "${VERSION}" ]; then + echo "Installing default version $scala_version" + VERSION=$scala_version + fi + fi + + PKG=scala-${VERSION} + + URL="http://downloads.lightbend.com/scala/${VERSION}" + + rpm -Uhv ${URL}/${PKG}.rpm +fi + +echo "END: installing Scala" + diff --git a/sahara/plugins/mapr/resources/images/centos/requiretty b/sahara/plugins/mapr/resources/images/centos/requiretty new file mode 100644 index 0000000..f7007e7 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/centos/requiretty @@ -0,0 +1,6 @@ +#!/bin/bash + + +if [ $test_only -eq 0 ]; then + sed '/^Defaults requiretty*/ s/^/#/' -i /etc/sudoers +fi diff --git a/sahara/plugins/mapr/resources/images/centos/selinux_permissive b/sahara/plugins/mapr/resources/images/centos/selinux_permissive new file mode 100644 index 0000000..b7aaff6 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/centos/selinux_permissive @@ -0,0 +1,12 @@ +#!/bin/bash + +check=$(cat /etc/selinux/config | grep "SELINUX=permissive" | wc -l) + +if [ $check -eq 0 ]; then + if [ $test_only -eq 0 ]; then + echo "SELINUX=permissive" > /etc/selinux/config + echo "SELINUXTYPE=targeted" >> /etc/selinux/config + else + exit 0 + fi +fi diff --git a/sahara/plugins/mapr/resources/images/centos/update_repository b/sahara/plugins/mapr/resources/images/centos/update_repository new file mode 100644 index 0000000..632de3e --- /dev/null +++ b/sahara/plugins/mapr/resources/images/centos/update_repository @@ -0,0 +1,5 @@ +#!/bin/bash + +if [ $test_only -eq 0 ]; then + yum clean all && yum repolist +fi diff --git a/sahara/plugins/mapr/resources/images/common/configure_extjs b/sahara/plugins/mapr/resources/images/common/configure_extjs new file mode 100644 index 0000000..2fcb93d --- /dev/null +++ b/sahara/plugins/mapr/resources/images/common/configure_extjs @@ -0,0 +1,23 @@ +#!/bin/bash + +EXTJS_DESTINATION_DIR="/opt/mapr-repository" +EXTJS_DOWNLOAD_URL="http://tarballs.openstack.org/sahara/dist/common-artifacts/ext-2.2.zip" +EXTJS_NO_UNPACK=1 + +extjs_archive=/tmp/$(basename $EXTJS_DOWNLOAD_URL) + +if [ ! -f "${EXTJS_DESTINATION_DIR}/${extjs_archive}" ]; then + if [ $test_only -eq 0 ]; then + wget -O $extjs_archive $EXTJS_DOWNLOAD_URL + + mkdir -p $EXTJS_DESTINATION_DIR + if [ -z "${EXTJS_NO_UNPACK:-}" ]; then + unzip -d "$EXTJS_DESTINATION_DIR" $extjs_archive + rm -f $extjs_archive + else + mv $extjs_archive $EXTJS_DESTINATION_DIR + fi + else + exit 0 + fi +fi diff --git a/sahara/plugins/mapr/resources/images/common/oracle_java b/sahara/plugins/mapr/resources/images/common/oracle_java new file mode 100644 index 0000000..b293ecf --- /dev/null +++ b/sahara/plugins/mapr/resources/images/common/oracle_java @@ -0,0 +1,42 @@ +#!/bin/bash + + +# NOTE: $(dirname $0) is read-only, use space under $TARGET_ROOT +JAVA_LOCATION=${JAVA_TARGET_LOCATION:-"/usr/java"} +JAVA_NAME="oracle-jdk" +JAVA_HOME=$JAVA_LOCATION/$JAVA_NAME +JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"} + +# FIXME: probably not idempotent, find a better condition +if [ ! -d $JAVA_LOCATION ]; then + if [ $test_only -eq 0 ]; then + echo "Begin: installation of Java" + mkdir -p $JAVA_LOCATION + + if [ -n "$JAVA_DOWNLOAD_URL" ]; then + JAVA_FILE=$(basename $JAVA_DOWNLOAD_URL) + wget --no-check-certificate --no-cookies -c \ + --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \ + -O $JAVA_LOCATION/$JAVA_FILE $JAVA_DOWNLOAD_URL + elif [ -n "$JAVA_FILE" ]; then + install -D -g root -o root -m 0755 $(dirname $0)/$JAVA_FILE $JAVA_LOCATION + fi + + cd $JAVA_LOCATION + + echo "Decompressing Java archive" + echo -e "\n" | tar -zxf $JAVA_FILE + echo "Setting up $JAVA_NAME" + chown -R root:root $JAVA_LOCATION + JAVA_DIR=`ls -1 $JAVA_LOCATION | grep -v tar.gz` + ln -s $JAVA_LOCATION/$JAVA_DIR $JAVA_HOME + + setup-java-home $JAVA_HOME $JAVA_HOME + + rm $JAVA_FILE + + echo "End: installation of Java" + else + exit 0 + fi +fi diff --git a/sahara/plugins/mapr/resources/images/common/resources/package_utils.sh b/sahara/plugins/mapr/resources/images/common/resources/package_utils.sh new file mode 100644 index 0000000..56e52cf --- /dev/null +++ b/sahara/plugins/mapr/resources/images/common/resources/package_utils.sh @@ -0,0 +1,226 @@ +# execute_in_directory +execute_in_directory() { + local directory="$(readlink -f "$1")"; shift + local cmd="$*" + + pushd "$directory" && eval "$cmd" && popd +} + +# get_distro +get_distro() { + echo "$DISTRO_NAME" +} + +# download_apt_package [version] [directory] +download_apt_package() { + local package="$1" + local version="${2:-}" + local directory="${3:-$(pwd)}" + local package_spec="$package${version:+=$version*}" + + execute_in_directory "$directory" apt-get --allow-unauthenticated download "$package_spec" +} + +# download_yum_package [version] [directory] +download_yum_package() { + local package="$1" + local version="${2:-}" + local directory="${3:-$(pwd)}" + local package_spec="$package${version:+-$version*}" + + yumdownloader --destdir "$directory" "$package_spec" +} + +# download_package [version] [directory] [distro] +download_package() { + local package="$1" + local version="${2:-}" + local directory="${3:-$(pwd)}" + local distro="${4:-$(get_distro)}" + + if [[ "$distro" == "ubuntu" ]]; then + download_apt_package "$package" "$version" "$directory" + elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then + download_yum_package "$package" "$version" "$directory" + fi +} + +# get_packages [version_separator] +get_packages() { + local package_groups_file="$1" + local spec_file="$2" + local version_separator="${3:-:}" + + python "$VERSIONS_PY" --separator "$version_separator" "$package_groups_file" "$spec_file" +} + +# download_packages [directory] [distro] +download_packages() { + local package_groups_file="$1" + local spec_file="$2" + local directory="${3:-$(pwd)}" + local distro="${4:-$(get_distro)}" + local version_separator=":" + + local packages="$(get_packages "$package_groups_file" "$spec_file" "$version_separator")" + for package in $packages; do + IFS="$version_separator" read -ra package_version <<< "$package" + download_package "${package_version[@]}" "$directory" "$distro" + done +} + +# create_apt_repo +create_apt_repo() { + local directory="$(readlink -f "$1")" + local binary_dir="$directory/binary" + local packages_gz="$binary_dir/Packages.gz" + + mkdir -p "$binary_dir" + execute_in_directory "$directory" "dpkg-scanpackages -m . /dev/null | gzip -9c > $packages_gz" +} + +# create_yum_repo +create_yum_repo() { + local directory="$(readlink -f "$1")" + + createrepo "$directory" +} + +# create_repo [distro] +create_repo() { + local directory="$(readlink -f "$1")" + local distro="${2:-$(get_distro)}" + + if [[ "$distro" == "ubuntu" ]]; then + create_apt_repo "$directory" + elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then + create_yum_repo "$directory" + fi +} + +# add_apt_repo +add_apt_repo() { + local repo_name="$1" + local repo_url="$2" + local repo="deb $repo_url" + local repo_path="/etc/apt/sources.list.d/$repo_name.list" + + echo "$repo" > "$repo_path" && apt-get update +} + +# add_yum_repo +add_yum_repo() { + local repo_name="$1" + local repo_url="$2" + local repo_path="/etc/yum.repos.d/$repo_name.repo" + + cat > "$repo_path" << EOF +[$repo_name] +name=$repo_name +baseurl=$repo_url +enabled=1 +gpgcheck=0 +protect=1 +EOF + yum clean all && rm -rf /var/cache/yum/* && yum check-update +} + +# add_repo [distro] +add_repo() { + local repo_name="$1" + local repo_url="$2" + local distro="${3:-$(get_distro)}" + + if [[ "$distro" == "ubuntu" ]]; then + add_apt_repo "$repo_name" "$repo_url" + elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then + add_yum_repo "$repo_name" "$repo_url" + fi +} + +# add_local_apt_repo +add_local_apt_repo() { + local repo_name="$1" + local directory="$(readlink -f "$2")" + local repo_url="file:$directory binary/" + + add_apt_repo "$repo_name" "$repo_url" +} + +# add_local_yum_repo +add_local_yum_repo() { + local repo_name="$1" + local directory="$(readlink -f "$2")" + local repo_url="file://$directory" + + add_yum_repo "$repo_name" "$repo_url" +} + +# add_local_repo [distro] +add_local_repo() { + local repo_name="$1" + local directory="$(readlink -f "$2")" + local distro="${3:-$(get_distro)}" + + if [[ "$distro" == "ubuntu" ]]; then + add_local_apt_repo "$repo_name" "$directory" + elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then + add_local_yum_repo "$repo_name" "$directory" + fi +} + +# remove_apt_repo +remove_apt_repo() { + local repo_name="$1" + local repo_path="/etc/apt/sources.list.d/$repo_name.list" + + rm "$repo_path" && apt-get update +} + +# remove_yum_repo +remove_yum_repo() { + local repo_name="$1" + local repo_path="/etc/yum.repos.d/$repo_name.repo" + + rm "$repo_path" +} + +# remove_repo [distro] +remove_repo() { + local repo_name="$1" + local distro="${2:-$(get_distro)}" + + if [[ "$distro" == "ubuntu" ]]; then + remove_apt_repo "$repo_name" + elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then + remove_yum_repo "$repo_name" + fi +} + +# create_local_repo +create_local_repo() { + local repo_name="$1" + local repo_url="$2" + local package_groups_file="$3" + local spec_file="$4" + local directory="$5" + + add_repo "$repo_name" "$repo_url" + mkdir -p "$directory" && directory="$(readlink -f "$directory")" + download_packages "$package_groups_file" "$spec_file" "$directory" + remove_repo "$repo_name" + create_repo "$directory" +} + +# localize_repo +localize_repo() { + local repo_name="$1" + local repo_url="$2" + local package_groups_file="$3" + local spec_file="$4" + local directory="$5" + + mkdir -p "$directory" && directory="$(readlink -f "$directory")" + create_local_repo "$repo_name" "$repo_url" "$package_groups_file" "$spec_file" "$directory" + add_local_repo "$repo_name" "$directory" +} diff --git a/sahara/plugins/mapr/resources/images/common/resources/packages.json b/sahara/plugins/mapr/resources/images/common/resources/packages.json new file mode 100644 index 0000000..b1cb5d0 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/common/resources/packages.json @@ -0,0 +1,140 @@ +{ + "asynchbase": { + "all": [ + "mapr-asynchbase" + ] + }, + "drill": { + "all": [ + "mapr-drill" + ] + }, + "flume": { + "all": [ + "mapr-flume" + ] + }, + "hbase": { + "all": [ + "mapr-hbase", + "mapr-hbase-internal", + "mapr-hbase-master", + "mapr-hbase-regionserver", + "mapr-hbasethrift", + "mapr-hbase-rest" + ], + "0.98.12": [ + "mapr-hbase", + "mapr-hbase-internal", + "mapr-hbase-master", + "mapr-hbase-regionserver", + "mapr-hbasethrift", + "mapr-libhbase", + "mapr-hbase-rest" + ], + "1.1.1": [ + "mapr-hbase", + "mapr-hbase-internal", + "mapr-hbase-master", + "mapr-hbase-regionserver", + "mapr-hbasethrift", + "mapr-libhbase", + "mapr-hbase-rest" + ] + }, + "hive": { + "all": [ + "mapr-hive", + "mapr-hivemetastore", + "mapr-hiveserver2" + ] + }, + "httpfs": { + "all": [ + "mapr-httpfs" + ] + }, + "hue": { + "all": [ + "mapr-hue", + "mapr-hue-base", + "mapr-hue-livy" + ], + "3.10.0": [ + "mapr-hue", + "mapr-hue-livy" + ] + }, + "impala": { + "all": [ + "mapr-impala", + "mapr-impala-catalog", + "mapr-impala-server", + "mapr-impala-statestore", + "mapr-impala-udf" + ] + }, + "mahout": { + "all": [ + "mapr-mahout" + ] + }, + "oozie": { + "all": [ + "mapr-oozie", + "mapr-oozie-internal" + ] + }, + "pig": { + "all": [ + "mapr-pig" + ] + }, + "sentry": { + "all": [ + "mapr-sentry" + ] + }, + "spark": { + "all": [ + "mapr-spark", + "mapr-spark-historyserver", + "mapr-spark-master" + ] + }, + "sqoop": { + "all": [ + "mapr-sqoop2-client", + "mapr-sqoop2-server" + ] + }, + "storm": { + "all": [ + "mapr-storm", + "mapr-storm-ui", + "mapr-storm-nimbus", + "mapr-storm-supervisor" + ] + }, + "tez": { + "all": [ + "mapr-tez" + ] + }, + "kafka": { + "all": [ + "mapr-kafka" + ] + }, + "kafka-connect": { + "all": [ + "mapr-kafka-connect-hdfs", + "mapr-kafka-connect-jdbc" + ] + }, + "kafka-rest": { + "all": [ + "mapr-kafka-rest" + ] + } +} diff --git a/sahara/plugins/mapr/resources/images/common/resources/spec_5.1.0.json b/sahara/plugins/mapr/resources/images/common/resources/spec_5.1.0.json new file mode 100644 index 0000000..11ce5be --- /dev/null +++ b/sahara/plugins/mapr/resources/images/common/resources/spec_5.1.0.json @@ -0,0 +1,46 @@ +{ + "drill": [ + "1.1.0", + "1.2.0", + "1.4.0" + ], + "flume": [ + "1.5.0", + "1.6.0" + ], + "hbase": [ + "0.98.9", + "0.98.12" + ], + "hive": [ + "0.13", + "1.0", + "1.2" + ], + "httpfs": [ + "1.0" + ], + "hue": [ + "3.8.1", + "3.9.0" + ], + "impala": [ + "1.4.1" + ], + "mahout": [ + "0.10.0" + ], + "oozie": [ + "4.2.0" + ], + "pig": [ + "0.14", + "0.15" + ], + "sqoop": [ + "2.0.0" + ], + "spark": [ + "1.5.2" + ] +} diff --git a/sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0.json b/sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0.json new file mode 100644 index 0000000..5519cd2 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0.json @@ -0,0 +1,50 @@ +{ + "drill": [ + "1.9.0" + ], + "flume": [ + "1.6.0" + ], + "hbase": [ + "1.1.1" + ], + "hive": [ + "1.2" + ], + "httpfs": [ + "1.0" + ], + "hue": [ + "3.10.0" + ], + "impala": [ + "2.5.0" + ], + "mahout": [ + "0.12.0" + ], + "oozie": [ + "4.2.0" + ], + "pig": [ + "0.16" + ], + "sqoop": [ + "2.0.0" + ], + "spark": [ + "2.0.1" + ], + "sentry": [ + "1.6.0" + ], + "kafka": [ + "0.9.0" + ], + "kafka-connect": [ + "2.0.1" + ], + "kafka-rest": [ + "2.0.1" + ] +} diff --git a/sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json b/sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json new file mode 100644 index 0000000..4327051 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json @@ -0,0 +1,47 @@ +{ + "drill": [ + "1.9.0" + ], + "flume": [ + "1.6.0" + ], + "hbase": [ + "1.1.1" + ], + "hive": [ + "1.2" + ], + "httpfs": [ + "1.0" + ], + "hue": [ + "3.10.0" + ], + "mahout": [ + "0.12.0" + ], + "oozie": [ + "4.2.0" + ], + "pig": [ + "0.16" + ], + "sqoop": [ + "2.0.0" + ], + "spark": [ + "2.0.1" + ], + "sentry": [ + "1.6.0" + ], + "kafka": [ + "0.9.0" + ], + "kafka-connect": [ + "2.0.1" + ], + "kafka-rest": [ + "2.0.1" + ] +} diff --git a/sahara/plugins/mapr/resources/images/common/resources/versions.py b/sahara/plugins/mapr/resources/images/common/resources/versions.py new file mode 100644 index 0000000..45dea1d --- /dev/null +++ b/sahara/plugins/mapr/resources/images/common/resources/versions.py @@ -0,0 +1,83 @@ +# Copyright (c) 2015, MapR Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import argparse +import sys + +from oslo_serialization import jsonutils as json + +_GROUP_VERSION_SEPARATOR = "," +_ALL_GROUP_VERSION = "all" + + +def _build_parser(): + parser = argparse.ArgumentParser() + + parser.add_argument("packages", help="path to the packages.json") + parser.add_argument("spec", help="path to the spec.json") + parser.add_argument("--separator", default=":", + help="separator between package name" + " and version in output") + + return parser + + +def _load_json(path): + with open(path) as json_file: + return json.load(json_file) + + +def _version_matches(version, group_version): + for gv in group_version.split(_GROUP_VERSION_SEPARATOR): + if version.startswith(gv): + return True + + return False + + +def _get_packages(version, group_spec): + for group_version in group_spec: + if _version_matches(version, group_version): + return group_spec[group_version] + + return group_spec[_ALL_GROUP_VERSION] + + +def _get_package_versions(spec, package_groups): + return [(package, version) + for pg_name, versions in spec.items() + for version in versions + for package in _get_packages(version, package_groups[pg_name])] + + +parser = _build_parser() + + +def main(args=None): + args = parser.parse_args(args or sys.argv[1:]) + + spec = _load_json(args.spec) + package_groups = _load_json(args.packages) + separator = args.separator + + package_versions = _get_package_versions(spec, package_groups) + package_format = "%s" + separator + "%s\n" + package_versions = map(lambda pv: package_format % pv, package_versions) + + sys.stdout.writelines(package_versions) + + +if __name__ == "__main__": + main() diff --git a/sahara/plugins/mapr/resources/images/image.yaml b/sahara/plugins/mapr/resources/images/image.yaml new file mode 100644 index 0000000..5b3bf39 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/image.yaml @@ -0,0 +1,195 @@ +arguments: + java_distro: + default: openjdk + description: The distribution of Java to install. Defaults to cloudera-jdk. + choices: + - openjdk + - oracle-java + plugin_version: + default: 5.2.0 + description: The distribution of MapR to install. Defaults to 5.2.0. + hidden: True + required: False + scala_version: + default: 2.11.6 + description: The version of scala to install. Defaults to 2.11.6. + hidden: True + required: False + hdfs_lib_dir: + default: /usr/lib/hadoop-mapreduce + description: The path to HDFS_LIB_DIR. Default to /usr/lib/hadoop-mapreduce + required: False + +validators: + - os_case: + - ubuntu: + - script: ubuntu/install_mapr_dependencies + - package: + - mtools + - rpcbind + - sdparm + - syslinux + - unzip + - wget + - zip + - os_case: + - centos: + - package: + - cups + - cdparanoia-libs + - cups-libs + - createrepo + - cvs + - cyrus-sasl-gssapi + - cyrus-sasl-plain + - foomatic + - foomatic-db + - foomatic-db-filesystem + - foomatic-db-ppds + - gdbm-devel + - gettext + - ghostscript + - ghostscript-fonts + - glibc + - glibc-common + - glibc-devel + - glibc-headers + - gstreamer + - gstreamer-plugins-base + - gstreamer-tools + - hdparm + - irqbalance + - iso-codes + - kernel-headers + - libXt + - libXv + - libXxf86vm + - libgomp + - libgudev1 + - libicu + - libmng + - liboil + - libtheora + - libtirpc + - libvisual + - libxslt + - mesa-dri-drivers + - mesa-libGL + - mesa-libGLU + - mesa-private-llvm + - nmap-ncat + - numactl + - openjpeg-libs + - patch + - pax + - perl-CGI + - perl-ExtUtils-MakeMaker + - perl-ExtUtils-ParseXS + - perl-Test-Harness + - perl-Test-Simple + - perl-devel + - phonon-backend-gstreamer + - poppler + - poppler-data + - poppler-utils + - portreserve + - qt + - qt-x11 + - qt3 + - redhat-lsb + - redhat-lsb-core + - redhat-lsb-printing + - urw-fonts + - yum-utils + - xml-common + - ubuntu: + - package: + - binutils + - daemon + - dpkg-dev + - dpkg-repack + - gcc + - gcc-4.8 + - gcc-doc + - gcc-multilib + - iputils-arping + - libasan0 + - libatomic1 + - libc-dev-bin + - libc6 + - libc6-dev + - libcrypt-passwdmd5-perl + - libgcc-4.8-dev + - libgomp1 + - libgssglue1 + - libicu48 + - libitm1 + - libmysqlclient-dev + - libmysqlclient16 + - libmysqlclient18 + - libnfsidmap2 + - libquadmath0 + - libsasl2-dev + - libsasl2-modules-gssapi-mit + - libssl0.9.8 + - libtirpc1 + - libtsan0 + - libxslt1.1 + - linux-libc-dev + - manpages-dev + - mysql-common + - nfs-common + - open-iscsi + - openjdk-7-jre + - syslinux-common + - zlib1g-dev + - script: common/configure_extjs + - os_case: + - centos: + - copy_script: common/resources/package_utils.sh + - copy_script: common/resources/packages.json + - copy_script: common/resources/spec_5.1.0.json + - copy_script: common/resources/spec_5.2.0.json + - copy_script: common/resources/versions.py + - script: + centos/install_scala: + env_vars: [scala_version] + - script: + centos/install_mapr_core_repository: + env_vars: [plugin_version] + - script: + centos/install_mapr_eco_repository: + env_vars: [plugin_version] + - script: centos/selinux_permissive + - argument_case: + argument_name: java_distro + cases: + openjdk: + - script: centos/install_openjdk + oracle-java: + - script: common/oracle_java + - ubuntu: + - copy_script: common/resources/package_utils.sh + - copy_script: common/resources/packages.json + - copy_script: common/resources/spec_5.1.0.json + - copy_script: common/resources/spec_5.2.0.json + - copy_script: common/resources/spec_5.2.0_ubuntu.json + - copy_script: common/resources/versions.py + - script: + ubuntu/install_scala: + env_vars: [scala_version] + - script: + ubuntu/install_mapr_core_repository: + env_vars: [plugin_version] + - script: + ubuntu/install_mapr_eco_repository: + env_vars: [plugin_version] + - os_case: + - ubuntu: + - argument_case: + argument_name: java_distro + cases: + openjdk: + - script: ubuntu/install_openjdk + oracle-java: + - script: common/oracle_java diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository b/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository new file mode 100644 index 0000000..d31a2b3 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository @@ -0,0 +1,27 @@ +#!/bin/bash + +DISTRO_NAME=$distro +source "/tmp/package_utils.sh" + +echo "START: installing MapR core repository" + +MAPR_REPO_URL="http://package.mapr.com/releases/v${plugin_version}/ubuntu/mapr-v${plugin_version}GA.deb.tgz" +MAPR_REPO_DIR="/opt/mapr-repository/core" + +if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then + if [ $test_only -eq 0 ]; then + + MAPR_REPO_NAME="mapr_core" + + echo "Downloading MapR repository archive" + mkdir -p "$MAPR_REPO_DIR" && curl "$MAPR_REPO_URL" | tar -xz -C "$MAPR_REPO_DIR" + + echo "Creating local repository" + create_repo "$MAPR_REPO_DIR" + + echo "Adding MapR repository" + add_local_repo "$MAPR_REPO_NAME" "$MAPR_REPO_DIR" + fi +fi + +echo "END: installing MapR core repository" diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies b/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies new file mode 100644 index 0000000..6d29cc0 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies @@ -0,0 +1,22 @@ +#!/bin/bash + +echo "START: installing MapR core dependencies" + +if [ ! -f /etc/apt/sources.list.d/security_repo.list ]; then + if [ $test_only -eq 0 ]; then + # Required for libicu48 + cat >> /etc/apt/sources.list.d/security_repo.list << EOF +deb http://security.ubuntu.com/ubuntu precise-security main +EOF + # Required for libmysqlclient16 + cat >> /etc/apt/sources.list.d/security_repo.list << EOF +deb http://old-releases.ubuntu.com/ubuntu lucid-security main +EOF + else + exit 0 + fi +fi + +apt-get update + +echo "END: installing MapR core dependencies" diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository b/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository new file mode 100644 index 0000000..949396c --- /dev/null +++ b/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository @@ -0,0 +1,32 @@ +#!/bin/bash + + +VERSIONS_PY="/tmp/versions.py" +DISTRO_NAME=$distro + +source "/tmp/package_utils.sh" + +echo "START: installing MapR ecosystem repository" + +MAPR_REPO_URL="http://package.mapr.com/releases/MEP/MEP-2.0.0/ubuntu/ binary trusty" +MAPR_REPO_DIR="/opt/mapr-repository/ecosystem" + +if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then + if [ $test_only -eq 0 ]; then + + MAPR_REPO_NAME="mapr_ecosystem" + MAPR_PKG_GROUPS="/tmp/packages.json" + + if [ -f /tmp/spec_$plugin_version_ubuntu.json ]; then + MAPR_SPEC="/tmp/spec_$plugin_version_ubuntu.json" + else + MAPR_SPEC="/tmp/spec_$plugin_version.json" + fi + + echo "Creating local MapR ecosystem repository" + localize_repo "$MAPR_REPO_NAME" "$MAPR_REPO_URL" "$MAPR_PKG_GROUPS" "$MAPR_SPEC" "$MAPR_REPO_DIR" + echo $MAPR_SPEC + fi +fi + +echo "END: installing MapR ecosystem repository" diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_openjdk b/sahara/plugins/mapr/resources/images/ubuntu/install_openjdk new file mode 100644 index 0000000..750c2e8 --- /dev/null +++ b/sahara/plugins/mapr/resources/images/ubuntu/install_openjdk @@ -0,0 +1,16 @@ +#!/bin/bash + +echo "Installing OpenJDK" + +if [ $test_only -eq 0 ]; then + apt-get install -y openjdk-7-jdk + + JRE_PATH=$(update-alternatives --list java) + JRE_HOME=${JRE_PATH%/bin/java} + JDK_PATH=$(update-alternatives --list javac) + JDK_HOME=${JDK_PATH%/bin/javac} + + echo "OpenJDK has been installed" +else + exit 0 +fi diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_scala b/sahara/plugins/mapr/resources/images/ubuntu/install_scala new file mode 100644 index 0000000..1bd694e --- /dev/null +++ b/sahara/plugins/mapr/resources/images/ubuntu/install_scala @@ -0,0 +1,35 @@ +#!/bin/bash + +echo "START: installing Scala" + + +if [ $test_only -eq 0 ]; then + + RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://www.scala-lang.org/)" + + if [ "$RETURN_CODE" != "200" ]; then + echo "http://www.scala-lang.org is unreachable" && exit 1 + fi + + if [ -n "${scala_version:-}" ]; then + VERSION=$scala_version + else + VERSION="$(curl -s --fail http://www.scala-lang.org| tr -d '\n' | sed 's/^.*[^0-9]\+\([0-9\.\?]\+\)<.\+$/\1/')" + + if [ $? != 0 -o -z "${VERSION}" ]; then + echo "Installing default version $scala_version" + VERSION=$scala_version + fi + fi + + PKG=scala-${VERSION} + + URL="http://downloads.lightbend.com/scala/${VERSION}" + + wget -N ${URL}/${PKG}.deb + dpkg -i ${PKG}.deb + rm ${PKG}.deb +fi + +echo "END: installing Scala" + diff --git a/sahara/plugins/mapr/resources/images/ubuntu/update_repository b/sahara/plugins/mapr/resources/images/ubuntu/update_repository new file mode 100644 index 0000000..0dd299b --- /dev/null +++ b/sahara/plugins/mapr/resources/images/ubuntu/update_repository @@ -0,0 +1,5 @@ +#!/bin/bash + +if [ $test_only -eq 0 ]; then + apt-get update +fi