Merge "Image generation for MapR"

sahara/plugins/mapr/base/base_version_handler.py

@@ -22,6 +22,7 @@ import sahara.plugins.mapr.base.base_cluster_validator as bv
 import sahara.plugins.mapr.base.base_edp_engine as edp
 import sahara.plugins.mapr.base.base_health_checker as health
 import sahara.plugins.mapr.base.base_node_manager as bs
+from sahara.plugins.mapr import images
 import sahara.plugins.mapr.util.general as util
 import sahara.plugins.provisioning as p
 import sahara.plugins.utils as u
@@ -38,6 +39,7 @@ class BaseVersionHandler(vh.AbstractVersionHandler):
         self._services = []
         self._node_processes = {}
         self._configs = []
+        self.images = images
 
     def get_edp_engine(self, cluster, job_type):
         if job_type in edp.MapROozieJobEngine.get_supported_job_types():
@@ -177,3 +179,20 @@ class BaseVersionHandler(vh.AbstractVersionHandler):
     def get_cluster_checks(self, cluster):
         cluster_context = self.get_context(cluster)
         return self._health_checker.get_checks(cluster_context)
+
+    def get_image_arguments(self):
+        if hasattr(self, 'images'):
+            return self.images.get_image_arguments()
+        else:
+            return NotImplemented
+
+    def pack_image(self, hadoop_version, remote, test_only=False,
+                   image_arguments=None):
+        if hasattr(self, 'images'):
+            self.images.pack_image(
+                remote, test_only=test_only, image_arguments=image_arguments)
+
+    def validate_images(self, cluster, test_only=False, image_arguments=None):
+        if hasattr(self, 'images'):
+            self.images.validate_images(
+                cluster, test_only=test_only, image_arguments=image_arguments)

sahara/plugins/mapr/images.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sahara.plugins import images
+from sahara.plugins import utils as plugin_utils
+
+
+_validator = images.SaharaImageValidator.from_yaml(
+    'plugins/mapr/resources/images/image.yaml',
+    resource_roots=['plugins/mapr/resources/images'])
+
+
+def get_image_arguments():
+    return _validator.get_argument_list()
+
+
+def pack_image(remote, test_only=False, image_arguments=None):
+    _validator.validate(remote, test_only=test_only,
+                        image_arguments=image_arguments)
+
+
+def validate_images(cluster, test_only=False, image_arguments=None):
+    image_arguments = get_image_arguments()
+    if not test_only:
+        instances = plugin_utils.get_instances(cluster)
+    else:
+        instances = plugin_utils.get_instances(cluster)[0]
+    for instance in instances:
+        with instance.remote() as r:
+            _validator.validate(r, test_only=test_only,
+                                image_arguments=image_arguments)

sahara/plugins/mapr/plugin.py

@@ -97,3 +97,16 @@ class MapRPlugin(p.ProvisioningPluginBase):
     def get_health_checks(self, cluster):
         v_handler = self._get_handler(cluster.hadoop_version)
         return v_handler.get_cluster_checks(cluster)
+
+    def get_image_arguments(self, hadoop_version):
+        return self._get_handler(hadoop_version).get_image_arguments()
+
+    def pack_image(self, hadoop_version, remote,
+                   test_only=False, image_arguments=None):
+        version = self._get_handler(hadoop_version)
+        version.pack_image(hadoop_version, remote, test_only=test_only,
+                           image_arguments=image_arguments)
+
+    def validate_images(self, cluster, test_only=False, image_arguments=None):
+        self._get_handler(cluster.hadoop_version).validate_images(
+            cluster, test_only=test_only, image_arguments=image_arguments)

sahara/plugins/mapr/resources/images/centos/disable_firewall

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+check=$(systemctl --no-pager list-unit-files iptables.service | grep 'enabled' | wc -l)
+
+if [ $check -eq 1 ]; then
+    if [ $test_only -eq 0 ]; then
+        if type -p systemctl && [[ "$(systemctl --no-pager list-unit-files firewalld)" =~ 'enabled' ]]; then
+            systemctl disable firewalld
+        fi
+
+        if type -p service; then
+            service ip6tables save
+            service iptables save
+            chkconfig ip6tables off
+            chkconfig iptables off
+        fi
+    else
+        exit 0
+    fi
+fi

sahara/plugins/mapr/resources/images/centos/install_mapr_core_repository

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+DISTRO_NAME=$distro
+source "/tmp/package_utils.sh"
+
+echo "START: installing MapR core repository"
+
+MAPR_REPO_URL="http://package.mapr.com/releases/v${plugin_version}/redhat/mapr-v${plugin_version}GA.rpm.tgz"
+MAPR_REPO_DIR="/opt/mapr-repository/core"
+
+if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then
+    if [ $test_only -eq 0 ]; then
+
+        MAPR_REPO_NAME="mapr_core"
+
+        echo "Downloading MapR repository archive"
+        mkdir -p "$MAPR_REPO_DIR" && curl "$MAPR_REPO_URL" | tar -xz -C "$MAPR_REPO_DIR"
+
+        echo "Creating local repository"
+        create_repo "$MAPR_REPO_DIR"
+
+        echo "Adding MapR repository"
+        add_local_repo "$MAPR_REPO_NAME" "$MAPR_REPO_DIR"
+    fi
+fi
+
+echo "END: installing MapR core repository"

sahara/plugins/mapr/resources/images/centos/install_mapr_eco_repository

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+
+VERSIONS_PY="/tmp/versions.py"
+DISTRO_NAME=$distro
+
+source "/tmp/package_utils.sh"
+
+echo "START: installing MapR ecosystem repository"
+
+MAPR_REPO_URL="http://package.mapr.com/releases/MEP/MEP-2.0.0/redhat/"
+MAPR_REPO_DIR="/opt/mapr-repository/ecosystem"
+
+if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then
+    if [ $test_only -eq 0 ]; then
+
+        MAPR_REPO_NAME="mapr_ecosystem"
+        MAPR_PKG_GROUPS="/tmp/packages.json"
+
+        MAPR_SPEC="/tmp/spec_$plugin_version.json"
+
+        echo "Creating local MapR ecosystem repository"
+        localize_repo "$MAPR_REPO_NAME" "$MAPR_REPO_URL" "$MAPR_PKG_GROUPS" "$MAPR_SPEC" "$MAPR_REPO_DIR"
+        echo $MAPR_SPEC
+    fi
+fi
+echo "END: installing MapR ecosystem repository"

sahara/plugins/mapr/resources/images/centos/install_openjdk

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+echo "Installing OpenJDK"
+
+if [ $test_only -eq 0 ]; then
+    yum install -y java-1.8.0-openjdk-devel
+
+    JRE_HOME="/usr/lib/jvm/java-openjdk/jre"
+    JDK_HOME="/usr/lib/jvm/java-openjdk"
+
+    echo "OpenJDK has been installed"
+else
+    exit 0
+fi

sahara/plugins/mapr/resources/images/centos/install_scala

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+echo "START: installing Scala"
+sudo yum -y update
+exit 0
+
+if [ $test_only -eq 0 ]; then
+
+   RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://www.scala-lang.org/)"
+
+   if [ "$RETURN_CODE" != "200" ]; then
+       echo "http://www.scala-lang.org is unreachable" && exit 1
+   fi
+
+   if [ -n "${scala_version:-}" ]; then
+       VERSION=$scala_version
+   else
+       VERSION="$(curl -s --fail http://www.scala-lang.org| tr -d '\n' | sed 's/^.*<div[^<]\+scala-version">[^0-9]\+\([0-9\.\?]\+\)<.\+$/\1/')"
+
+       if [ $? != 0 -o -z "${VERSION}" ]; then
+           echo "Installing default version $scala_version"
+           VERSION=$scala_version
+       fi
+   fi
+
+   PKG=scala-${VERSION}
+
+   URL="http://downloads.lightbend.com/scala/${VERSION}"
+
+   rpm -Uhv ${URL}/${PKG}.rpm
+fi
+
+echo "END: installing Scala"
+

sahara/plugins/mapr/resources/images/centos/requiretty

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+
+if [ $test_only -eq 0 ]; then
+    sed '/^Defaults    requiretty*/ s/^/#/' -i /etc/sudoers
+fi

sahara/plugins/mapr/resources/images/centos/selinux_permissive

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+check=$(cat /etc/selinux/config | grep "SELINUX=permissive" | wc -l)
+
+if [ $check -eq  0 ]; then
+    if [ $test_only -eq 0 ]; then
+        echo "SELINUX=permissive" > /etc/selinux/config
+        echo "SELINUXTYPE=targeted" >> /etc/selinux/config
+    else
+        exit 0
+    fi
+fi

sahara/plugins/mapr/resources/images/centos/update_repository

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+if [ $test_only -eq 0 ]; then
+    yum clean all && yum repolist
+fi

sahara/plugins/mapr/resources/images/common/configure_extjs

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+EXTJS_DESTINATION_DIR="/opt/mapr-repository"
+EXTJS_DOWNLOAD_URL="http://tarballs.openstack.org/sahara/dist/common-artifacts/ext-2.2.zip"
+EXTJS_NO_UNPACK=1
+
+extjs_archive=/tmp/$(basename $EXTJS_DOWNLOAD_URL)
+
+if [ ! -f "${EXTJS_DESTINATION_DIR}/${extjs_archive}" ]; then
+    if [ $test_only -eq 0 ]; then
+        wget -O $extjs_archive $EXTJS_DOWNLOAD_URL
+
+        mkdir -p $EXTJS_DESTINATION_DIR
+        if [ -z "${EXTJS_NO_UNPACK:-}" ]; then
+            unzip -d "$EXTJS_DESTINATION_DIR" $extjs_archive
+            rm -f $extjs_archive
+        else
+            mv $extjs_archive $EXTJS_DESTINATION_DIR
+        fi
+    else
+        exit 0
+    fi
+fi

sahara/plugins/mapr/resources/images/common/oracle_java

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+
+# NOTE: $(dirname $0) is read-only, use space under $TARGET_ROOT
+JAVA_LOCATION=${JAVA_TARGET_LOCATION:-"/usr/java"}
+JAVA_NAME="oracle-jdk"
+JAVA_HOME=$JAVA_LOCATION/$JAVA_NAME
+JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"}
+
+# FIXME: probably not idempotent, find a better condition
+if [ ! -d $JAVA_LOCATION ]; then
+    if [ $test_only -eq 0 ]; then
+        echo "Begin: installation of Java"
+        mkdir -p $JAVA_LOCATION
+
+        if [ -n "$JAVA_DOWNLOAD_URL" ]; then
+            JAVA_FILE=$(basename $JAVA_DOWNLOAD_URL)
+            wget --no-check-certificate --no-cookies -c \
+                --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \
+                -O $JAVA_LOCATION/$JAVA_FILE $JAVA_DOWNLOAD_URL
+        elif [ -n "$JAVA_FILE" ]; then
+            install -D -g root -o root -m 0755 $(dirname $0)/$JAVA_FILE $JAVA_LOCATION
+        fi
+
+        cd $JAVA_LOCATION
+
+        echo "Decompressing Java archive"
+        echo -e "\n" | tar -zxf $JAVA_FILE
+        echo "Setting up $JAVA_NAME"
+        chown -R root:root $JAVA_LOCATION
+        JAVA_DIR=`ls -1 $JAVA_LOCATION | grep -v tar.gz`
+        ln -s $JAVA_LOCATION/$JAVA_DIR $JAVA_HOME
+
+        setup-java-home $JAVA_HOME $JAVA_HOME
+
+        rm $JAVA_FILE
+
+        echo "End: installation of Java"
+    else
+        exit 0
+    fi
+fi

sahara/plugins/mapr/resources/images/common/resources/package_utils.sh

@@ -0,0 +1,226 @@
+# execute_in_directory <directory> <command>
+execute_in_directory() {
+    local directory="$(readlink -f "$1")"; shift
+    local cmd="$*"
+
+    pushd "$directory" && eval "$cmd" && popd
+}
+
+# get_distro
+get_distro() {
+    echo "$DISTRO_NAME"
+}
+
+# download_apt_package <package> [version] [directory]
+download_apt_package() {
+    local package="$1"
+    local version="${2:-}"
+    local directory="${3:-$(pwd)}"
+    local package_spec="$package${version:+=$version*}"
+
+    execute_in_directory "$directory" apt-get --allow-unauthenticated download "$package_spec"
+}
+
+# download_yum_package <package> [version] [directory]
+download_yum_package() {
+    local package="$1"
+    local version="${2:-}"
+    local directory="${3:-$(pwd)}"
+    local package_spec="$package${version:+-$version*}"
+
+    yumdownloader --destdir "$directory" "$package_spec"
+}
+
+# download_package <package> [version] [directory] [distro]
+download_package() {
+    local package="$1"
+    local version="${2:-}"
+    local directory="${3:-$(pwd)}"
+    local distro="${4:-$(get_distro)}"
+
+    if [[ "$distro" == "ubuntu" ]]; then
+        download_apt_package "$package" "$version" "$directory"
+    elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
+        download_yum_package "$package" "$version" "$directory"
+    fi
+}
+
+# get_packages <package_groups_file> <spec_file> [version_separator]
+get_packages() {
+    local package_groups_file="$1"
+    local spec_file="$2"
+    local version_separator="${3:-:}"
+
+    python "$VERSIONS_PY" --separator "$version_separator" "$package_groups_file" "$spec_file"
+}
+
+# download_packages <package_groups_file> <spec_file> [directory] [distro]
+download_packages() {
+    local package_groups_file="$1"
+    local spec_file="$2"
+    local directory="${3:-$(pwd)}"
+    local distro="${4:-$(get_distro)}"
+    local version_separator=":"
+
+    local packages="$(get_packages "$package_groups_file" "$spec_file" "$version_separator")"
+    for package in $packages; do
+        IFS="$version_separator" read -ra package_version <<< "$package"
+        download_package "${package_version[@]}" "$directory" "$distro"
+    done
+}
+
+# create_apt_repo <directory>
+create_apt_repo() {
+    local directory="$(readlink -f "$1")"
+    local binary_dir="$directory/binary"
+    local packages_gz="$binary_dir/Packages.gz"
+
+    mkdir -p "$binary_dir"
+    execute_in_directory "$directory" "dpkg-scanpackages -m . /dev/null | gzip -9c > $packages_gz"
+}
+
+# create_yum_repo <directory>
+create_yum_repo() {
+    local directory="$(readlink -f "$1")"
+
+    createrepo "$directory"
+}
+
+# create_repo <directory> [distro]
+create_repo() {
+    local directory="$(readlink -f "$1")"
+    local distro="${2:-$(get_distro)}"
+
+    if [[ "$distro" == "ubuntu" ]]; then
+        create_apt_repo "$directory"
+    elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
+        create_yum_repo "$directory"
+    fi
+}
+
+# add_apt_repo <repo_name> <repo_url>
+add_apt_repo() {
+    local repo_name="$1"
+    local repo_url="$2"
+    local repo="deb $repo_url"
+    local repo_path="/etc/apt/sources.list.d/$repo_name.list"
+
+    echo "$repo" > "$repo_path" && apt-get update
+}
+
+# add_yum_repo <repo_name> <repo_url>
+add_yum_repo() {
+    local repo_name="$1"
+    local repo_url="$2"
+    local repo_path="/etc/yum.repos.d/$repo_name.repo"
+
+    cat > "$repo_path" << EOF
+[$repo_name]
+name=$repo_name
+baseurl=$repo_url
+enabled=1
+gpgcheck=0
+protect=1
+EOF
+    yum clean all && rm -rf /var/cache/yum/* && yum check-update
+}
+
+# add_repo <repo_name> <repo_url> [distro]
+add_repo() {
+    local repo_name="$1"
+    local repo_url="$2"
+    local distro="${3:-$(get_distro)}"
+
+    if [[ "$distro" == "ubuntu" ]]; then
+        add_apt_repo "$repo_name" "$repo_url"
+    elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
+        add_yum_repo "$repo_name" "$repo_url"
+    fi
+}
+
+# add_local_apt_repo <repo_name> <directory>
+add_local_apt_repo() {
+    local repo_name="$1"
+    local directory="$(readlink -f "$2")"
+    local repo_url="file:$directory binary/"
+
+    add_apt_repo "$repo_name" "$repo_url"
+}
+
+# add_local_yum_repo <repo_name> <directory>
+add_local_yum_repo() {
+    local repo_name="$1"
+    local directory="$(readlink -f "$2")"
+    local repo_url="file://$directory"
+
+    add_yum_repo "$repo_name" "$repo_url"
+}
+
+# add_local_repo <repo_name> <directory> [distro]
+add_local_repo() {
+    local repo_name="$1"
+    local directory="$(readlink -f "$2")"
+    local distro="${3:-$(get_distro)}"
+
+    if [[ "$distro" == "ubuntu" ]]; then
+        add_local_apt_repo "$repo_name" "$directory"
+    elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
+        add_local_yum_repo "$repo_name" "$directory"
+    fi
+}
+
+# remove_apt_repo <repo_name>
+remove_apt_repo() {
+    local repo_name="$1"
+    local repo_path="/etc/apt/sources.list.d/$repo_name.list"
+
+    rm "$repo_path" && apt-get update
+}
+
+# remove_yum_repo <repo_name>
+remove_yum_repo() {
+    local repo_name="$1"
+    local repo_path="/etc/yum.repos.d/$repo_name.repo"
+
+    rm "$repo_path"
+}
+
+# remove_repo <repo_name> [distro]
+remove_repo() {
+    local repo_name="$1"
+    local distro="${2:-$(get_distro)}"
+
+    if [[ "$distro" == "ubuntu" ]]; then
+        remove_apt_repo "$repo_name"
+    elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
+        remove_yum_repo "$repo_name"
+    fi
+}
+
+# create_local_repo <repo_name> <repo_url> <package_groups_file> <spec_file> <directory>
+create_local_repo() {
+    local repo_name="$1"
+    local repo_url="$2"
+    local package_groups_file="$3"
+    local spec_file="$4"
+    local directory="$5"
+
+    add_repo "$repo_name" "$repo_url"
+    mkdir -p "$directory" && directory="$(readlink -f "$directory")"
+    download_packages "$package_groups_file" "$spec_file" "$directory"
+    remove_repo "$repo_name"
+    create_repo "$directory"
+}
+
+# localize_repo <repo_name> <repo_url> <package_groups_file> <spec_file> <directory>
+localize_repo() {
+    local repo_name="$1"
+    local repo_url="$2"
+    local package_groups_file="$3"
+    local spec_file="$4"
+    local directory="$5"
+
+    mkdir -p "$directory" && directory="$(readlink -f "$directory")"
+    create_local_repo "$repo_name" "$repo_url" "$package_groups_file" "$spec_file" "$directory"
+    add_local_repo "$repo_name" "$directory"
+}

sahara/plugins/mapr/resources/images/common/resources/packages.json

@@ -0,0 +1,140 @@
+{
+  "asynchbase": {
+    "all": [
+      "mapr-asynchbase"
+    ]
+  },
+  "drill": {
+    "all": [
+      "mapr-drill"
+    ]
+  },
+  "flume": {
+    "all": [
+      "mapr-flume"
+    ]
+  },
+  "hbase": {
+    "all": [
+      "mapr-hbase",
+      "mapr-hbase-internal",
+      "mapr-hbase-master",
+      "mapr-hbase-regionserver",
+      "mapr-hbasethrift",
+      "mapr-hbase-rest"
+    ],
+    "0.98.12": [
+      "mapr-hbase",
+      "mapr-hbase-internal",
+      "mapr-hbase-master",
+      "mapr-hbase-regionserver",
+      "mapr-hbasethrift",
+      "mapr-libhbase",
+      "mapr-hbase-rest"
+    ],
+    "1.1.1": [
+      "mapr-hbase",
+      "mapr-hbase-internal",
+      "mapr-hbase-master",
+      "mapr-hbase-regionserver",
+      "mapr-hbasethrift",
+      "mapr-libhbase",
+      "mapr-hbase-rest"
+    ]
+  },
+  "hive": {
+    "all": [
+      "mapr-hive",
+      "mapr-hivemetastore",
+      "mapr-hiveserver2"
+    ]
+  },
+  "httpfs": {
+    "all": [
+      "mapr-httpfs"
+    ]
+  },
+  "hue": {
+    "all": [
+      "mapr-hue",
+      "mapr-hue-base",
+      "mapr-hue-livy"
+    ],
+    "3.10.0": [
+      "mapr-hue",
+      "mapr-hue-livy"
+    ]
+  },
+  "impala": {
+    "all": [
+      "mapr-impala",
+      "mapr-impala-catalog",
+      "mapr-impala-server",
+      "mapr-impala-statestore",
+      "mapr-impala-udf"
+    ]
+  },
+  "mahout": {
+    "all": [
+      "mapr-mahout"
+    ]
+  },
+  "oozie": {
+    "all": [
+      "mapr-oozie",
+      "mapr-oozie-internal"
+    ]
+  },
+  "pig": {
+    "all": [
+      "mapr-pig"
+    ]
+  },
+  "sentry": {
+    "all": [
+      "mapr-sentry"
+    ]
+  },
+  "spark": {
+    "all": [
+      "mapr-spark",
+      "mapr-spark-historyserver",
+      "mapr-spark-master"
+    ]
+  },
+  "sqoop": {
+    "all": [
+      "mapr-sqoop2-client",
+      "mapr-sqoop2-server"
+    ]
+  },
+  "storm": {
+    "all": [
+      "mapr-storm",
+      "mapr-storm-ui",
+      "mapr-storm-nimbus",
+      "mapr-storm-supervisor"
+    ]
+  },
+  "tez": {
+    "all": [
+      "mapr-tez"
+    ]
+  },
+  "kafka": {
+    "all": [
+      "mapr-kafka"
+    ]
+  },
+  "kafka-connect": {
+    "all": [
+      "mapr-kafka-connect-hdfs",
+      "mapr-kafka-connect-jdbc"
+    ]
+  },
+  "kafka-rest": {
+    "all": [
+      "mapr-kafka-rest"
+    ]
+  }
+}

sahara/plugins/mapr/resources/images/common/resources/spec_5.1.0.json

@@ -0,0 +1,46 @@
+{
+  "drill": [
+    "1.1.0",
+    "1.2.0",
+    "1.4.0"
+  ],
+  "flume": [
+    "1.5.0",
+    "1.6.0"
+  ],
+  "hbase": [
+    "0.98.9",
+    "0.98.12"
+  ],
+  "hive": [
+    "0.13",
+    "1.0",
+    "1.2"
+  ],
+  "httpfs": [
+    "1.0"
+  ],
+  "hue": [
+    "3.8.1",
+    "3.9.0"
+  ],
+  "impala": [
+    "1.4.1"
+  ],
+  "mahout": [
+    "0.10.0"
+  ],
+  "oozie": [
+    "4.2.0"
+  ],
+  "pig": [
+    "0.14",
+    "0.15"
+  ],
+  "sqoop": [
+    "2.0.0"
+  ],
+  "spark": [
+    "1.5.2"
+  ]
+}

sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0.json

@@ -0,0 +1,50 @@
+{
+  "drill": [
+    "1.9.0"
+  ],
+  "flume": [
+    "1.6.0"
+  ],
+  "hbase": [
+    "1.1.1"
+  ],
+  "hive": [
+    "1.2"
+  ],
+  "httpfs": [
+    "1.0"
+  ],
+  "hue": [
+    "3.10.0"
+  ],
+  "impala": [
+    "2.5.0"
+  ],
+  "mahout": [
+    "0.12.0"
+  ],
+  "oozie": [
+    "4.2.0"
+  ],
+  "pig": [
+    "0.16"
+  ],
+  "sqoop": [
+    "2.0.0"
+  ],
+  "spark": [
+    "2.0.1"
+  ],
+  "sentry": [
+    "1.6.0"
+  ],
+  "kafka": [
+    "0.9.0"
+  ],
+  "kafka-connect": [
+    "2.0.1"
+  ],
+  "kafka-rest": [
+    "2.0.1"
+  ]
+}

sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json

@@ -0,0 +1,47 @@
+{
+  "drill": [
+    "1.9.0"
+  ],
+  "flume": [
+    "1.6.0"
+  ],
+  "hbase": [
+    "1.1.1"
+  ],
+  "hive": [
+    "1.2"
+  ],
+  "httpfs": [
+    "1.0"
+  ],
+  "hue": [
+    "3.10.0"
+  ],
+  "mahout": [
+    "0.12.0"
+  ],
+  "oozie": [
+    "4.2.0"
+  ],
+  "pig": [
+    "0.16"
+  ],
+  "sqoop": [
+    "2.0.0"
+  ],
+  "spark": [
+    "2.0.1"
+  ],
+  "sentry": [
+    "1.6.0"
+  ],
+  "kafka": [
+    "0.9.0"
+  ],
+  "kafka-connect": [
+    "2.0.1"
+  ],
+  "kafka-rest": [
+    "2.0.1"
+  ]
+}

sahara/plugins/mapr/resources/images/common/resources/versions.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2015, MapR Technologies
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import argparse
+import sys
+
+from oslo_serialization import jsonutils as json
+
+_GROUP_VERSION_SEPARATOR = ","
+_ALL_GROUP_VERSION = "all"
+
+
+def _build_parser():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("packages", help="path to the packages.json")
+    parser.add_argument("spec", help="path to the spec.json")
+    parser.add_argument("--separator", default=":",
+                        help="separator between package name"
+                             " and version in output")
+
+    return parser
+
+
+def _load_json(path):
+    with open(path) as json_file:
+        return json.load(json_file)
+
+
+def _version_matches(version, group_version):
+    for gv in group_version.split(_GROUP_VERSION_SEPARATOR):
+        if version.startswith(gv):
+            return True
+
+    return False
+
+
+def _get_packages(version, group_spec):
+    for group_version in group_spec:
+        if _version_matches(version, group_version):
+            return group_spec[group_version]
+
+    return group_spec[_ALL_GROUP_VERSION]
+
+
+def _get_package_versions(spec, package_groups):
+    return [(package, version)
+            for pg_name, versions in spec.items()
+            for version in versions
+            for package in _get_packages(version, package_groups[pg_name])]
+
+
+parser = _build_parser()
+
+
+def main(args=None):
+    args = parser.parse_args(args or sys.argv[1:])
+
+    spec = _load_json(args.spec)
+    package_groups = _load_json(args.packages)
+    separator = args.separator
+
+    package_versions = _get_package_versions(spec, package_groups)
+    package_format = "%s" + separator + "%s\n"
+    package_versions = map(lambda pv: package_format % pv, package_versions)
+
+    sys.stdout.writelines(package_versions)
+
+
+if __name__ == "__main__":
+    main()

sahara/plugins/mapr/resources/images/image.yaml

@@ -0,0 +1,195 @@
+arguments:
+  java_distro:
+    default: openjdk
+    description: The distribution of Java to install. Defaults to cloudera-jdk.
+    choices:
+      - openjdk
+      - oracle-java
+  plugin_version:
+    default: 5.2.0
+    description: The distribution of MapR to install. Defaults to 5.2.0.
+    hidden: True
+    required: False
+  scala_version:
+    default: 2.11.6
+    description: The version of scala to install. Defaults to 2.11.6.
+    hidden: True
+    required: False
+  hdfs_lib_dir:
+    default: /usr/lib/hadoop-mapreduce
+    description: The path to HDFS_LIB_DIR. Default to /usr/lib/hadoop-mapreduce
+    required: False
+
+validators:
+  - os_case:
+      - ubuntu:
+          - script: ubuntu/install_mapr_dependencies
+  - package:
+    - mtools
+    - rpcbind
+    - sdparm
+    - syslinux
+    - unzip
+    - wget
+    - zip
+  - os_case:
+      - centos:
+          - package:
+              - cups
+              - cdparanoia-libs
+              - cups-libs
+              - createrepo
+              - cvs
+              - cyrus-sasl-gssapi
+              - cyrus-sasl-plain
+              - foomatic
+              - foomatic-db
+              - foomatic-db-filesystem
+              - foomatic-db-ppds
+              - gdbm-devel
+              - gettext
+              - ghostscript
+              - ghostscript-fonts
+              - glibc
+              - glibc-common
+              - glibc-devel
+              - glibc-headers
+              - gstreamer
+              - gstreamer-plugins-base
+              - gstreamer-tools
+              - hdparm
+              - irqbalance
+              - iso-codes
+              - kernel-headers
+              - libXt
+              - libXv
+              - libXxf86vm
+              - libgomp
+              - libgudev1
+              - libicu
+              - libmng
+              - liboil
+              - libtheora
+              - libtirpc
+              - libvisual
+              - libxslt
+              - mesa-dri-drivers
+              - mesa-libGL
+              - mesa-libGLU
+              - mesa-private-llvm
+              - nmap-ncat
+              - numactl
+              - openjpeg-libs
+              - patch
+              - pax
+              - perl-CGI
+              - perl-ExtUtils-MakeMaker
+              - perl-ExtUtils-ParseXS
+              - perl-Test-Harness
+              - perl-Test-Simple
+              - perl-devel
+              - phonon-backend-gstreamer
+              - poppler
+              - poppler-data
+              - poppler-utils
+              - portreserve
+              - qt
+              - qt-x11
+              - qt3
+              - redhat-lsb
+              - redhat-lsb-core
+              - redhat-lsb-printing
+              - urw-fonts
+              - yum-utils
+              - xml-common
+      - ubuntu:
+          - package:
+              - binutils
+              - daemon
+              - dpkg-dev
+              - dpkg-repack
+              - gcc
+              - gcc-4.8
+              - gcc-doc
+              - gcc-multilib
+              - iputils-arping
+              - libasan0
+              - libatomic1
+              - libc-dev-bin
+              - libc6
+              - libc6-dev
+              - libcrypt-passwdmd5-perl
+              - libgcc-4.8-dev
+              - libgomp1
+              - libgssglue1
+              - libicu48
+              - libitm1
+              - libmysqlclient-dev
+              - libmysqlclient16
+              - libmysqlclient18
+              - libnfsidmap2
+              - libquadmath0
+              - libsasl2-dev
+              - libsasl2-modules-gssapi-mit
+              - libssl0.9.8
+              - libtirpc1
+              - libtsan0
+              - libxslt1.1
+              - linux-libc-dev
+              - manpages-dev
+              - mysql-common
+              - nfs-common
+              - open-iscsi
+              - openjdk-7-jre
+              - syslinux-common
+              - zlib1g-dev
+  - script: common/configure_extjs
+  - os_case:
+      - centos:
+          - copy_script: common/resources/package_utils.sh
+          - copy_script: common/resources/packages.json
+          - copy_script: common/resources/spec_5.1.0.json
+          - copy_script: common/resources/spec_5.2.0.json
+          - copy_script: common/resources/versions.py
+          - script:
+              centos/install_scala:
+                  env_vars: [scala_version]
+          - script:
+              centos/install_mapr_core_repository:
+                  env_vars: [plugin_version]
+          - script:
+              centos/install_mapr_eco_repository:
+                  env_vars: [plugin_version]
+          - script: centos/selinux_permissive
+          - argument_case:
+              argument_name: java_distro
+              cases:
+                  openjdk:
+                      - script: centos/install_openjdk
+                  oracle-java:
+                      - script: common/oracle_java
+      - ubuntu:
+          - copy_script: common/resources/package_utils.sh
+          - copy_script: common/resources/packages.json
+          - copy_script: common/resources/spec_5.1.0.json
+          - copy_script: common/resources/spec_5.2.0.json
+          - copy_script: common/resources/spec_5.2.0_ubuntu.json
+          - copy_script: common/resources/versions.py
+          - script:
+              ubuntu/install_scala:
+                  env_vars: [scala_version]
+          - script:
+              ubuntu/install_mapr_core_repository:
+                  env_vars: [plugin_version]
+          - script:
+              ubuntu/install_mapr_eco_repository:
+                  env_vars: [plugin_version]
+  - os_case:
+      - ubuntu:
+          - argument_case:
+              argument_name: java_distro
+              cases:
+                  openjdk:
+                      - script: ubuntu/install_openjdk
+                  oracle-java:
+                      - script: common/oracle_java

sahara/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+DISTRO_NAME=$distro
+source "/tmp/package_utils.sh"
+
+echo "START: installing MapR core repository"
+
+MAPR_REPO_URL="http://package.mapr.com/releases/v${plugin_version}/ubuntu/mapr-v${plugin_version}GA.deb.tgz"
+MAPR_REPO_DIR="/opt/mapr-repository/core"
+
+if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then
+    if [ $test_only -eq 0 ]; then
+
+        MAPR_REPO_NAME="mapr_core"
+
+        echo "Downloading MapR repository archive"
+        mkdir -p "$MAPR_REPO_DIR" && curl "$MAPR_REPO_URL" | tar -xz -C "$MAPR_REPO_DIR"
+
+        echo "Creating local repository"
+        create_repo "$MAPR_REPO_DIR"
+
+        echo "Adding MapR repository"
+        add_local_repo "$MAPR_REPO_NAME" "$MAPR_REPO_DIR"
+    fi
+fi
+
+echo "END: installing MapR core repository"

sahara/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+echo "START: installing MapR core dependencies"
+
+if [ ! -f /etc/apt/sources.list.d/security_repo.list ]; then
+    if [ $test_only -eq 0 ]; then
+        # Required for libicu48
+        cat >> /etc/apt/sources.list.d/security_repo.list << EOF
+deb http://security.ubuntu.com/ubuntu precise-security main
+EOF
+        # Required for libmysqlclient16
+        cat >> /etc/apt/sources.list.d/security_repo.list << EOF
+deb http://old-releases.ubuntu.com/ubuntu lucid-security main
+EOF
+    else
+        exit 0
+    fi
+fi
+
+apt-get update
+
+echo "END: installing MapR core dependencies"

sahara/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+
+VERSIONS_PY="/tmp/versions.py"
+DISTRO_NAME=$distro
+
+source "/tmp/package_utils.sh"
+
+echo "START: installing MapR ecosystem repository"
+
+MAPR_REPO_URL="http://package.mapr.com/releases/MEP/MEP-2.0.0/ubuntu/ binary trusty"
+MAPR_REPO_DIR="/opt/mapr-repository/ecosystem"
+
+if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then
+    if [ $test_only -eq 0 ]; then
+
+        MAPR_REPO_NAME="mapr_ecosystem"
+        MAPR_PKG_GROUPS="/tmp/packages.json"
+
+        if [ -f /tmp/spec_$plugin_version_ubuntu.json ]; then
+            MAPR_SPEC="/tmp/spec_$plugin_version_ubuntu.json"
+        else
+            MAPR_SPEC="/tmp/spec_$plugin_version.json"
+        fi
+
+        echo "Creating local MapR ecosystem repository"
+        localize_repo "$MAPR_REPO_NAME" "$MAPR_REPO_URL" "$MAPR_PKG_GROUPS" "$MAPR_SPEC" "$MAPR_REPO_DIR"
+        echo $MAPR_SPEC
+    fi
+fi
+
+echo "END: installing MapR ecosystem repository"

sahara/plugins/mapr/resources/images/ubuntu/install_openjdk

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo "Installing OpenJDK"
+
+if [ $test_only -eq 0 ]; then
+    apt-get install -y openjdk-7-jdk
+
+    JRE_PATH=$(update-alternatives --list java)
+    JRE_HOME=${JRE_PATH%/bin/java}
+    JDK_PATH=$(update-alternatives --list javac)
+    JDK_HOME=${JDK_PATH%/bin/javac}
+
+    echo "OpenJDK has been installed"
+else
+    exit 0
+fi

sahara/plugins/mapr/resources/images/ubuntu/install_scala

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+echo "START: installing Scala"
+
+
+if [ $test_only -eq 0 ]; then
+
+   RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://www.scala-lang.org/)"
+
+   if [ "$RETURN_CODE" != "200" ]; then
+       echo "http://www.scala-lang.org is unreachable" && exit 1
+   fi
+
+   if [ -n "${scala_version:-}" ]; then
+       VERSION=$scala_version
+   else
+       VERSION="$(curl -s --fail http://www.scala-lang.org| tr -d '\n' | sed 's/^.*<div[^<]\+scala-version">[^0-9]\+\([0-9\.\?]\+\)<.\+$/\1/')"
+
+       if [ $? != 0 -o -z "${VERSION}" ]; then
+           echo "Installing default version $scala_version"
+           VERSION=$scala_version
+       fi
+   fi
+
+   PKG=scala-${VERSION}
+
+   URL="http://downloads.lightbend.com/scala/${VERSION}"
+
+   wget -N ${URL}/${PKG}.deb
+   dpkg -i ${PKG}.deb
+   rm ${PKG}.deb
+fi
+
+echo "END: installing Scala"
+

sahara/plugins/mapr/resources/images/ubuntu/update_repository

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+if [ $test_only -eq 0 ]; then
+    apt-get update
+fi