From 61823d408c1272af79c872ce3f497f643dd99499 Mon Sep 17 00:00:00 2001 From: Artem Osadchiy Date: Thu, 31 Jul 2014 13:49:59 +0300 Subject: [PATCH] Image Elements for MapR Change-Id: I87fde2f5d5b430b935c3261d31b86a7b68ac3244 Implements: blueprint mapr-image-elements --- diskimage-create/README.rst | 2 +- diskimage-create/diskimage-create.sh | 88 +++++++++- elements/hadoop-mapr/README.rst | 21 +++ elements/hadoop-mapr/element-deps | 3 + .../install.d/40-mapr-dependencies | 159 ++++++++++++++++++ elements/hadoop-mapr/install.d/41-scala | 39 +++++ elements/hadoop-mapr/install.d/42-versions | 5 + .../install.d/43-mapr-core-repository | 63 +++++++ .../install.d/44-mapr-eco-repository | 87 ++++++++++ elements/hadoop-mapr/install.d/44-oozie-libs | 15 ++ elements/hadoop-mapr/install.d/versions.py | 102 +++++++++++ .../hadoop-mapr/post-install.d/99-java-home | 6 + .../hadoop-mapr/post-install.d/99-requiretty | 10 ++ .../post-install.d/99-update-repository | 11 ++ .../hadoop-mapr/root.d/0-mapr-check-version | 10 ++ tox.ini | 2 +- 16 files changed, 618 insertions(+), 5 deletions(-) create mode 100644 elements/hadoop-mapr/README.rst create mode 100644 elements/hadoop-mapr/element-deps create mode 100755 elements/hadoop-mapr/install.d/40-mapr-dependencies create mode 100755 elements/hadoop-mapr/install.d/41-scala create mode 100755 elements/hadoop-mapr/install.d/42-versions create mode 100755 elements/hadoop-mapr/install.d/43-mapr-core-repository create mode 100755 elements/hadoop-mapr/install.d/44-mapr-eco-repository create mode 100755 elements/hadoop-mapr/install.d/44-oozie-libs create mode 100644 elements/hadoop-mapr/install.d/versions.py create mode 100755 elements/hadoop-mapr/post-install.d/99-java-home create mode 100755 elements/hadoop-mapr/post-install.d/99-requiretty create mode 100755 elements/hadoop-mapr/post-install.d/99-update-repository create mode 100755 elements/hadoop-mapr/root.d/0-mapr-check-version diff --git a/diskimage-create/README.rst b/diskimage-create/README.rst index 34e2ff95..07da003e 100644 --- a/diskimage-create/README.rst +++ b/diskimage-create/README.rst @@ -27,7 +27,7 @@ NOTE: Do not create all images for all plugins with the same mirrors. Different .. sourcecode:: bash - sudo bash sahara-image-elements/diskimage-create/diskimage-create.sh -p [vanilla|spark|hdp|cloudera|storm] + sudo bash sahara-image-elements/diskimage-create/diskimage-create.sh -p [vanilla|spark|hdp|cloudera|storm|mapr] 5. To select which hadoop version to target use the '-v' commandline option like this: diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index 9275d304..3ee4166a 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -12,20 +12,25 @@ DEBUG_MODE="false" # The default tag to use for the DIB repo DEFAULT_DIB_REPO_BRANCH="0.1.29" +# The default version for a MapR plugin +DIB_DEFAULT_MAPR_VERSION="4.0.1" + # Default list of datasource modules for ubuntu. Workaround for bug #1375645 export CLOUD_INIT_DATASOURCES=${DIB_CLOUD_INIT_DATASOURCES:-"NoCloud, ConfigDrive, OVF, MAAS, Ec2"} usage() { echo echo "Usage: $(basename $0)" - echo " [-p vanilla|spark|hdp|cloudera|storm]" + echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]" echo " [-i ubuntu|fedora|centos]" echo " [-v 1|2|2.3|2.4|plain]" + echo " [-r 3.1.1|4.0.1]" echo " [-d]" echo " [-m]" echo " '-p' is plugin version (default: all plugins)" echo " '-i' is operating system of the base image (default: all supported by plugin)" echo " '-v' is hadoop version (default: all supported by plugin)" + echo " '-r' is MapR Version (default: ${DIB_DEFAULT_MAPR_VERSION})" echo " '-d' enable debug mode, root account will have password 'hadoop'" echo " '-m' set the diskimage-builder repo to the master branch (default: $DEFAULT_DIB_REPO_BRANCH)" echo @@ -38,7 +43,7 @@ usage() { exit 1 } -while getopts "p:i:v:dm" opt; do +while getopts "p:i:v:dmr:" opt; do case $opt in p) PLUGIN=$OPTARG @@ -60,6 +65,9 @@ while getopts "p:i:v:dm" opt; do DIB_REPO_BRANCH="master" fi ;; + r) + DIB_MAPR_VERSION=$OPTARG + ;; *) usage ;; @@ -93,7 +101,7 @@ if [ "$DEBUG_MODE" = "true" -a "$platform" != 'NAME="Ubuntu"' ]; then fi fi -if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" -a "$PLUGIN" != "storm" ]; then +if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" -a "$PLUGIN" != "storm" -a "$PLUGIN" != "mapr" ]; then echo -e "Unknown plugin selected.\nAborting" exit 1 fi @@ -120,6 +128,27 @@ if [ "$PLUGIN" = "cloudera" -a "$BASE_IMAGE_OS" = "fedora" ]; then exit 1 fi +if [ "$PLUGIN" = "mapr" -a "$BASE_IMAGE_OS" = "fedora" ]; then + echo "'fedora' image type is not supported by 'mapr' plugin.\nAborting" + exit 1 +fi + +if [ "$PLUGIN" != "mapr" -a -n "$DIB_MAPR_VERSION" ]; then + echo "'-r' parameter should be used only with 'mapr' plugin.\nAborting" + exit 1 +fi + +if [ "$PLUGIN" = "mapr" -a -z "$DIB_MAPR_VERSION" ]; then + echo "MapR version is not specified.\n" + echo "${DIB_DEFAULT_MAPR_VERSION} version would be used.\n" + DIB_MAPR_VERSION=${DIB_DEFAULT_MAPR_VERSION} +fi + +if [ "$PLUGIN" = "mapr" -a "${DIB_MAPR_VERSION}" != "3.1.1" -a "${DIB_MAPR_VERSION}" != "4.0.1" ]; then + echo "Unknown MapR version.\nExit" + exit 1 +fi + ################# if [ "$platform" = 'NAME="Ubuntu"' ]; then @@ -485,5 +514,58 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then unset EXTJS_DOWNLOAD_URL fi +########################## +# Images for MapR plugin # +########################## +if [ -z "$PLUGIN" -o "$PLUGIN" = "mapr" ]; then + echo "For mapr plugin option -v is ignored" + export DIB_MAPR_VERSION=${DIB_MAPR_VERSION:-4.0.1} + + export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES + + export DIB_IMAGE_SIZE=${IMAGE_SIZE:-"10"} + #MapR repository requires additional space + export DIB_MIN_TMPFS=10 + + export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"} + + mapr_ubuntu_elements_sequence="base vm ssh ubuntu hadoop-mapr" + mapr_centos_elements_sequence="base vm rhel ssh hadoop-mapr redhat-lsb selinux-permissive" + + if [ "$DEBUG_MODE" = "true" ]; then + mapr_ubuntu_elements_sequence="$mapr_ubuntu_elements_sequence root-passwd" + mapr_centos_elements_sequence="$mapr_centos_elements_sequence root-passwd" + fi + + if [ -n "$USE_MIRRORS" ]; then + [ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$mapr_ubuntu_elements_sequence apt-mirror" + [ -n "$CENTOS_MIRROR" ] && centos_elements_sequence="$mapr_centos_elements_sequence centos-mirror" + fi + + if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then + export DIB_RELEASE=${DIB_RELEASE:-trusty} + + mapr_ubuntu_image_name=${mapr_ubuntu_image_name:-ubuntu_${DIB_RELEASE}_mapr_${DIB_MAPR_VERSION}_latest} + + disk-image-create $mapr_ubuntu_elements_sequence -n -o $mapr_ubuntu_image_name + mv $mapr_ubuntu_image_name.qcow2 ../ + + unset DIB_RELEASE + fi + + if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then + export BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-"CentOS-6.6-cloud-init-20141118.qcow2"} + export DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-"http://sahara-files.mirantis.com"} + + mapr_centos_image_name=${mapr_centos_image_name:-centos_6.5_mapr_${DIB_MAPR_VERSION}_latest} + + disk-image-create $mapr_centos_elements_sequence -n -o $mapr_centos_image_name + mv $mapr_centos_image_name.qcow2 ../ + + unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES + unset DIB_CLOUD_INIT_DATASOURCES + fi +fi + popd # out of $TEMP rm -rf $TEMP diff --git a/elements/hadoop-mapr/README.rst b/elements/hadoop-mapr/README.rst new file mode 100644 index 00000000..4d948880 --- /dev/null +++ b/elements/hadoop-mapr/README.rst @@ -0,0 +1,21 @@ +Creates images with local mirrors of MapR repositories: `core `_ and `ecosystem `_. +Installs `OpenJDK `_ and `Scala `_. + +In order to create the MapR images with the diskimage-create.sh script, use the following syntax to select the "MapR" plugin: +.. sourcecode:: bash + + sudo bash diskimage-create.sh -p mapr [-i ubuntu|centos] [-r 3.1.1|4.0.1] + +NOTE: By default MapR 4.0.1 version will be used + +In order to speed up image creation process you can download archives with MapR repositories and specify environment variable: +``DIB_MAPR_CORE_DEB_REPO``, ``DIB_MAPR_CORE_RPM_REPO``, ``DIB_MAPR_ECO_DEB_REPO``, ``DIB_MAPR_ECO_RPM_REPO`` + +For example: +.. sourcecode:: bash + + export DIB_MAPR_CORE_DEB_REPO="file:///mapr-v4.0.1GA.deb.tgz" + export DIB_MAPR_CORE_RPM_REPO="file:///mapr-v4.0.1GA.rpm.tgz" + export DIB_MAPR_ECO_DEB_REPO="http:///mapr-ecosystem.deb.tgz" + export DIB_MAPR_ECO_RPM_REPO="http:///mapr-ecosystem.rpm.tgz" + sudo -E bash diskimage-create.sh -p mapr -r 4.0.1 diff --git a/elements/hadoop-mapr/element-deps b/elements/hadoop-mapr/element-deps new file mode 100644 index 00000000..17bc062c --- /dev/null +++ b/elements/hadoop-mapr/element-deps @@ -0,0 +1,3 @@ +java +mysql +disable-firewall diff --git a/elements/hadoop-mapr/install.d/40-mapr-dependencies b/elements/hadoop-mapr/install.d/40-mapr-dependencies new file mode 100755 index 00000000..f63e538c --- /dev/null +++ b/elements/hadoop-mapr/install.d/40-mapr-dependencies @@ -0,0 +1,159 @@ +#!/bin/bash + +set -eux + +echo "START: installing MapR core dependencies" + +DISTRO=$(lsb_release -is || :) + +if [ "$DISTRO" = "Ubuntu" ]; then + # Required for libicu48 + cat >> /etc/apt/sources.list.d/security_repo.list << EOF +deb http://security.ubuntu.com/ubuntu precise-security main +EOF + # Required for libmysqlclient16 + cat >> /etc/apt/sources.list.d/security_repo.list << EOF +deb http://security.ubuntu.com/ubuntu lucid-security main +EOF + apt-get update + install-packages binutils \ + daemon \ + dpkg-dev \ + dpkg-repack \ + gcc \ + gcc-4.8 \ + gcc-doc \ + gcc-multilib \ + iputils-arping \ + libasan0 \ + libatomic1 \ + libc-dev-bin \ + libc6 \ + libc6-dev \ + libcrypt-passwdmd5-perl \ + libevent-1.4-2 \ + libevent-2.0-5 \ + libgcc-4.8-dev \ + libgomp1 \ + libgssglue1 \ + libicu48 \ + libitm1 \ + libmysqlclient-dev \ + libmysqlclient16 \ + libmysqlclient18 \ + libnfsidmap2 \ + libquadmath0 \ + libsasl2-dev \ + libsasl2-modules-gssapi-mit \ + libssl0.9.8 \ + libtirpc1 \ + libtsan0 \ + libxslt1.1 \ + linux-libc-dev \ + manpages-dev \ + mtools \ + mysql-common \ + nfs-common \ + open-iscsi \ + openjdk-6-jre \ + rpcbind \ + sdparm \ + syslinux \ + syslinux-common \ + unzip \ + watchdog \ + zlib1g-dev \ + zip +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + install-packages cdparanoia-libs \ + cups \ + cups-libs \ + createrepo \ + cvs \ + cyrus-sasl-gssapi \ + cyrus-sasl-plain \ + db4-cxx \ + db4-devel \ + foomatic \ + foomatic-db \ + foomatic-db-filesystem \ + foomatic-db-ppds \ + gdbm-devel \ + gettext \ + ghostscript \ + ghostscript-fonts \ + glibc \ + glibc-common \ + glibc-devel \ + glibc-headers \ + gstreamer \ + gstreamer-plugins-base \ + gstreamer-tools \ + hdparm \ + irqbalance \ + iso-codes \ + kernel-headers \ + lcms-libs \ + libXt \ + libXv \ + libXxf86vm \ + libgomp \ + libgssglue \ + libgudev1 \ + libicu \ + libmng \ + liboil \ + libtheora \ + libtirpc \ + libvisual \ + libxslt \ + mesa-dri-drivers \ + mesa-dri-filesystem \ + mesa-dri1-drivers \ + mesa-libGL \ + mesa-libGLU \ + mesa-private-llvm \ + mtools \ + nc \ + numactl \ + openjpeg-libs \ + patch \ + pax \ + perl-CGI \ + perl-ExtUtils-MakeMaker \ + perl-ExtUtils-ParseXS \ + perl-Test-Harness \ + perl-Test-Simple \ + perl-devel \ + phonon-backend-gstreamer \ + poppler \ + poppler-data \ + poppler-utils \ + portreserve \ + qt \ + qt-sqlite \ + qt-x11 \ + qt3 \ + redhat-lsb \ + redhat-lsb-compat \ + redhat-lsb-core \ + redhat-lsb-graphics \ + redhat-lsb-printing \ + rpcbind \ + sdparm \ + soci \ + soci-mysql \ + syslinux \ + syslinux-nonlinux \ + unzip \ + urw-fonts \ + xml-common \ + zip +fi + +if [ "$?" != "0" ]; then + echo "Installation failed. Exit" + exit 1 +fi + +echo "END: installing MapR core dependencies" diff --git a/elements/hadoop-mapr/install.d/41-scala b/elements/hadoop-mapr/install.d/41-scala new file mode 100755 index 00000000..a9f9d7be --- /dev/null +++ b/elements/hadoop-mapr/install.d/41-scala @@ -0,0 +1,39 @@ +#!/bin/bash + +set -eux + +echo "START: installing Scala" + +DISTRO=$(lsb_release -is || :) + +#Current available version +DEF_VERSION="2.11.2" + +RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://www.scala-lang.org/)" + +if [ "$RETURN_CODE" != "200" ]; then + echo "http://www.scala-lang.org is unreachable" + exit 1 +fi + +VERSION="$(curl -s --fail http://www.scala-lang.org|grep 'scala-version'|grep -Eo '([0-9]\.?)+')" + +if [ $? != 0 -o -z ${VERSION} ]; then + echo "Installing default version $DEF_VERSION" + VERSION=${DEF_VERSION} +fi + +PKG=scala-${VERSION} + +URL="http://downloads.typesafe.com/scala/${VERSION}" + +if [ "$DISTRO" = "Ubuntu" ]; then + wget -N ${URL}/${PKG}.deb + dpkg -i ${PKG}.deb + rm ${PKG}.deb +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + rpm -Uhv ${URL}/${PKG}.rpm +fi + +echo "END: installing Scala" + diff --git a/elements/hadoop-mapr/install.d/42-versions b/elements/hadoop-mapr/install.d/42-versions new file mode 100755 index 00000000..93e066a3 --- /dev/null +++ b/elements/hadoop-mapr/install.d/42-versions @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eux + +install -D -g root -o root -m 0644 $(dirname $0)/versions.py /tmp/versions.py diff --git a/elements/hadoop-mapr/install.d/43-mapr-core-repository b/elements/hadoop-mapr/install.d/43-mapr-core-repository new file mode 100755 index 00000000..df224849 --- /dev/null +++ b/elements/hadoop-mapr/install.d/43-mapr-core-repository @@ -0,0 +1,63 @@ +#!/bin/bash + +set -eux + +echo "START: installing MapR core repository" + +install-packages curl + +MAPR_REPO_DIR=/opt/mapr-repository/core + +DISTRO=$(lsb_release -is || :) + +if [ "$DISTRO" = "Ubuntu" ]; then + MAPR_REPO_URL=${DIB_MAPR_CORE_DEB_REPO:-"http://package.mapr.com/releases/v"${DIB_MAPR_VERSION}"/ubuntu/mapr-v"${DIB_MAPR_VERSION}"GA.deb.tgz"} +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + MAPR_REPO_URL=${DIB_MAPR_CORE_RPM_REPO:-"http://package.mapr.com/releases/v"${DIB_MAPR_VERSION}"/redhat/mapr-v"${DIB_MAPR_VERSION}"GA.rpm.tgz"} +else + echo "Unknown distribution" + exit 1 +fi + +echo "Downloading MapR repository archive" +mkdir -p ${MAPR_REPO_DIR} +sudo curl ${MAPR_REPO_URL} -o ${MAPR_REPO_DIR}/mapr-v"${DIB_MAPR_VERSION}"GA.tgz + +if [ $? != 0 ]; then + echo "MapR repository archive download failed" + exit 1 +fi + +echo "Unpacking MapR repository archive" +tar -zxf ${MAPR_REPO_DIR}/mapr-v${DIB_MAPR_VERSION}GA.tgz -C ${MAPR_REPO_DIR} + +echo "Removing MapR repository archive" +rm -f ${MAPR_REPO_DIR}/mapr-v${DIB_MAPR_VERSION}GA.tgz + +if [ "$DISTRO" = "Ubuntu" ]; then + mkdir -p "${MAPR_REPO_DIR}"/dists/binary/optional/binary-amd64 + cd ${MAPR_REPO_DIR} + dpkg-scanpackages -m . /dev/null | gzip -9c > dists/binary/optional/binary-amd64/Packages.gz +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + createrepo "${MAPR_REPO_DIR}" +fi + +echo "Adding MapR repository" +if [ "$DISTRO" = "Ubuntu" ]; then + cat >> /etc/apt/sources.list.d/maprtech.list << EOF +deb [arch=amd64] file://${MAPR_REPO_DIR} binary optional +EOF + +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + cat >> /etc/yum.repos.d/maprtech.repo << EOF +[mapr-core] +name=MapR Technologies +baseurl=file://${MAPR_REPO_DIR} +enabled=1 +gpgcheck=0 +protect=1 +EOF + +fi + +echo "END: installing MapR core repository" diff --git a/elements/hadoop-mapr/install.d/44-mapr-eco-repository b/elements/hadoop-mapr/install.d/44-mapr-eco-repository new file mode 100755 index 00000000..f8680b94 --- /dev/null +++ b/elements/hadoop-mapr/install.d/44-mapr-eco-repository @@ -0,0 +1,87 @@ +#!/bin/bash + +set -eux + +echo "START: installing MapR ecosystem repository" + +MAPR_REPO_DIR=/opt/mapr-repository/ecosystem + +DISTRO=$(lsb_release -is || :) + +RPM_REPO=${DIB_MAPR_ECO_RPM_REPO:-} +DEB_REPO=${DIB_MAPR_ECO_DEB_REPO:-} + +if [ "$DISTRO" = "Ubuntu" ]; then + if [ "$DIB_MAPR_VERSION" = "3.1.1" ]; then + MAPR_REPO_URL=${DIB_MAPR_ECO_DEB_REPO:-"http://package.mapr.com/releases/ecosystem/ubuntu/"} + elif [ "$DIB_MAPR_VERSION" = "4.0.1" ]; then + MAPR_REPO_URL=${DIB_MAPR_ECO_DEB_REPO:-"http://package.mapr.com/releases/ecosystem-4.x/ubuntu/"} + fi +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + if [ "$DIB_MAPR_VERSION" = "3.1.1" ]; then + MAPR_REPO_URL=${DIB_MAPR_ECO_RPM_REPO:-"http://package.mapr.com/releases/ecosystem/redhat/"} + elif [ "$DIB_MAPR_VERSION" = "4.0.1" ]; then + MAPR_REPO_URL=${DIB_MAPR_ECO_RPM_REPO:-"http://package.mapr.com/releases/ecosystem-4.x/redhat/"} + fi +else + echo "Unknown distribution" + exit 1 +fi + +echo "Downloading MapR repository archive" +if [ -z ${DEB_REPO} -o -z ${RPM_REPO} ]; then + LATEST_ARCHIVE=$(curl -s ${MAPR_REPO_URL} | grep -Po '(?<=)' | uniq | sort | tail -n 1) + if [ -z ${LATEST_ARCHIVE} ]; then + echo "No repository archive found" + exit 1 + fi + MAPR_REPO_URL=${MAPR_REPO_URL}/${LATEST_ARCHIVE} +fi + +mkdir -p ${MAPR_REPO_DIR} + +sudo curl ${MAPR_REPO_URL} -o ${MAPR_REPO_DIR}/mapr-ecosystem.tgz + +if [ $? != 0 ]; then + echo "MapR repository archive download failed" + exit 1 +fi + +echo "Unpacking MapR repository archive" +tar -zxf ${MAPR_REPO_DIR}/mapr-ecosystem.tgz -C ${MAPR_REPO_DIR} + +echo "Removing MapR repository archive" +rm -f ${MAPR_REPO_DIR}/mapr-ecosystem.tgz + +echo "Removing old packages" +SCRIPT_FILE="/tmp/versions.py" +SPEC_FILE="/opt/mapr-repository/version-spec" +python ${SCRIPT_FILE} --path ${MAPR_REPO_DIR} --spec-file ${SPEC_FILE} | xargs sudo rm -f || echo "Operation failed" + +if [ "$DISTRO" = "Ubuntu" ]; then + mkdir -p "${MAPR_REPO_DIR}"/dists/binary/optional/binary-amd64 + cd ${MAPR_REPO_DIR} + dpkg-scanpackages -m . /dev/null | gzip -9c > dists/binary/optional/binary-amd64/Packages.gz +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + createrepo "${MAPR_REPO_DIR}" +fi + +echo "Adding MapR repository" +if [ "$DISTRO" = "Ubuntu" ]; then + cat >> /etc/apt/sources.list.d/maprtech.list << EOF +deb [arch=amd64] file://${MAPR_REPO_DIR} binary optional +EOF + +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + cat >> /etc/yum.repos.d/maprtech.repo << EOF +[mapr-ecosystem] +name=MapR Technologies +baseurl=file://${MAPR_REPO_DIR} +enabled=1 +gpgcheck=0 +protect=1 +EOF +fi + +echo "END: installing MapR ecosystem repository" + diff --git a/elements/hadoop-mapr/install.d/44-oozie-libs b/elements/hadoop-mapr/install.d/44-oozie-libs new file mode 100755 index 00000000..a9619ddd --- /dev/null +++ b/elements/hadoop-mapr/install.d/44-oozie-libs @@ -0,0 +1,15 @@ +#!/bin/bash + +set -eux + +echo "START: downloading ExtJS library" + +install-packages wget + +MAPR_REPO_DIR="/opt/mapr-repository/" +EXT_JS_URL="http://extjs.com/deploy/ext-2.2.zip" + +mkdir -p ${MAPR_REPO_DIR} +sudo wget ${EXT_JS_URL} -P ${MAPR_REPO_DIR} + +echo "END: downloading ExtJS library" diff --git a/elements/hadoop-mapr/install.d/versions.py b/elements/hadoop-mapr/install.d/versions.py new file mode 100644 index 00000000..a7461957 --- /dev/null +++ b/elements/hadoop-mapr/install.d/versions.py @@ -0,0 +1,102 @@ +# Copyright (c) 2014, MapR Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import collections as c +import itertools as _iter +import operator as op +import os +import re +import sys + + +spec = { + 'asynchbase': 1, 'cascading': 1, 'flume': 1, 'hbase': 1, + 'hbasethrift': 1, 'hcatalog': 1, 'hive': 2, 'hivemetastore': 2, + 'hiveserver2': 2, 'httpfs': 1, 'hue': 1, 'impala': 1, 'mahout': 1, + 'oozie': 2, 'pig': 1, 'spark': 1, 'sqoop': 0, 'sqoop2': 1, 'whirr': 1 +} + +reg_expr = ur"mapr-([a-z]+\d?)[-_]([a-z]+)?[-_]?(\d+).(\d+).(\d+).(\d+)?" +pattern = re.compile(reg_expr) + +file_map = c.defaultdict(list) +version_map = c.defaultdict(list) +comp_map = c.defaultdict(tuple) + + +def parse_filename(args): + def mapper(arg): + try: + new_int = int(arg) + return new_int + except ValueError: + return arg + except TypeError: + pass + + def predicate(_type): + return lambda x: isinstance(x, _type) + + arg_list = filter(lambda x: op.is_not(x, None), + map(mapper, args)) + + pair = (tuple(filter(predicate(str), arg_list)), + filter(predicate(int), arg_list)) + + return pair + + +def main(arg_list): + if os.path.exists(arg_list.path): + for f_name in os.listdir(arg_list.path): + match = pattern.match(f_name) + if match: + file_map[f_name] = parse_filename(match.groups()) + + for k, g in _iter.groupby(file_map.values(), + key=lambda i: op.getitem(i, 0)): + for elem in g: + version_map[k].append(op.getitem(elem, 1)) + + for (k, v) in version_map.items(): + comp_map[k] = sorted(v, reverse=True)[:spec.get(k[0], 1)] + + version_spec = set() + + with open(arg_list.spec_file, mode='w') as f: + not_build_version = lambda a: len(str(a)) < 3 + for k, v in comp_map.items(): + join_versions = lambda seq: '.'.join( + str(el) for el in filter(not_build_version, seq)) + v_str = ','.join(map(join_versions, v)) + version_spec.add('{comp:s} = {versions:s}\n' + .format(**{'comp': k[0], 'versions': v_str})) + f.writelines(version_spec) + + for (f_name, comp) in file_map.items(): + if not comp[1] in comp_map[comp[0]]: + target = os.path.join(arg_list.path, f_name) + sys.stdout.write(str(target).strip() + '\n') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--path', help='path to repository files') + parser.add_argument('--spec-file', help='path to specification files') + args = parser.parse_args() + if not args.path or not args.spec_file: + parser.print_help() + exit(-1) + main(args) diff --git a/elements/hadoop-mapr/post-install.d/99-java-home b/elements/hadoop-mapr/post-install.d/99-java-home new file mode 100755 index 00000000..4eefc6e6 --- /dev/null +++ b/elements/hadoop-mapr/post-install.d/99-java-home @@ -0,0 +1,6 @@ +#!/bin/bash + +set -eux + +echo "/etc/profile.d/java.sh" >> /etc/ssh/sshrc +chmod 775 /etc/profile.d/java.sh diff --git a/elements/hadoop-mapr/post-install.d/99-requiretty b/elements/hadoop-mapr/post-install.d/99-requiretty new file mode 100755 index 00000000..a4fb5bcf --- /dev/null +++ b/elements/hadoop-mapr/post-install.d/99-requiretty @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eux + +DISTRO=$(lsb_release -is || :) + +if [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + sed '/^Defaults requiretty*/ s/^/#/' -i /etc/sudoers +fi + diff --git a/elements/hadoop-mapr/post-install.d/99-update-repository b/elements/hadoop-mapr/post-install.d/99-update-repository new file mode 100755 index 00000000..be6abefc --- /dev/null +++ b/elements/hadoop-mapr/post-install.d/99-update-repository @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eux + +DISTRO=$(lsb_release -is || :) + +if [ "$DISTRO" = "Ubuntu" ]; then + apt-get update +elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then + yum clean all && yum repolist +fi diff --git a/elements/hadoop-mapr/root.d/0-mapr-check-version b/elements/hadoop-mapr/root.d/0-mapr-check-version new file mode 100755 index 00000000..38817505 --- /dev/null +++ b/elements/hadoop-mapr/root.d/0-mapr-check-version @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eux + +echo "MapR version ${DIB_MAPR_VERSION}" + +if [ -z ${DIB_MAPR_VERSION} ]; then + echo "DIB_MAPR_VERSION not set.\nExit" + exit 1 +fi diff --git a/tox.ini b/tox.ini index 0c291c80..2210b465 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ downloadcache = ~/cache/pip [testenv:bashate] whitelist_externals = bash -commands = bash -c "find {toxinidir} -not -wholename \*.tox/\* -and \( -name \*.sh -or -wholename \*.d/\* -and -not -name \*.md -and -not -name \*.rst \) -print0 | xargs -0 bashate -v" +commands = bash -c "find {toxinidir} -not -wholename \*.tox/\* -and \( -name \*.sh -or -wholename \*.d/\* -and -not -name \*.md -and -not -name \*.rst -and -not -name \*.py \) -print0 | xargs -0 bashate -v" [testenv:pep8] commands = flake8 {posargs}