Merge "Image generation for MapR"

This commit is contained in:
Zuul 2018-01-21 18:15:24 +00:00 committed by Gerrit Code Review
commit 2119477466
26 changed files with 1209 additions and 0 deletions

View File

@ -22,6 +22,7 @@ import sahara.plugins.mapr.base.base_cluster_validator as bv
import sahara.plugins.mapr.base.base_edp_engine as edp
import sahara.plugins.mapr.base.base_health_checker as health
import sahara.plugins.mapr.base.base_node_manager as bs
from sahara.plugins.mapr import images
import sahara.plugins.mapr.util.general as util
import sahara.plugins.provisioning as p
import sahara.plugins.utils as u
@ -38,6 +39,7 @@ class BaseVersionHandler(vh.AbstractVersionHandler):
self._services = []
self._node_processes = {}
self._configs = []
self.images = images
def get_edp_engine(self, cluster, job_type):
if job_type in edp.MapROozieJobEngine.get_supported_job_types():
@ -177,3 +179,20 @@ class BaseVersionHandler(vh.AbstractVersionHandler):
def get_cluster_checks(self, cluster):
cluster_context = self.get_context(cluster)
return self._health_checker.get_checks(cluster_context)
def get_image_arguments(self):
if hasattr(self, 'images'):
return self.images.get_image_arguments()
else:
return NotImplemented
def pack_image(self, hadoop_version, remote, test_only=False,
image_arguments=None):
if hasattr(self, 'images'):
self.images.pack_image(
remote, test_only=test_only, image_arguments=image_arguments)
def validate_images(self, cluster, test_only=False, image_arguments=None):
if hasattr(self, 'images'):
self.images.validate_images(
cluster, test_only=test_only, image_arguments=image_arguments)

View File

@ -0,0 +1,43 @@
# Copyright (c) 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins import images
from sahara.plugins import utils as plugin_utils
_validator = images.SaharaImageValidator.from_yaml(
'plugins/mapr/resources/images/image.yaml',
resource_roots=['plugins/mapr/resources/images'])
def get_image_arguments():
return _validator.get_argument_list()
def pack_image(remote, test_only=False, image_arguments=None):
_validator.validate(remote, test_only=test_only,
image_arguments=image_arguments)
def validate_images(cluster, test_only=False, image_arguments=None):
image_arguments = get_image_arguments()
if not test_only:
instances = plugin_utils.get_instances(cluster)
else:
instances = plugin_utils.get_instances(cluster)[0]
for instance in instances:
with instance.remote() as r:
_validator.validate(r, test_only=test_only,
image_arguments=image_arguments)

View File

@ -97,3 +97,16 @@ class MapRPlugin(p.ProvisioningPluginBase):
def get_health_checks(self, cluster):
v_handler = self._get_handler(cluster.hadoop_version)
return v_handler.get_cluster_checks(cluster)
def get_image_arguments(self, hadoop_version):
return self._get_handler(hadoop_version).get_image_arguments()
def pack_image(self, hadoop_version, remote,
test_only=False, image_arguments=None):
version = self._get_handler(hadoop_version)
version.pack_image(hadoop_version, remote, test_only=test_only,
image_arguments=image_arguments)
def validate_images(self, cluster, test_only=False, image_arguments=None):
self._get_handler(cluster.hadoop_version).validate_images(
cluster, test_only=test_only, image_arguments=image_arguments)

View File

@ -0,0 +1,20 @@
#!/bin/bash
check=$(systemctl --no-pager list-unit-files iptables.service | grep 'enabled' | wc -l)
if [ $check -eq 1 ]; then
if [ $test_only -eq 0 ]; then
if type -p systemctl && [[ "$(systemctl --no-pager list-unit-files firewalld)" =~ 'enabled' ]]; then
systemctl disable firewalld
fi
if type -p service; then
service ip6tables save
service iptables save
chkconfig ip6tables off
chkconfig iptables off
fi
else
exit 0
fi
fi

View File

@ -0,0 +1,27 @@
#!/bin/bash
DISTRO_NAME=$distro
source "/tmp/package_utils.sh"
echo "START: installing MapR core repository"
MAPR_REPO_URL="http://package.mapr.com/releases/v${plugin_version}/redhat/mapr-v${plugin_version}GA.rpm.tgz"
MAPR_REPO_DIR="/opt/mapr-repository/core"
if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then
if [ $test_only -eq 0 ]; then
MAPR_REPO_NAME="mapr_core"
echo "Downloading MapR repository archive"
mkdir -p "$MAPR_REPO_DIR" && curl "$MAPR_REPO_URL" | tar -xz -C "$MAPR_REPO_DIR"
echo "Creating local repository"
create_repo "$MAPR_REPO_DIR"
echo "Adding MapR repository"
add_local_repo "$MAPR_REPO_NAME" "$MAPR_REPO_DIR"
fi
fi
echo "END: installing MapR core repository"

View File

@ -0,0 +1,27 @@
#!/bin/bash
VERSIONS_PY="/tmp/versions.py"
DISTRO_NAME=$distro
source "/tmp/package_utils.sh"
echo "START: installing MapR ecosystem repository"
MAPR_REPO_URL="http://package.mapr.com/releases/MEP/MEP-2.0.0/redhat/"
MAPR_REPO_DIR="/opt/mapr-repository/ecosystem"
if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then
if [ $test_only -eq 0 ]; then
MAPR_REPO_NAME="mapr_ecosystem"
MAPR_PKG_GROUPS="/tmp/packages.json"
MAPR_SPEC="/tmp/spec_$plugin_version.json"
echo "Creating local MapR ecosystem repository"
localize_repo "$MAPR_REPO_NAME" "$MAPR_REPO_URL" "$MAPR_PKG_GROUPS" "$MAPR_SPEC" "$MAPR_REPO_DIR"
echo $MAPR_SPEC
fi
fi
echo "END: installing MapR ecosystem repository"

View File

@ -0,0 +1,14 @@
#!/bin/bash
echo "Installing OpenJDK"
if [ $test_only -eq 0 ]; then
yum install -y java-1.8.0-openjdk-devel
JRE_HOME="/usr/lib/jvm/java-openjdk/jre"
JDK_HOME="/usr/lib/jvm/java-openjdk"
echo "OpenJDK has been installed"
else
exit 0
fi

View File

@ -0,0 +1,34 @@
#!/bin/bash
echo "START: installing Scala"
sudo yum -y update
exit 0
if [ $test_only -eq 0 ]; then
RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://www.scala-lang.org/)"
if [ "$RETURN_CODE" != "200" ]; then
echo "http://www.scala-lang.org is unreachable" && exit 1
fi
if [ -n "${scala_version:-}" ]; then
VERSION=$scala_version
else
VERSION="$(curl -s --fail http://www.scala-lang.org| tr -d '\n' | sed 's/^.*<div[^<]\+scala-version">[^0-9]\+\([0-9\.\?]\+\)<.\+$/\1/')"
if [ $? != 0 -o -z "${VERSION}" ]; then
echo "Installing default version $scala_version"
VERSION=$scala_version
fi
fi
PKG=scala-${VERSION}
URL="http://downloads.lightbend.com/scala/${VERSION}"
rpm -Uhv ${URL}/${PKG}.rpm
fi
echo "END: installing Scala"

View File

@ -0,0 +1,6 @@
#!/bin/bash
if [ $test_only -eq 0 ]; then
sed '/^Defaults requiretty*/ s/^/#/' -i /etc/sudoers
fi

View File

@ -0,0 +1,12 @@
#!/bin/bash
check=$(cat /etc/selinux/config | grep "SELINUX=permissive" | wc -l)
if [ $check -eq 0 ]; then
if [ $test_only -eq 0 ]; then
echo "SELINUX=permissive" > /etc/selinux/config
echo "SELINUXTYPE=targeted" >> /etc/selinux/config
else
exit 0
fi
fi

View File

@ -0,0 +1,5 @@
#!/bin/bash
if [ $test_only -eq 0 ]; then
yum clean all && yum repolist
fi

View File

@ -0,0 +1,23 @@
#!/bin/bash
EXTJS_DESTINATION_DIR="/opt/mapr-repository"
EXTJS_DOWNLOAD_URL="http://tarballs.openstack.org/sahara/dist/common-artifacts/ext-2.2.zip"
EXTJS_NO_UNPACK=1
extjs_archive=/tmp/$(basename $EXTJS_DOWNLOAD_URL)
if [ ! -f "${EXTJS_DESTINATION_DIR}/${extjs_archive}" ]; then
if [ $test_only -eq 0 ]; then
wget -O $extjs_archive $EXTJS_DOWNLOAD_URL
mkdir -p $EXTJS_DESTINATION_DIR
if [ -z "${EXTJS_NO_UNPACK:-}" ]; then
unzip -d "$EXTJS_DESTINATION_DIR" $extjs_archive
rm -f $extjs_archive
else
mv $extjs_archive $EXTJS_DESTINATION_DIR
fi
else
exit 0
fi
fi

View File

@ -0,0 +1,42 @@
#!/bin/bash
# NOTE: $(dirname $0) is read-only, use space under $TARGET_ROOT
JAVA_LOCATION=${JAVA_TARGET_LOCATION:-"/usr/java"}
JAVA_NAME="oracle-jdk"
JAVA_HOME=$JAVA_LOCATION/$JAVA_NAME
JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"}
# FIXME: probably not idempotent, find a better condition
if [ ! -d $JAVA_LOCATION ]; then
if [ $test_only -eq 0 ]; then
echo "Begin: installation of Java"
mkdir -p $JAVA_LOCATION
if [ -n "$JAVA_DOWNLOAD_URL" ]; then
JAVA_FILE=$(basename $JAVA_DOWNLOAD_URL)
wget --no-check-certificate --no-cookies -c \
--header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \
-O $JAVA_LOCATION/$JAVA_FILE $JAVA_DOWNLOAD_URL
elif [ -n "$JAVA_FILE" ]; then
install -D -g root -o root -m 0755 $(dirname $0)/$JAVA_FILE $JAVA_LOCATION
fi
cd $JAVA_LOCATION
echo "Decompressing Java archive"
echo -e "\n" | tar -zxf $JAVA_FILE
echo "Setting up $JAVA_NAME"
chown -R root:root $JAVA_LOCATION
JAVA_DIR=`ls -1 $JAVA_LOCATION | grep -v tar.gz`
ln -s $JAVA_LOCATION/$JAVA_DIR $JAVA_HOME
setup-java-home $JAVA_HOME $JAVA_HOME
rm $JAVA_FILE
echo "End: installation of Java"
else
exit 0
fi
fi

View File

@ -0,0 +1,226 @@
# execute_in_directory <directory> <command>
execute_in_directory() {
local directory="$(readlink -f "$1")"; shift
local cmd="$*"
pushd "$directory" && eval "$cmd" && popd
}
# get_distro
get_distro() {
echo "$DISTRO_NAME"
}
# download_apt_package <package> [version] [directory]
download_apt_package() {
local package="$1"
local version="${2:-}"
local directory="${3:-$(pwd)}"
local package_spec="$package${version:+=$version*}"
execute_in_directory "$directory" apt-get --allow-unauthenticated download "$package_spec"
}
# download_yum_package <package> [version] [directory]
download_yum_package() {
local package="$1"
local version="${2:-}"
local directory="${3:-$(pwd)}"
local package_spec="$package${version:+-$version*}"
yumdownloader --destdir "$directory" "$package_spec"
}
# download_package <package> [version] [directory] [distro]
download_package() {
local package="$1"
local version="${2:-}"
local directory="${3:-$(pwd)}"
local distro="${4:-$(get_distro)}"
if [[ "$distro" == "ubuntu" ]]; then
download_apt_package "$package" "$version" "$directory"
elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
download_yum_package "$package" "$version" "$directory"
fi
}
# get_packages <package_groups_file> <spec_file> [version_separator]
get_packages() {
local package_groups_file="$1"
local spec_file="$2"
local version_separator="${3:-:}"
python "$VERSIONS_PY" --separator "$version_separator" "$package_groups_file" "$spec_file"
}
# download_packages <package_groups_file> <spec_file> [directory] [distro]
download_packages() {
local package_groups_file="$1"
local spec_file="$2"
local directory="${3:-$(pwd)}"
local distro="${4:-$(get_distro)}"
local version_separator=":"
local packages="$(get_packages "$package_groups_file" "$spec_file" "$version_separator")"
for package in $packages; do
IFS="$version_separator" read -ra package_version <<< "$package"
download_package "${package_version[@]}" "$directory" "$distro"
done
}
# create_apt_repo <directory>
create_apt_repo() {
local directory="$(readlink -f "$1")"
local binary_dir="$directory/binary"
local packages_gz="$binary_dir/Packages.gz"
mkdir -p "$binary_dir"
execute_in_directory "$directory" "dpkg-scanpackages -m . /dev/null | gzip -9c > $packages_gz"
}
# create_yum_repo <directory>
create_yum_repo() {
local directory="$(readlink -f "$1")"
createrepo "$directory"
}
# create_repo <directory> [distro]
create_repo() {
local directory="$(readlink -f "$1")"
local distro="${2:-$(get_distro)}"
if [[ "$distro" == "ubuntu" ]]; then
create_apt_repo "$directory"
elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
create_yum_repo "$directory"
fi
}
# add_apt_repo <repo_name> <repo_url>
add_apt_repo() {
local repo_name="$1"
local repo_url="$2"
local repo="deb $repo_url"
local repo_path="/etc/apt/sources.list.d/$repo_name.list"
echo "$repo" > "$repo_path" && apt-get update
}
# add_yum_repo <repo_name> <repo_url>
add_yum_repo() {
local repo_name="$1"
local repo_url="$2"
local repo_path="/etc/yum.repos.d/$repo_name.repo"
cat > "$repo_path" << EOF
[$repo_name]
name=$repo_name
baseurl=$repo_url
enabled=1
gpgcheck=0
protect=1
EOF
yum clean all && rm -rf /var/cache/yum/* && yum check-update
}
# add_repo <repo_name> <repo_url> [distro]
add_repo() {
local repo_name="$1"
local repo_url="$2"
local distro="${3:-$(get_distro)}"
if [[ "$distro" == "ubuntu" ]]; then
add_apt_repo "$repo_name" "$repo_url"
elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
add_yum_repo "$repo_name" "$repo_url"
fi
}
# add_local_apt_repo <repo_name> <directory>
add_local_apt_repo() {
local repo_name="$1"
local directory="$(readlink -f "$2")"
local repo_url="file:$directory binary/"
add_apt_repo "$repo_name" "$repo_url"
}
# add_local_yum_repo <repo_name> <directory>
add_local_yum_repo() {
local repo_name="$1"
local directory="$(readlink -f "$2")"
local repo_url="file://$directory"
add_yum_repo "$repo_name" "$repo_url"
}
# add_local_repo <repo_name> <directory> [distro]
add_local_repo() {
local repo_name="$1"
local directory="$(readlink -f "$2")"
local distro="${3:-$(get_distro)}"
if [[ "$distro" == "ubuntu" ]]; then
add_local_apt_repo "$repo_name" "$directory"
elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
add_local_yum_repo "$repo_name" "$directory"
fi
}
# remove_apt_repo <repo_name>
remove_apt_repo() {
local repo_name="$1"
local repo_path="/etc/apt/sources.list.d/$repo_name.list"
rm "$repo_path" && apt-get update
}
# remove_yum_repo <repo_name>
remove_yum_repo() {
local repo_name="$1"
local repo_path="/etc/yum.repos.d/$repo_name.repo"
rm "$repo_path"
}
# remove_repo <repo_name> [distro]
remove_repo() {
local repo_name="$1"
local distro="${2:-$(get_distro)}"
if [[ "$distro" == "ubuntu" ]]; then
remove_apt_repo "$repo_name"
elif [[ "$distro" == "centos" || "$distro" == "centos7" || "$distro" == "rhel" || "$distro" == "rhel7" ]]; then
remove_yum_repo "$repo_name"
fi
}
# create_local_repo <repo_name> <repo_url> <package_groups_file> <spec_file> <directory>
create_local_repo() {
local repo_name="$1"
local repo_url="$2"
local package_groups_file="$3"
local spec_file="$4"
local directory="$5"
add_repo "$repo_name" "$repo_url"
mkdir -p "$directory" && directory="$(readlink -f "$directory")"
download_packages "$package_groups_file" "$spec_file" "$directory"
remove_repo "$repo_name"
create_repo "$directory"
}
# localize_repo <repo_name> <repo_url> <package_groups_file> <spec_file> <directory>
localize_repo() {
local repo_name="$1"
local repo_url="$2"
local package_groups_file="$3"
local spec_file="$4"
local directory="$5"
mkdir -p "$directory" && directory="$(readlink -f "$directory")"
create_local_repo "$repo_name" "$repo_url" "$package_groups_file" "$spec_file" "$directory"
add_local_repo "$repo_name" "$directory"
}

View File

@ -0,0 +1,140 @@
{
"asynchbase": {
"all": [
"mapr-asynchbase"
]
},
"drill": {
"all": [
"mapr-drill"
]
},
"flume": {
"all": [
"mapr-flume"
]
},
"hbase": {
"all": [
"mapr-hbase",
"mapr-hbase-internal",
"mapr-hbase-master",
"mapr-hbase-regionserver",
"mapr-hbasethrift",
"mapr-hbase-rest"
],
"0.98.12": [
"mapr-hbase",
"mapr-hbase-internal",
"mapr-hbase-master",
"mapr-hbase-regionserver",
"mapr-hbasethrift",
"mapr-libhbase",
"mapr-hbase-rest"
],
"1.1.1": [
"mapr-hbase",
"mapr-hbase-internal",
"mapr-hbase-master",
"mapr-hbase-regionserver",
"mapr-hbasethrift",
"mapr-libhbase",
"mapr-hbase-rest"
]
},
"hive": {
"all": [
"mapr-hive",
"mapr-hivemetastore",
"mapr-hiveserver2"
]
},
"httpfs": {
"all": [
"mapr-httpfs"
]
},
"hue": {
"all": [
"mapr-hue",
"mapr-hue-base",
"mapr-hue-livy"
],
"3.10.0": [
"mapr-hue",
"mapr-hue-livy"
]
},
"impala": {
"all": [
"mapr-impala",
"mapr-impala-catalog",
"mapr-impala-server",
"mapr-impala-statestore",
"mapr-impala-udf"
]
},
"mahout": {
"all": [
"mapr-mahout"
]
},
"oozie": {
"all": [
"mapr-oozie",
"mapr-oozie-internal"
]
},
"pig": {
"all": [
"mapr-pig"
]
},
"sentry": {
"all": [
"mapr-sentry"
]
},
"spark": {
"all": [
"mapr-spark",
"mapr-spark-historyserver",
"mapr-spark-master"
]
},
"sqoop": {
"all": [
"mapr-sqoop2-client",
"mapr-sqoop2-server"
]
},
"storm": {
"all": [
"mapr-storm",
"mapr-storm-ui",
"mapr-storm-nimbus",
"mapr-storm-supervisor"
]
},
"tez": {
"all": [
"mapr-tez"
]
},
"kafka": {
"all": [
"mapr-kafka"
]
},
"kafka-connect": {
"all": [
"mapr-kafka-connect-hdfs",
"mapr-kafka-connect-jdbc"
]
},
"kafka-rest": {
"all": [
"mapr-kafka-rest"
]
}
}

View File

@ -0,0 +1,46 @@
{
"drill": [
"1.1.0",
"1.2.0",
"1.4.0"
],
"flume": [
"1.5.0",
"1.6.0"
],
"hbase": [
"0.98.9",
"0.98.12"
],
"hive": [
"0.13",
"1.0",
"1.2"
],
"httpfs": [
"1.0"
],
"hue": [
"3.8.1",
"3.9.0"
],
"impala": [
"1.4.1"
],
"mahout": [
"0.10.0"
],
"oozie": [
"4.2.0"
],
"pig": [
"0.14",
"0.15"
],
"sqoop": [
"2.0.0"
],
"spark": [
"1.5.2"
]
}

View File

@ -0,0 +1,50 @@
{
"drill": [
"1.9.0"
],
"flume": [
"1.6.0"
],
"hbase": [
"1.1.1"
],
"hive": [
"1.2"
],
"httpfs": [
"1.0"
],
"hue": [
"3.10.0"
],
"impala": [
"2.5.0"
],
"mahout": [
"0.12.0"
],
"oozie": [
"4.2.0"
],
"pig": [
"0.16"
],
"sqoop": [
"2.0.0"
],
"spark": [
"2.0.1"
],
"sentry": [
"1.6.0"
],
"kafka": [
"0.9.0"
],
"kafka-connect": [
"2.0.1"
],
"kafka-rest": [
"2.0.1"
]
}

View File

@ -0,0 +1,47 @@
{
"drill": [
"1.9.0"
],
"flume": [
"1.6.0"
],
"hbase": [
"1.1.1"
],
"hive": [
"1.2"
],
"httpfs": [
"1.0"
],
"hue": [
"3.10.0"
],
"mahout": [
"0.12.0"
],
"oozie": [
"4.2.0"
],
"pig": [
"0.16"
],
"sqoop": [
"2.0.0"
],
"spark": [
"2.0.1"
],
"sentry": [
"1.6.0"
],
"kafka": [
"0.9.0"
],
"kafka-connect": [
"2.0.1"
],
"kafka-rest": [
"2.0.1"
]
}

View File

@ -0,0 +1,83 @@
# Copyright (c) 2015, MapR Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import sys
from oslo_serialization import jsonutils as json
_GROUP_VERSION_SEPARATOR = ","
_ALL_GROUP_VERSION = "all"
def _build_parser():
parser = argparse.ArgumentParser()
parser.add_argument("packages", help="path to the packages.json")
parser.add_argument("spec", help="path to the spec.json")
parser.add_argument("--separator", default=":",
help="separator between package name"
" and version in output")
return parser
def _load_json(path):
with open(path) as json_file:
return json.load(json_file)
def _version_matches(version, group_version):
for gv in group_version.split(_GROUP_VERSION_SEPARATOR):
if version.startswith(gv):
return True
return False
def _get_packages(version, group_spec):
for group_version in group_spec:
if _version_matches(version, group_version):
return group_spec[group_version]
return group_spec[_ALL_GROUP_VERSION]
def _get_package_versions(spec, package_groups):
return [(package, version)
for pg_name, versions in spec.items()
for version in versions
for package in _get_packages(version, package_groups[pg_name])]
parser = _build_parser()
def main(args=None):
args = parser.parse_args(args or sys.argv[1:])
spec = _load_json(args.spec)
package_groups = _load_json(args.packages)
separator = args.separator
package_versions = _get_package_versions(spec, package_groups)
package_format = "%s" + separator + "%s\n"
package_versions = map(lambda pv: package_format % pv, package_versions)
sys.stdout.writelines(package_versions)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,195 @@
arguments:
java_distro:
default: openjdk
description: The distribution of Java to install. Defaults to cloudera-jdk.
choices:
- openjdk
- oracle-java
plugin_version:
default: 5.2.0
description: The distribution of MapR to install. Defaults to 5.2.0.
hidden: True
required: False
scala_version:
default: 2.11.6
description: The version of scala to install. Defaults to 2.11.6.
hidden: True
required: False
hdfs_lib_dir:
default: /usr/lib/hadoop-mapreduce
description: The path to HDFS_LIB_DIR. Default to /usr/lib/hadoop-mapreduce
required: False
validators:
- os_case:
- ubuntu:
- script: ubuntu/install_mapr_dependencies
- package:
- mtools
- rpcbind
- sdparm
- syslinux
- unzip
- wget
- zip
- os_case:
- centos:
- package:
- cups
- cdparanoia-libs
- cups-libs
- createrepo
- cvs
- cyrus-sasl-gssapi
- cyrus-sasl-plain
- foomatic
- foomatic-db
- foomatic-db-filesystem
- foomatic-db-ppds
- gdbm-devel
- gettext
- ghostscript
- ghostscript-fonts
- glibc
- glibc-common
- glibc-devel
- glibc-headers
- gstreamer
- gstreamer-plugins-base
- gstreamer-tools
- hdparm
- irqbalance
- iso-codes
- kernel-headers
- libXt
- libXv
- libXxf86vm
- libgomp
- libgudev1
- libicu
- libmng
- liboil
- libtheora
- libtirpc
- libvisual
- libxslt
- mesa-dri-drivers
- mesa-libGL
- mesa-libGLU
- mesa-private-llvm
- nmap-ncat
- numactl
- openjpeg-libs
- patch
- pax
- perl-CGI
- perl-ExtUtils-MakeMaker
- perl-ExtUtils-ParseXS
- perl-Test-Harness
- perl-Test-Simple
- perl-devel
- phonon-backend-gstreamer
- poppler
- poppler-data
- poppler-utils
- portreserve
- qt
- qt-x11
- qt3
- redhat-lsb
- redhat-lsb-core
- redhat-lsb-printing
- urw-fonts
- yum-utils
- xml-common
- ubuntu:
- package:
- binutils
- daemon
- dpkg-dev
- dpkg-repack
- gcc
- gcc-4.8
- gcc-doc
- gcc-multilib
- iputils-arping
- libasan0
- libatomic1
- libc-dev-bin
- libc6
- libc6-dev
- libcrypt-passwdmd5-perl
- libgcc-4.8-dev
- libgomp1
- libgssglue1
- libicu48
- libitm1
- libmysqlclient-dev
- libmysqlclient16
- libmysqlclient18
- libnfsidmap2
- libquadmath0
- libsasl2-dev
- libsasl2-modules-gssapi-mit
- libssl0.9.8
- libtirpc1
- libtsan0
- libxslt1.1
- linux-libc-dev
- manpages-dev
- mysql-common
- nfs-common
- open-iscsi
- openjdk-7-jre
- syslinux-common
- zlib1g-dev
- script: common/configure_extjs
- os_case:
- centos:
- copy_script: common/resources/package_utils.sh
- copy_script: common/resources/packages.json
- copy_script: common/resources/spec_5.1.0.json
- copy_script: common/resources/spec_5.2.0.json
- copy_script: common/resources/versions.py
- script:
centos/install_scala:
env_vars: [scala_version]
- script:
centos/install_mapr_core_repository:
env_vars: [plugin_version]
- script:
centos/install_mapr_eco_repository:
env_vars: [plugin_version]
- script: centos/selinux_permissive
- argument_case:
argument_name: java_distro
cases:
openjdk:
- script: centos/install_openjdk
oracle-java:
- script: common/oracle_java
- ubuntu:
- copy_script: common/resources/package_utils.sh
- copy_script: common/resources/packages.json
- copy_script: common/resources/spec_5.1.0.json
- copy_script: common/resources/spec_5.2.0.json
- copy_script: common/resources/spec_5.2.0_ubuntu.json
- copy_script: common/resources/versions.py
- script:
ubuntu/install_scala:
env_vars: [scala_version]
- script:
ubuntu/install_mapr_core_repository:
env_vars: [plugin_version]
- script:
ubuntu/install_mapr_eco_repository:
env_vars: [plugin_version]
- os_case:
- ubuntu:
- argument_case:
argument_name: java_distro
cases:
openjdk:
- script: ubuntu/install_openjdk
oracle-java:
- script: common/oracle_java

View File

@ -0,0 +1,27 @@
#!/bin/bash
DISTRO_NAME=$distro
source "/tmp/package_utils.sh"
echo "START: installing MapR core repository"
MAPR_REPO_URL="http://package.mapr.com/releases/v${plugin_version}/ubuntu/mapr-v${plugin_version}GA.deb.tgz"
MAPR_REPO_DIR="/opt/mapr-repository/core"
if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then
if [ $test_only -eq 0 ]; then
MAPR_REPO_NAME="mapr_core"
echo "Downloading MapR repository archive"
mkdir -p "$MAPR_REPO_DIR" && curl "$MAPR_REPO_URL" | tar -xz -C "$MAPR_REPO_DIR"
echo "Creating local repository"
create_repo "$MAPR_REPO_DIR"
echo "Adding MapR repository"
add_local_repo "$MAPR_REPO_NAME" "$MAPR_REPO_DIR"
fi
fi
echo "END: installing MapR core repository"

View File

@ -0,0 +1,22 @@
#!/bin/bash
echo "START: installing MapR core dependencies"
if [ ! -f /etc/apt/sources.list.d/security_repo.list ]; then
if [ $test_only -eq 0 ]; then
# Required for libicu48
cat >> /etc/apt/sources.list.d/security_repo.list << EOF
deb http://security.ubuntu.com/ubuntu precise-security main
EOF
# Required for libmysqlclient16
cat >> /etc/apt/sources.list.d/security_repo.list << EOF
deb http://old-releases.ubuntu.com/ubuntu lucid-security main
EOF
else
exit 0
fi
fi
apt-get update
echo "END: installing MapR core dependencies"

View File

@ -0,0 +1,32 @@
#!/bin/bash
VERSIONS_PY="/tmp/versions.py"
DISTRO_NAME=$distro
source "/tmp/package_utils.sh"
echo "START: installing MapR ecosystem repository"
MAPR_REPO_URL="http://package.mapr.com/releases/MEP/MEP-2.0.0/ubuntu/ binary trusty"
MAPR_REPO_DIR="/opt/mapr-repository/ecosystem"
if [ ! -d "$MAPR_REPO_DIR" ] || [ -z "$(ls -A $MAPR_REPO_DIR)" ]; then
if [ $test_only -eq 0 ]; then
MAPR_REPO_NAME="mapr_ecosystem"
MAPR_PKG_GROUPS="/tmp/packages.json"
if [ -f /tmp/spec_$plugin_version_ubuntu.json ]; then
MAPR_SPEC="/tmp/spec_$plugin_version_ubuntu.json"
else
MAPR_SPEC="/tmp/spec_$plugin_version.json"
fi
echo "Creating local MapR ecosystem repository"
localize_repo "$MAPR_REPO_NAME" "$MAPR_REPO_URL" "$MAPR_PKG_GROUPS" "$MAPR_SPEC" "$MAPR_REPO_DIR"
echo $MAPR_SPEC
fi
fi
echo "END: installing MapR ecosystem repository"

View File

@ -0,0 +1,16 @@
#!/bin/bash
echo "Installing OpenJDK"
if [ $test_only -eq 0 ]; then
apt-get install -y openjdk-7-jdk
JRE_PATH=$(update-alternatives --list java)
JRE_HOME=${JRE_PATH%/bin/java}
JDK_PATH=$(update-alternatives --list javac)
JDK_HOME=${JDK_PATH%/bin/javac}
echo "OpenJDK has been installed"
else
exit 0
fi

View File

@ -0,0 +1,35 @@
#!/bin/bash
echo "START: installing Scala"
if [ $test_only -eq 0 ]; then
RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://www.scala-lang.org/)"
if [ "$RETURN_CODE" != "200" ]; then
echo "http://www.scala-lang.org is unreachable" && exit 1
fi
if [ -n "${scala_version:-}" ]; then
VERSION=$scala_version
else
VERSION="$(curl -s --fail http://www.scala-lang.org| tr -d '\n' | sed 's/^.*<div[^<]\+scala-version">[^0-9]\+\([0-9\.\?]\+\)<.\+$/\1/')"
if [ $? != 0 -o -z "${VERSION}" ]; then
echo "Installing default version $scala_version"
VERSION=$scala_version
fi
fi
PKG=scala-${VERSION}
URL="http://downloads.lightbend.com/scala/${VERSION}"
wget -N ${URL}/${PKG}.deb
dpkg -i ${PKG}.deb
rm ${PKG}.deb
fi
echo "END: installing Scala"

View File

@ -0,0 +1,5 @@
#!/bin/bash
if [ $test_only -eq 0 ]; then
apt-get update
fi