Merge "Image generation for Ambari Plugin"
This commit is contained in:
Zuul
Gerrit Code Review
commit
85b46adbf4
@ -0,0 +1,4 @@
|
||||
---
|
||||
features:
|
||||
- Enables the creation and validation of Ambari 2.4 images using the
|
||||
new image generation process where libguestfs replaces the use of DIB.
|
||||
@ -23,6 +23,7 @@ from sahara.plugins.ambari import deploy
|
||||
from sahara.plugins.ambari import edp_engine
|
||||
from sahara.plugins.ambari import health
|
||||
from sahara.plugins.ambari import validation
|
||||
from sahara.plugins import images
|
||||
from sahara.plugins import kerberos
|
||||
from sahara.plugins import provisioning as p
|
||||
from sahara.plugins import utils as plugin_utils
|
||||
@ -263,3 +264,28 @@ class AmbariPluginProvider(p.ProvisioningPluginBase):
|
||||
|
||||
def get_health_checks(self, cluster):
|
||||
return health.get_health_checks(cluster)
|
||||
|
||||
validator = images.SaharaImageValidator.from_yaml(
|
||||
'plugins/ambari/resources/images/image.yaml',
|
||||
resource_roots=['plugins/ambari/resources/images'])
|
||||
|
||||
def get_image_arguments(self, hadoop_version):
|
||||
if hadoop_version != '2.4':
|
||||
return NotImplemented
|
||||
return self.validator.get_argument_list()
|
||||
|
||||
def pack_image(self, hadoop_version, remote,
|
||||
test_only=False, image_arguments=None):
|
||||
self.validator.validate(remote, test_only=test_only,
|
||||
image_arguments=image_arguments)
|
||||
|
||||
def validate_images(self, cluster, test_only=False, image_arguments=None):
|
||||
image_arguments = self.get_image_arguments(cluster['hadoop_version'])
|
||||
if not test_only:
|
||||
instances = plugin_utils.get_instances(cluster)
|
||||
else:
|
||||
instances = plugin_utils.get_instances(cluster)[0]
|
||||
for instance in instances:
|
||||
with instance.remote() as r:
|
||||
self.validator.validate(r, test_only=test_only,
|
||||
image_arguments=image_arguments)
|
||||
|
||||
@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
if [ $test_only -eq 0 ]; then
|
||||
chkconfig ambari-server off
|
||||
chkconfig ambari-agent off
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
|
||||
check=$(systemctl --no-pager list-unit-files iptables.service | grep 'enabled' | wc -l)
|
||||
|
||||
if [ $check -eq 1 ]; then
|
||||
if [ $test_only -eq 0 ]; then
|
||||
if type -p systemctl && [[ "$(systemctl --no-pager list-unit-files firewalld)" =~ 'enabled' ]]; then
|
||||
systemctl disable firewalld
|
||||
fi
|
||||
|
||||
if type -p service; then
|
||||
service ip6tables save
|
||||
service iptables save
|
||||
chkconfig ip6tables off
|
||||
chkconfig iptables off
|
||||
fi
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
check=$(cat /etc/selinux/config | grep 'SELINUX=disabled' | wc -l)
|
||||
|
||||
if [ $check -eq 0 ]; then
|
||||
if [ $test_only -eq 0 ]; then
|
||||
config=/etc/selinux/config
|
||||
[ -e $config ] && sed -i "s%^\(SELINUX=\s*\).*$%SELINUX=disabled%" $config
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
|
||||
JAVA_RC="/etc/profile.d/99-java.sh"
|
||||
JAVA_BIN_RC="/etc/profile.d/98-java-bin.sh"
|
||||
|
||||
if [ ! -f $JAVA_RC ]; then
|
||||
if [ $test_only -eq 0 ]; then
|
||||
case "$java_distro" in
|
||||
openjdk )
|
||||
JRE_HOME="/usr/lib/jvm/java-openjdk/jre"
|
||||
JDK_HOME="/usr/lib/jvm/java-openjdk"
|
||||
;;
|
||||
oracle-java )
|
||||
JRE_HOME="/usr/java/oracle-jdk"
|
||||
JDK_HOME="/usr/java/oracle-jdk"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "export JAVA_HOME=$JRE_HOME" >> $JAVA_RC
|
||||
chmod +x $JAVA_RC
|
||||
|
||||
echo "export PATH=$JRE_HOME/bin:\$PATH" >> $JAVA_BIN_RC
|
||||
echo "export PATH=$JDK_HOME/bin:\$PATH" >> $JAVA_BIN_RC
|
||||
chmod +x $JAVA_BIN_RC
|
||||
|
||||
alternatives --install /usr/bin/java java $JRE_HOME/bin/java 200000
|
||||
alternatives --install /usr/bin/javac javac $JDK_HOME/bin/javac 200000
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ ! -d /tmp/UnlimitedPolicy/ ]; then
|
||||
if [ $test_only -eq 0 ]; then
|
||||
mkdir /tmp/UnlimitedPolicy/
|
||||
wget http://tarballs.openstack.org/sahara/dist/common-artifacts/local_policy.jar -O /tmp/UnlimitedPolicy/local_policy.jar
|
||||
wget http://tarballs.openstack.org/sahara/dist/common-artifacts/US_export_policy.jar -O /tmp/UnlimitedPolicy/US_export_policy.jar
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
9
sahara/plugins/ambari/resources/images/centos/wget_repo
Normal file
9
sahara/plugins/ambari/resources/images/centos/wget_repo
Normal file
@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
if [ ! -f /etc/yum.repos.d/ambari.repo ]; then
|
||||
if [ $test_only -eq 0 ]; then
|
||||
wget http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/$ambari_version/ambari.repo -O /etc/yum.repos.d/ambari.repo
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
31
sahara/plugins/ambari/resources/images/common/add_jar
Normal file
31
sahara/plugins/ambari/resources/images/common/add_jar
Normal file
@ -0,0 +1,31 @@
|
||||
#! /bin/bash
|
||||
|
||||
hadoop="2.7.1"
|
||||
|
||||
HDFS_LIB_DIR=${hdfs_lib_dir:-"/usr/share/hadoop/lib"}
|
||||
JAR_BUILD_DATE="2016-03-17"
|
||||
SWIFT_LIB_URI="http://tarballs.openstack.org/sahara/dist/hadoop-openstack/hadoop-openstack-${hadoop}.jar"
|
||||
HADOOP_SWIFT_JAR_NAME=hadoop-openstack.jar
|
||||
|
||||
if [ ! -f $HDFS_LIB_DIR/$HADOOP_SWIFT_JAR_NAME ]; then
|
||||
if [ $test_only -eq 0 ]; then
|
||||
if [ -z "${swift_url:-}" ]; then
|
||||
wget -O $HDFS_LIB_DIR/$HADOOP_SWIFT_JAR_NAME $SWIFT_LIB_URI
|
||||
else
|
||||
wget -O $HDFS_LIB_DIR/$HADOOP_SWIFT_JAR_NAME $swift_url
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "Could not download Swift Hadoop FS implementation.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chmod 0644 $HDFS_LIB_DIR/$HADOOP_SWIFT_JAR_NAME
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
|
||||
|
||||
41
sahara/plugins/ambari/resources/images/common/oracle_java
Normal file
41
sahara/plugins/ambari/resources/images/common/oracle_java
Normal file
@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
|
||||
|
||||
# NOTE: $(dirname $0) is read-only, use space under $TARGET_ROOT
|
||||
JAVA_LOCATION=${JAVA_TARGET_LOCATION:-"/usr/java"}
|
||||
JAVA_NAME="oracle-jdk"
|
||||
JAVA_HOME=$JAVA_LOCATION/$JAVA_NAME
|
||||
JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"}
|
||||
|
||||
if [ ! -d $JAVA_LOCATION ]; then
|
||||
if [ $test_only -eq 0 ]; then
|
||||
echo "Begin: installation of Java"
|
||||
mkdir -p $JAVA_LOCATION
|
||||
|
||||
if [ -n "$JAVA_DOWNLOAD_URL" ]; then
|
||||
JAVA_FILE=$(basename $JAVA_DOWNLOAD_URL)
|
||||
wget --no-check-certificate --no-cookies -c \
|
||||
--header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \
|
||||
-O $JAVA_LOCATION/$JAVA_FILE $JAVA_DOWNLOAD_URL
|
||||
elif [ -n "$JAVA_FILE" ]; then
|
||||
install -D -g root -o root -m 0755 $(dirname $0)/$JAVA_FILE $JAVA_LOCATION
|
||||
fi
|
||||
|
||||
cd $JAVA_LOCATION
|
||||
|
||||
echo "Decompressing Java archive"
|
||||
echo -e "\n" | tar -zxf $JAVA_FILE
|
||||
echo "Setting up $JAVA_NAME"
|
||||
chown -R root:root $JAVA_LOCATION
|
||||
JAVA_DIR=`ls -1 $JAVA_LOCATION | grep -v tar.gz`
|
||||
ln -s $JAVA_LOCATION/$JAVA_DIR $JAVA_HOME
|
||||
|
||||
setup-java-home $JAVA_HOME $JAVA_HOME
|
||||
|
||||
rm $JAVA_FILE
|
||||
|
||||
echo "End: installation of Java"
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
@ -1,6 +1,103 @@
|
||||
arguments:
|
||||
fish:
|
||||
description: awesome
|
||||
default: trout
|
||||
ambari_version:
|
||||
description: The version of Ambari to install. Defaults to 2.2.1.0.
|
||||
default: 2.2.1.0
|
||||
choices:
|
||||
- 2.2.0.0 # HDP 2.3
|
||||
- 2.2.1.0 # HDP 2.4
|
||||
java_distro:
|
||||
default: openjdk
|
||||
description: The distribution of Java to install. Defaults to openjdk.
|
||||
choices:
|
||||
- openjdk
|
||||
- oracle-java
|
||||
hdfs_lib_dir:
|
||||
default: /opt
|
||||
description: The path to HDFS lib. Defaults to /opt.
|
||||
required: False
|
||||
swift_url:
|
||||
default: https://tarballs.openstack.org/sahara/dist/hadoop-openstack/master/
|
||||
description: Location of the swift jar file.
|
||||
required: False
|
||||
|
||||
validators:
|
||||
- argument_case:
|
||||
argument_name: java_distro
|
||||
cases:
|
||||
openjdk:
|
||||
- package: java-1.7.0-openjdk-devel
|
||||
oracle-java:
|
||||
- script: common/oracle_java
|
||||
- os_case:
|
||||
- centos:
|
||||
- script: centos/disable_selinux
|
||||
- script:
|
||||
centos/setup_java_home:
|
||||
env_vars: [java_distro]
|
||||
- package: wget
|
||||
- script:
|
||||
centos/wget_repo:
|
||||
env_vars: [ambari_version]
|
||||
- package: redhat-lsb
|
||||
- package:
|
||||
- mariadb
|
||||
- mariadb-libs
|
||||
- mariadb-server
|
||||
- mysql-connector-java
|
||||
- package: ntp
|
||||
- package:
|
||||
- ambari-metrics-monitor
|
||||
- ambari-server
|
||||
- ambari-metrics-collector
|
||||
- ambari-metrics-hadoop-sink
|
||||
- package: nmap-ncat
|
||||
- package: fuse-libs
|
||||
- package: snappy-devel
|
||||
- ubuntu:
|
||||
- script:
|
||||
ubuntu/wget_repo:
|
||||
env_vars: [ambari_version]
|
||||
- package:
|
||||
- ambari-metrics-assembly
|
||||
- netcat
|
||||
- package: fuse
|
||||
- package:
|
||||
- mysql-client-5.5
|
||||
- mysql-server-5.5
|
||||
- libmysql-java
|
||||
- package: ambari-agent
|
||||
- package:
|
||||
- unzip
|
||||
- zip
|
||||
- curl
|
||||
- tar
|
||||
- rpcbind
|
||||
- rng-tools
|
||||
- iptables-services
|
||||
- os_case:
|
||||
- centos:
|
||||
- script: centos/disable_ambari
|
||||
- script: centos/disable_firewall
|
||||
- script:
|
||||
common/add_jar:
|
||||
env_vars: [hdfs_lib_dir, swift_url]
|
||||
- script:
|
||||
centos/unlimited_security_artifacts:
|
||||
env_vars: [unlimited_security_location]
|
||||
- ubuntu:
|
||||
- script:
|
||||
common/add_jar:
|
||||
env_vars: [hdfs_lib_dir, swift_url]
|
||||
- os_case:
|
||||
- centos:
|
||||
- package:
|
||||
- krb5-server
|
||||
- krb5-libs
|
||||
- krb5-workstation
|
||||
- ubuntu:
|
||||
- package:
|
||||
- krb5-admin-server
|
||||
- libpam-krb5
|
||||
- krb5-user
|
||||
- ldap-utils
|
||||
|
||||
validators: []
|
||||
10
sahara/plugins/ambari/resources/images/ubuntu/wget_repo
Normal file
10
sahara/plugins/ambari/resources/images/ubuntu/wget_repo
Normal file
@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
if [ ! -f /etc/apt/sources.list.d/ambari.list ]; then
|
||||
if [ $test_only -eq 0 ]; then
|
||||
wget http://public-repo-1.hortonworks.com/ambari/ubuntu12/2.x/updates/$ambari_version/ambari.list -O /etc/apt/sources.list.d/ambari.list
|
||||
apt-key adv --recv-keys --keyserver keyserver.ubuntu.com B9733A7A07513CAD
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
@ -684,7 +684,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase):
|
||||
path = '/tmp/%s.sh' % uuidutils.generate_uuid()
|
||||
remote.write_file_to(path, script, run_as_root=True)
|
||||
_sudo(remote, 'chmod +x %s' % path)
|
||||
code, stdout = _sudo(remote, '%s' % path)
|
||||
code, stdout = _sudo(remote, path)
|
||||
if self.output_var:
|
||||
image_arguments[self.output_var] = stdout
|
||||
|
||||
|
||||
@ -67,6 +67,9 @@ case "$PLUGIN" in
|
||||
"cloudera")
|
||||
build_images "cdh" "5.7.0" "centos7"
|
||||
;;
|
||||
"ambari")
|
||||
build_images "ambari" "2.4" "centos7"
|
||||
;;
|
||||
*)
|
||||
echo "Invalid version"
|
||||
;;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user