From d214c228a097929f2409d9c1676ac2961978a52b Mon Sep 17 00:00:00 2001 From: Telles Nobrega Date: Thu, 16 Feb 2017 13:22:08 -0300 Subject: [PATCH] Image generation for Ambari Plugin Adds image generation for Ambari Change-Id: I0e798696de7fb2fcf72d826ad6b7356f526db836 --- ...eneration_validation-47eabb9fa90384c8.yaml | 4 + sahara/plugins/ambari/plugin.py | 26 +++++ .../resources/images/centos/disable_ambari | 8 ++ .../resources/images/centos/disable_firewall | 20 ++++ .../resources/images/centos/disable_selinux | 12 ++ .../resources/images/centos/setup_java_home | 31 ++++++ .../centos/unlimited_security_artifacts | 11 ++ .../ambari/resources/images/centos/wget_repo | 9 ++ .../ambari/resources/images/common/add_jar | 31 ++++++ .../resources/images/common/oracle_java | 41 +++++++ .../ambari/resources/images/image.yaml | 105 +++++++++++++++++- .../ambari/resources/images/ubuntu/wget_repo | 10 ++ sahara/plugins/images.py | 2 +- tools/gate/build-images | 3 + 14 files changed, 308 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/ambari_2_4_image_generation_validation-47eabb9fa90384c8.yaml create mode 100644 sahara/plugins/ambari/resources/images/centos/disable_ambari create mode 100644 sahara/plugins/ambari/resources/images/centos/disable_firewall create mode 100644 sahara/plugins/ambari/resources/images/centos/disable_selinux create mode 100644 sahara/plugins/ambari/resources/images/centos/setup_java_home create mode 100644 sahara/plugins/ambari/resources/images/centos/unlimited_security_artifacts create mode 100644 sahara/plugins/ambari/resources/images/centos/wget_repo create mode 100644 sahara/plugins/ambari/resources/images/common/add_jar create mode 100644 sahara/plugins/ambari/resources/images/common/oracle_java create mode 100644 sahara/plugins/ambari/resources/images/ubuntu/wget_repo diff --git a/releasenotes/notes/ambari_2_4_image_generation_validation-47eabb9fa90384c8.yaml b/releasenotes/notes/ambari_2_4_image_generation_validation-47eabb9fa90384c8.yaml new file mode 100644 index 0000000000..d9a7ee23c5 --- /dev/null +++ b/releasenotes/notes/ambari_2_4_image_generation_validation-47eabb9fa90384c8.yaml @@ -0,0 +1,4 @@ +--- +features: + - Enables the creation and validation of Ambari 2.4 images using the + new image generation process where libguestfs replaces the use of DIB. diff --git a/sahara/plugins/ambari/plugin.py b/sahara/plugins/ambari/plugin.py index bea5a21fb0..b04c3a829f 100644 --- a/sahara/plugins/ambari/plugin.py +++ b/sahara/plugins/ambari/plugin.py @@ -23,6 +23,7 @@ from sahara.plugins.ambari import deploy from sahara.plugins.ambari import edp_engine from sahara.plugins.ambari import health from sahara.plugins.ambari import validation +from sahara.plugins import images from sahara.plugins import kerberos from sahara.plugins import provisioning as p from sahara.plugins import utils as plugin_utils @@ -263,3 +264,28 @@ class AmbariPluginProvider(p.ProvisioningPluginBase): def get_health_checks(self, cluster): return health.get_health_checks(cluster) + + validator = images.SaharaImageValidator.from_yaml( + 'plugins/ambari/resources/images/image.yaml', + resource_roots=['plugins/ambari/resources/images']) + + def get_image_arguments(self, hadoop_version): + if hadoop_version != '2.4': + return NotImplemented + return self.validator.get_argument_list() + + def pack_image(self, hadoop_version, remote, + test_only=False, image_arguments=None): + self.validator.validate(remote, test_only=test_only, + image_arguments=image_arguments) + + def validate_images(self, cluster, test_only=False, image_arguments=None): + image_arguments = self.get_image_arguments(cluster['hadoop_version']) + if not test_only: + instances = plugin_utils.get_instances(cluster) + else: + instances = plugin_utils.get_instances(cluster)[0] + for instance in instances: + with instance.remote() as r: + self.validator.validate(r, test_only=test_only, + image_arguments=image_arguments) diff --git a/sahara/plugins/ambari/resources/images/centos/disable_ambari b/sahara/plugins/ambari/resources/images/centos/disable_ambari new file mode 100644 index 0000000000..a181823873 --- /dev/null +++ b/sahara/plugins/ambari/resources/images/centos/disable_ambari @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [ $test_only -eq 0 ]; then + chkconfig ambari-server off + chkconfig ambari-agent off +else + exit 0 +fi diff --git a/sahara/plugins/ambari/resources/images/centos/disable_firewall b/sahara/plugins/ambari/resources/images/centos/disable_firewall new file mode 100644 index 0000000000..e82c456521 --- /dev/null +++ b/sahara/plugins/ambari/resources/images/centos/disable_firewall @@ -0,0 +1,20 @@ +#!/bin/bash + +check=$(systemctl --no-pager list-unit-files iptables.service | grep 'enabled' | wc -l) + +if [ $check -eq 1 ]; then + if [ $test_only -eq 0 ]; then + if type -p systemctl && [[ "$(systemctl --no-pager list-unit-files firewalld)" =~ 'enabled' ]]; then + systemctl disable firewalld + fi + + if type -p service; then + service ip6tables save + service iptables save + chkconfig ip6tables off + chkconfig iptables off + fi + else + exit 0 + fi +fi diff --git a/sahara/plugins/ambari/resources/images/centos/disable_selinux b/sahara/plugins/ambari/resources/images/centos/disable_selinux new file mode 100644 index 0000000000..b24dc12916 --- /dev/null +++ b/sahara/plugins/ambari/resources/images/centos/disable_selinux @@ -0,0 +1,12 @@ +#!/bin/bash + +check=$(cat /etc/selinux/config | grep 'SELINUX=disabled' | wc -l) + +if [ $check -eq 0 ]; then + if [ $test_only -eq 0 ]; then + config=/etc/selinux/config + [ -e $config ] && sed -i "s%^\(SELINUX=\s*\).*$%SELINUX=disabled%" $config + else + exit 0 + fi +fi diff --git a/sahara/plugins/ambari/resources/images/centos/setup_java_home b/sahara/plugins/ambari/resources/images/centos/setup_java_home new file mode 100644 index 0000000000..7bbe27a0b4 --- /dev/null +++ b/sahara/plugins/ambari/resources/images/centos/setup_java_home @@ -0,0 +1,31 @@ +#!/bin/bash + +JAVA_RC="/etc/profile.d/99-java.sh" +JAVA_BIN_RC="/etc/profile.d/98-java-bin.sh" + +if [ ! -f $JAVA_RC ]; then + if [ $test_only -eq 0 ]; then + case "$java_distro" in + openjdk ) + JRE_HOME="/usr/lib/jvm/java-openjdk/jre" + JDK_HOME="/usr/lib/jvm/java-openjdk" + ;; + oracle-java ) + JRE_HOME="/usr/java/oracle-jdk" + JDK_HOME="/usr/java/oracle-jdk" + ;; + esac + + echo "export JAVA_HOME=$JRE_HOME" >> $JAVA_RC + chmod +x $JAVA_RC + + echo "export PATH=$JRE_HOME/bin:\$PATH" >> $JAVA_BIN_RC + echo "export PATH=$JDK_HOME/bin:\$PATH" >> $JAVA_BIN_RC + chmod +x $JAVA_BIN_RC + + alternatives --install /usr/bin/java java $JRE_HOME/bin/java 200000 + alternatives --install /usr/bin/javac javac $JDK_HOME/bin/javac 200000 + else + exit 0 + fi +fi diff --git a/sahara/plugins/ambari/resources/images/centos/unlimited_security_artifacts b/sahara/plugins/ambari/resources/images/centos/unlimited_security_artifacts new file mode 100644 index 0000000000..bcf70675bf --- /dev/null +++ b/sahara/plugins/ambari/resources/images/centos/unlimited_security_artifacts @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ ! -d /tmp/UnlimitedPolicy/ ]; then + if [ $test_only -eq 0 ]; then + mkdir /tmp/UnlimitedPolicy/ + wget http://tarballs.openstack.org/sahara/dist/common-artifacts/local_policy.jar -O /tmp/UnlimitedPolicy/local_policy.jar + wget http://tarballs.openstack.org/sahara/dist/common-artifacts/US_export_policy.jar -O /tmp/UnlimitedPolicy/US_export_policy.jar + else + exit 0 + fi +fi diff --git a/sahara/plugins/ambari/resources/images/centos/wget_repo b/sahara/plugins/ambari/resources/images/centos/wget_repo new file mode 100644 index 0000000000..9033c84530 --- /dev/null +++ b/sahara/plugins/ambari/resources/images/centos/wget_repo @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [ ! -f /etc/yum.repos.d/ambari.repo ]; then + if [ $test_only -eq 0 ]; then + wget http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/$ambari_version/ambari.repo -O /etc/yum.repos.d/ambari.repo + else + exit 0 + fi +fi diff --git a/sahara/plugins/ambari/resources/images/common/add_jar b/sahara/plugins/ambari/resources/images/common/add_jar new file mode 100644 index 0000000000..9c0560be45 --- /dev/null +++ b/sahara/plugins/ambari/resources/images/common/add_jar @@ -0,0 +1,31 @@ +#! /bin/bash + +hadoop="2.7.1" + +HDFS_LIB_DIR=${hdfs_lib_dir:-"/usr/share/hadoop/lib"} +JAR_BUILD_DATE="2016-03-17" +SWIFT_LIB_URI="http://tarballs.openstack.org/sahara/dist/hadoop-openstack/hadoop-openstack-${hadoop}.jar" +HADOOP_SWIFT_JAR_NAME=hadoop-openstack.jar + +if [ ! -f $HDFS_LIB_DIR/$HADOOP_SWIFT_JAR_NAME ]; then + if [ $test_only -eq 0 ]; then + if [ -z "${swift_url:-}" ]; then + wget -O $HDFS_LIB_DIR/$HADOOP_SWIFT_JAR_NAME $SWIFT_LIB_URI + else + wget -O $HDFS_LIB_DIR/$HADOOP_SWIFT_JAR_NAME $swift_url + fi + + if [ $? -ne 0 ]; then + echo -e "Could not download Swift Hadoop FS implementation.\nAborting" + exit 1 + fi + + chmod 0644 $HDFS_LIB_DIR/$HADOOP_SWIFT_JAR_NAME + else + exit 0 + fi +fi + + + + diff --git a/sahara/plugins/ambari/resources/images/common/oracle_java b/sahara/plugins/ambari/resources/images/common/oracle_java new file mode 100644 index 0000000000..eec05823dd --- /dev/null +++ b/sahara/plugins/ambari/resources/images/common/oracle_java @@ -0,0 +1,41 @@ +#!/bin/bash + + +# NOTE: $(dirname $0) is read-only, use space under $TARGET_ROOT +JAVA_LOCATION=${JAVA_TARGET_LOCATION:-"/usr/java"} +JAVA_NAME="oracle-jdk" +JAVA_HOME=$JAVA_LOCATION/$JAVA_NAME +JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"} + +if [ ! -d $JAVA_LOCATION ]; then + if [ $test_only -eq 0 ]; then + echo "Begin: installation of Java" + mkdir -p $JAVA_LOCATION + + if [ -n "$JAVA_DOWNLOAD_URL" ]; then + JAVA_FILE=$(basename $JAVA_DOWNLOAD_URL) + wget --no-check-certificate --no-cookies -c \ + --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \ + -O $JAVA_LOCATION/$JAVA_FILE $JAVA_DOWNLOAD_URL + elif [ -n "$JAVA_FILE" ]; then + install -D -g root -o root -m 0755 $(dirname $0)/$JAVA_FILE $JAVA_LOCATION + fi + + cd $JAVA_LOCATION + + echo "Decompressing Java archive" + echo -e "\n" | tar -zxf $JAVA_FILE + echo "Setting up $JAVA_NAME" + chown -R root:root $JAVA_LOCATION + JAVA_DIR=`ls -1 $JAVA_LOCATION | grep -v tar.gz` + ln -s $JAVA_LOCATION/$JAVA_DIR $JAVA_HOME + + setup-java-home $JAVA_HOME $JAVA_HOME + + rm $JAVA_FILE + + echo "End: installation of Java" + else + exit 0 + fi +fi diff --git a/sahara/plugins/ambari/resources/images/image.yaml b/sahara/plugins/ambari/resources/images/image.yaml index 39dcb1345a..c6ff462409 100644 --- a/sahara/plugins/ambari/resources/images/image.yaml +++ b/sahara/plugins/ambari/resources/images/image.yaml @@ -1,6 +1,103 @@ arguments: - fish: - description: awesome - default: trout + ambari_version: + description: The version of Ambari to install. Defaults to 2.2.1.0. + default: 2.2.1.0 + choices: + - 2.2.0.0 # HDP 2.3 + - 2.2.1.0 # HDP 2.4 + java_distro: + default: openjdk + description: The distribution of Java to install. Defaults to openjdk. + choices: + - openjdk + - oracle-java + hdfs_lib_dir: + default: /opt + description: The path to HDFS lib. Defaults to /opt. + required: False + swift_url: + default: https://tarballs.openstack.org/sahara/dist/hadoop-openstack/master/ + description: Location of the swift jar file. + required: False + +validators: + - argument_case: + argument_name: java_distro + cases: + openjdk: + - package: java-1.7.0-openjdk-devel + oracle-java: + - script: common/oracle_java + - os_case: + - centos: + - script: centos/disable_selinux + - script: + centos/setup_java_home: + env_vars: [java_distro] + - package: wget + - script: + centos/wget_repo: + env_vars: [ambari_version] + - package: redhat-lsb + - package: + - mariadb + - mariadb-libs + - mariadb-server + - mysql-connector-java + - package: ntp + - package: + - ambari-metrics-monitor + - ambari-server + - ambari-metrics-collector + - ambari-metrics-hadoop-sink + - package: nmap-ncat + - package: fuse-libs + - package: snappy-devel + - ubuntu: + - script: + ubuntu/wget_repo: + env_vars: [ambari_version] + - package: + - ambari-metrics-assembly + - netcat + - package: fuse + - package: + - mysql-client-5.5 + - mysql-server-5.5 + - libmysql-java + - package: ambari-agent + - package: + - unzip + - zip + - curl + - tar + - rpcbind + - rng-tools + - iptables-services + - os_case: + - centos: + - script: centos/disable_ambari + - script: centos/disable_firewall + - script: + common/add_jar: + env_vars: [hdfs_lib_dir, swift_url] + - script: + centos/unlimited_security_artifacts: + env_vars: [unlimited_security_location] + - ubuntu: + - script: + common/add_jar: + env_vars: [hdfs_lib_dir, swift_url] + - os_case: + - centos: + - package: + - krb5-server + - krb5-libs + - krb5-workstation + - ubuntu: + - package: + - krb5-admin-server + - libpam-krb5 + - krb5-user + - ldap-utils -validators: [] \ No newline at end of file diff --git a/sahara/plugins/ambari/resources/images/ubuntu/wget_repo b/sahara/plugins/ambari/resources/images/ubuntu/wget_repo new file mode 100644 index 0000000000..2825c30e08 --- /dev/null +++ b/sahara/plugins/ambari/resources/images/ubuntu/wget_repo @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +if [ ! -f /etc/apt/sources.list.d/ambari.list ]; then + if [ $test_only -eq 0 ]; then + wget http://public-repo-1.hortonworks.com/ambari/ubuntu12/2.x/updates/$ambari_version/ambari.list -O /etc/apt/sources.list.d/ambari.list + apt-key adv --recv-keys --keyserver keyserver.ubuntu.com B9733A7A07513CAD + else + exit 0 + fi +fi diff --git a/sahara/plugins/images.py b/sahara/plugins/images.py index fdbcdc5c0f..66bb607b31 100644 --- a/sahara/plugins/images.py +++ b/sahara/plugins/images.py @@ -684,7 +684,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase): path = '/tmp/%s.sh' % uuidutils.generate_uuid() remote.write_file_to(path, script, run_as_root=True) _sudo(remote, 'chmod +x %s' % path) - code, stdout = _sudo(remote, '%s' % path) + code, stdout = _sudo(remote, path) if self.output_var: image_arguments[self.output_var] = stdout diff --git a/tools/gate/build-images b/tools/gate/build-images index 4c30ae8d99..28b7a0ff27 100755 --- a/tools/gate/build-images +++ b/tools/gate/build-images @@ -67,6 +67,9 @@ case "$PLUGIN" in "cloudera") build_images "cdh" "5.7.0" "centos7" ;; + "ambari") + build_images "ambari" "2.4" "centos7" + ;; *) echo "Invalid version" ;;