diff --git a/elements/disable-firewall/element-deps b/elements/disable-firewall/element-deps new file mode 100644 index 00000000..7076aba9 --- /dev/null +++ b/elements/disable-firewall/element-deps @@ -0,0 +1 @@ +package-installs diff --git a/elements/disable-firewall/package-installs.yaml b/elements/disable-firewall/package-installs.yaml new file mode 100644 index 00000000..b320c880 --- /dev/null +++ b/elements/disable-firewall/package-installs.yaml @@ -0,0 +1,2 @@ +which: + phase: post-install.d diff --git a/elements/disable-firewall/post-install.d/30-disable-firewalld b/elements/disable-firewall/post-install.d/30-disable-firewalld index daad8711..108a9929 100755 --- a/elements/disable-firewall/post-install.d/30-disable-firewalld +++ b/elements/disable-firewall/post-install.d/30-disable-firewalld @@ -6,8 +6,6 @@ fi set -eu set -o pipefail -install-packages which - if which systemctl; then systemctl disable firewalld fi diff --git a/elements/hadoop-cdh/element-deps b/elements/hadoop-cdh/element-deps index 6460d715..856f4bb8 100644 --- a/elements/hadoop-cdh/element-deps +++ b/elements/hadoop-cdh/element-deps @@ -1,3 +1,4 @@ java +package-installs sahara-version ssh diff --git a/elements/hadoop-cdh/package-installs.yaml b/elements/hadoop-cdh/package-installs.yaml new file mode 100644 index 00000000..2ed9e5ec --- /dev/null +++ b/elements/hadoop-cdh/package-installs.yaml @@ -0,0 +1,4 @@ +curl: + phase: post-install.d +wget: + phase: post-install.d diff --git a/elements/hadoop-cdh/post-install.d/40-setup-hadoop b/elements/hadoop-cdh/post-install.d/40-setup-hadoop index 1daeb752..8c7fd764 100755 --- a/elements/hadoop-cdh/post-install.d/40-setup-hadoop +++ b/elements/hadoop-cdh/post-install.d/40-setup-hadoop @@ -29,7 +29,6 @@ esac echo "CDH 4 will be injected into image. Starting the download" -install-packages wget # Here more versions of CDH could be supported by downloading the right repository package. wget -P $tmp_dir "http://archive.cloudera.com/cdh4/one-click-install/precise/amd64/cdh4-repository_1.0_all.deb" if [ $? -ne 0 ]; then diff --git a/elements/hadoop-cloudera/element-deps b/elements/hadoop-cloudera/element-deps index d5e2cf8c..074d490b 100644 --- a/elements/hadoop-cloudera/element-deps +++ b/elements/hadoop-cloudera/element-deps @@ -1,2 +1,3 @@ extjs +package-installs ssh diff --git a/elements/hadoop-cloudera/package-installs.yaml b/elements/hadoop-cloudera/package-installs.yaml new file mode 100644 index 00000000..6d45f106 --- /dev/null +++ b/elements/hadoop-cloudera/package-installs.yaml @@ -0,0 +1,2 @@ +wget: + phase: pre-install.d diff --git a/elements/hadoop-hdp/element-deps b/elements/hadoop-hdp/element-deps index f3179f99..2f3ec8c6 100644 --- a/elements/hadoop-hdp/element-deps +++ b/elements/hadoop-hdp/element-deps @@ -1,5 +1,6 @@ disable-firewall disable-selinux java +package-installs sahara-version source-repositories diff --git a/elements/hadoop-hdp/install.d/30-init-hdp-install b/elements/hadoop-hdp/install.d/30-init-hdp-install index d6875071..930a779b 100755 --- a/elements/hadoop-hdp/install.d/30-init-hdp-install +++ b/elements/hadoop-hdp/install.d/30-init-hdp-install @@ -26,14 +26,6 @@ fi set -eu set -o pipefail -install-packages wget -install-packages ntp - -install-packages bind-utils - -#install cloud-inint which is necessary for all images -install-packages cloud-init - # change the cloud-init user to root sed "s/ec2-user/root/" -i /etc/cloud/cloud.cfg sed "s/disable_root: 1/disable_root: 0/" -i /etc/cloud/cloud.cfg diff --git a/elements/hadoop-hdp/install.d/40-install-hdp b/elements/hadoop-hdp/install.d/40-install-hdp index 84ab05cf..28a0d10d 100755 --- a/elements/hadoop-hdp/install.d/40-install-hdp +++ b/elements/hadoop-hdp/install.d/40-install-hdp @@ -150,9 +150,6 @@ function installHDP_2_0 { # Start of Main HDP Install Element -# Make sure wget is installed -install-packages wget - # Call version-specific script to install the desired version of HDP if [[ $DIB_HDP_VERSION == "1.3" ]]; then echo "Installing HDP Version $DIB_HDP_VERSION..." diff --git a/elements/hadoop-hdp/package-installs.yaml b/elements/hadoop-hdp/package-installs.yaml new file mode 100644 index 00000000..9a1b889e --- /dev/null +++ b/elements/hadoop-hdp/package-installs.yaml @@ -0,0 +1,5 @@ +wget: +ntp: +bind-utils: +# install cloud-init which is necessary for all images +cloud-init: diff --git a/elements/hadoop-mapr/element-deps b/elements/hadoop-mapr/element-deps index 6c4f1980..5dcf86aa 100644 --- a/elements/hadoop-mapr/element-deps +++ b/elements/hadoop-mapr/element-deps @@ -1,3 +1,4 @@ extjs java mysql +package-installs diff --git a/elements/hadoop-mapr/install.d/43-mapr-core-repository b/elements/hadoop-mapr/install.d/43-mapr-core-repository index c10bcdbc..60601cb1 100755 --- a/elements/hadoop-mapr/install.d/43-mapr-core-repository +++ b/elements/hadoop-mapr/install.d/43-mapr-core-repository @@ -8,8 +8,6 @@ set -o pipefail echo "START: installing MapR core repository" -install-packages curl - MAPR_REPO_DIR=/opt/mapr-repository/core if [ "$DISTRO_NAME" = "ubuntu" ]; then diff --git a/elements/hadoop-mapr/package-installs.yaml b/elements/hadoop-mapr/package-installs.yaml new file mode 100644 index 00000000..a3aacbce --- /dev/null +++ b/elements/hadoop-mapr/package-installs.yaml @@ -0,0 +1,2 @@ +curl: +wget: diff --git a/elements/hadoop/element-deps b/elements/hadoop/element-deps index 7bdff9fd..217cbbe3 100644 --- a/elements/hadoop/element-deps +++ b/elements/hadoop/element-deps @@ -1,4 +1,5 @@ cache-url java +package-installs sahara-version ssh diff --git a/elements/hadoop/package-installs.yaml b/elements/hadoop/package-installs.yaml new file mode 100644 index 00000000..14d466c6 --- /dev/null +++ b/elements/hadoop/package-installs.yaml @@ -0,0 +1,4 @@ +wget: + phase: post-install.d +tar: + phase: post-install.d diff --git a/elements/hadoop/post-install.d/40-setup-hadoop b/elements/hadoop/post-install.d/40-setup-hadoop index 6cc00e8c..69fd01d5 100755 --- a/elements/hadoop/post-install.d/40-setup-hadoop +++ b/elements/hadoop/post-install.d/40-setup-hadoop @@ -13,8 +13,6 @@ fi set -eu set -o pipefail -install-packages wget tar - function install_hadoop_v1 { case "$DISTRO_NAME" in ubuntu ) diff --git a/elements/hive/element-deps b/elements/hive/element-deps index b3590f9f..3bab9a0d 100644 --- a/elements/hive/element-deps +++ b/elements/hive/element-deps @@ -1,3 +1,4 @@ cache-url hadoop mysql +package-installs diff --git a/elements/hive/package-installs.yaml b/elements/hive/package-installs.yaml new file mode 100644 index 00000000..d6cd6ed1 --- /dev/null +++ b/elements/hive/package-installs.yaml @@ -0,0 +1,2 @@ +tar: + phase: post-install.d diff --git a/elements/hive/post-install.d/60-hive b/elements/hive/post-install.d/60-hive index 2647f092..a1648f2d 100755 --- a/elements/hive/post-install.d/60-hive +++ b/elements/hive/post-install.d/60-hive @@ -6,8 +6,6 @@ fi set -eu set -o pipefail -install-packages tar - tmp_dir=/tmp/hive cd $tmp_dir diff --git a/elements/oozie/element-deps b/elements/oozie/element-deps index a0bda404..e0035a99 100644 --- a/elements/oozie/element-deps +++ b/elements/oozie/element-deps @@ -1,2 +1,3 @@ cache-url +package-installs sahara-version diff --git a/elements/oozie/install.d/50-setup-oozie b/elements/oozie/install.d/50-setup-oozie index c6c24c55..3b51ac68 100755 --- a/elements/oozie/install.d/50-setup-oozie +++ b/elements/oozie/install.d/50-setup-oozie @@ -8,8 +8,6 @@ set -o pipefail echo "Oozie setup" -install-packages zip unzip tar - tmp_dir=/tmp/oozie if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then diff --git a/elements/oozie/package-installs.yaml b/elements/oozie/package-installs.yaml new file mode 100644 index 00000000..7571e3ff --- /dev/null +++ b/elements/oozie/package-installs.yaml @@ -0,0 +1,5 @@ +zip: +unzip: +tar: +wget: + phase: post-install.d diff --git a/elements/oozie/post-install.d/90-oozie-war b/elements/oozie/post-install.d/90-oozie-war index 034edf1f..6ef967cb 100755 --- a/elements/oozie/post-install.d/90-oozie-war +++ b/elements/oozie/post-install.d/90-oozie-war @@ -6,8 +6,6 @@ fi set -eu set -o pipefail -install-packages wget - ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/libtools/mysql.jar ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar diff --git a/elements/redhat-lsb/element-deps b/elements/redhat-lsb/element-deps new file mode 100644 index 00000000..7076aba9 --- /dev/null +++ b/elements/redhat-lsb/element-deps @@ -0,0 +1 @@ +package-installs diff --git a/elements/redhat-lsb/package-installs.yaml b/elements/redhat-lsb/package-installs.yaml new file mode 100644 index 00000000..c1e7e0fa --- /dev/null +++ b/elements/redhat-lsb/package-installs.yaml @@ -0,0 +1,2 @@ +redhat-lsb: + phase: pre-install.d diff --git a/elements/redhat-lsb/pre-install.d/02-lsb b/elements/redhat-lsb/pre-install.d/02-lsb deleted file mode 100755 index 0de9ba01..00000000 --- a/elements/redhat-lsb/pre-install.d/02-lsb +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -install-packages redhat-lsb diff --git a/elements/root-passwd/element-deps b/elements/root-passwd/element-deps new file mode 100644 index 00000000..7076aba9 --- /dev/null +++ b/elements/root-passwd/element-deps @@ -0,0 +1 @@ +package-installs diff --git a/elements/root-passwd/package-installs.yaml b/elements/root-passwd/package-installs.yaml new file mode 100644 index 00000000..b520b92d --- /dev/null +++ b/elements/root-passwd/package-installs.yaml @@ -0,0 +1,6 @@ +augeas-tools: + phase: post-install.d +openssh-server: + phase: post-install.d +openssh-client: + phase: post-install.d diff --git a/elements/root-passwd/post-install.d/99-setup b/elements/root-passwd/post-install.d/99-setup index 5ae2c1ec..38053199 100755 --- a/elements/root-passwd/post-install.d/99-setup +++ b/elements/root-passwd/post-install.d/99-setup @@ -11,7 +11,6 @@ if [ -z "$DIB_PASSWORD" ]; then exit 1 fi sed -i "s/disable_root: true/disable_root: false/" /etc/cloud/cloud.cfg -install-packages augeas-tools openssh-server openssh-client augtool -s set /files/etc/ssh/sshd_config/PasswordAuthentication yes augtool -s set /files/etc/ssh/sshd_config/PermitRootLogin yes augtool -s set /files/etc/ssh/ssh_config/PasswordAuthentication yes diff --git a/elements/spark/element-deps b/elements/spark/element-deps index be6899ad..a548793d 100644 --- a/elements/spark/element-deps +++ b/elements/spark/element-deps @@ -1,2 +1,3 @@ cache-url hadoop-cdh +package-installs diff --git a/elements/spark/install.d/60-spark b/elements/spark/install.d/60-spark index c7cb50b8..f93d9eb4 100755 --- a/elements/spark/install.d/60-spark +++ b/elements/spark/install.d/60-spark @@ -8,8 +8,6 @@ fi set -eu set -o pipefail -install-packages tar - tmp_dir=/tmp/spark pushd $tmp_dir diff --git a/elements/spark/package-installs.yaml b/elements/spark/package-installs.yaml new file mode 100644 index 00000000..56fd8234 --- /dev/null +++ b/elements/spark/package-installs.yaml @@ -0,0 +1 @@ +tar: diff --git a/elements/ssh/element-deps b/elements/ssh/element-deps new file mode 100644 index 00000000..7076aba9 --- /dev/null +++ b/elements/ssh/element-deps @@ -0,0 +1 @@ +package-installs diff --git a/elements/ssh/install.d/33-ssh b/elements/ssh/install.d/33-ssh index 508430cc..2447f604 100755 --- a/elements/ssh/install.d/33-ssh +++ b/elements/ssh/install.d/33-ssh @@ -7,12 +7,6 @@ set -u set -o pipefail echo "Begin: install and configure SSH" -# /etc/ssh/sshd_config is provided by openssh-server -# /etc/ssh/ssh_config is provided by openssh-client -# Note: You need diskimage-builder w/ SHA 82eacdec (11 July 2013) for -# this install to work on Fedora - https://review.openstack.org/#/c/36739/ -install-packages augeas-tools openssh-server openssh-client - augtool -s set /files/etc/ssh/sshd_config/GSSAPIAuthentication no augtool -s set /files/etc/ssh/sshd_config/UseDNS no augtool -s set /files/etc/ssh/sshd_config/PermitTunnel yes diff --git a/elements/ssh/package-installs.yaml b/elements/ssh/package-installs.yaml new file mode 100644 index 00000000..0f6aa9a3 --- /dev/null +++ b/elements/ssh/package-installs.yaml @@ -0,0 +1,5 @@ +augeas-tools: +# /etc/ssh/sshd_config is provided by openssh-server +openssh-server: +# /etc/ssh/ssh_config is provided by openssh-client +openssh-client: diff --git a/elements/storm/element-deps b/elements/storm/element-deps index 8c3d4384..80c1313c 100644 --- a/elements/storm/element-deps +++ b/elements/storm/element-deps @@ -1,2 +1,3 @@ cache-url +package-installs zookeeper diff --git a/elements/storm/install.d/60-storm b/elements/storm/install.d/60-storm index 2d079a53..9994b3da 100755 --- a/elements/storm/install.d/60-storm +++ b/elements/storm/install.d/60-storm @@ -8,8 +8,6 @@ fi set -u set -o pipefail -install-packages wget unzip build-essential uuid-dev git libtool autoconf pkg-config tar - tmp_dir=/tmp/storm echo "Creating Storm User" diff --git a/elements/storm/package-installs.yaml b/elements/storm/package-installs.yaml new file mode 100644 index 00000000..4579c010 --- /dev/null +++ b/elements/storm/package-installs.yaml @@ -0,0 +1,8 @@ +unzip: +build-essential: +uuid-dev: +git: +libtool: +autoconf: +pkg-config: +tar: diff --git a/elements/swift_hadoop/element-deps b/elements/swift_hadoop/element-deps new file mode 100644 index 00000000..7076aba9 --- /dev/null +++ b/elements/swift_hadoop/element-deps @@ -0,0 +1 @@ +package-installs diff --git a/elements/swift_hadoop/package-installs.yaml b/elements/swift_hadoop/package-installs.yaml new file mode 100644 index 00000000..cc77790e --- /dev/null +++ b/elements/swift_hadoop/package-installs.yaml @@ -0,0 +1,2 @@ +wget: + phase: post-install.d diff --git a/elements/swift_hadoop/post-install.d/81-add-jar b/elements/swift_hadoop/post-install.d/81-add-jar index fa8a2629..ecd3ebf9 100755 --- a/elements/swift_hadoop/post-install.d/81-add-jar +++ b/elements/swift_hadoop/post-install.d/81-add-jar @@ -6,8 +6,6 @@ fi set -eu set -o pipefail -install-packages wget - HDFS_LIB_DIR=${DIB_HDFS_LIB_DIR:-"/usr/share/hadoop/lib"} SWIFT_LIB_URI="http://sahara-files.mirantis.com/hadoop-swift/hadoop-swift-latest.jar" diff --git a/elements/zookeeper/element-deps b/elements/zookeeper/element-deps new file mode 100644 index 00000000..7076aba9 --- /dev/null +++ b/elements/zookeeper/element-deps @@ -0,0 +1 @@ +package-installs diff --git a/elements/zookeeper/install.d/40-zookeeper b/elements/zookeeper/install.d/40-zookeeper index ed127ada..fb5fc6c2 100755 --- a/elements/zookeeper/install.d/40-zookeeper +++ b/elements/zookeeper/install.d/40-zookeeper @@ -8,8 +8,6 @@ fi set -eu set -o pipefail -install-packages wget - echo "Downloading Zookeeper" mkdir -p /opt/zookeeper diff --git a/elements/zookeeper/package-installs.yaml b/elements/zookeeper/package-installs.yaml new file mode 100644 index 00000000..a23f1f36 --- /dev/null +++ b/elements/zookeeper/package-installs.yaml @@ -0,0 +1 @@ +wget: