Start switching to declarative package-installs

Make use of the package-installs element to declare in yaml format the packages to be installed at the beginning of a phase. Other than reducing the amont of explicit 'install-packages pkg1 ...' invocations, they can also be installed just once per-phase. Change-Id: I1f1acfb2bd74fed5cf4c0b48bc739f7f75c35d83
2015-04-09 10:15:29 +02:00 · 2015-04-09 10:15:29 +02:00 · c8cbe25f1b
commit c8cbe25f1b
parent c3e937604a
46 changed files with 66 additions and 48 deletions
--- a/elements/disable-firewall/element-deps
+++ b/elements/disable-firewall/element-deps
@ -0,0 +1 @@
+package-installs
--- a/elements/disable-firewall/package-installs.yaml
+++ b/elements/disable-firewall/package-installs.yaml
@ -0,0 +1,2 @@
+which:
+  phase: post-install.d
--- a/elements/disable-firewall/post-install.d/30-disable-firewalld
+++ b/elements/disable-firewall/post-install.d/30-disable-firewalld
@ -6,8 +6,6 @@ fi
 set -eu
 set -o pipefail

-install-packages which
-
 if which systemctl; then
    systemctl disable firewalld
 fi
--- a/elements/hadoop-cdh/element-deps
+++ b/elements/hadoop-cdh/element-deps
@ -1,3 +1,4 @@
 java
+package-installs
 sahara-version
 ssh
--- a/elements/hadoop-cdh/package-installs.yaml
+++ b/elements/hadoop-cdh/package-installs.yaml
@ -0,0 +1,4 @@
+curl:
+  phase: post-install.d
+wget:
+  phase: post-install.d
--- a/elements/hadoop-cdh/post-install.d/40-setup-hadoop
+++ b/elements/hadoop-cdh/post-install.d/40-setup-hadoop
@ -29,7 +29,6 @@ esac

 echo "CDH 4 will be injected into image. Starting the download"

-install-packages wget
 # Here more versions of CDH could be supported by downloading the right repository package.
 wget -P $tmp_dir "http://archive.cloudera.com/cdh4/one-click-install/precise/amd64/cdh4-repository_1.0_all.deb"
 if [ $? -ne 0 ]; then
--- a/elements/hadoop-cloudera/element-deps
+++ b/elements/hadoop-cloudera/element-deps
@ -1,2 +1,3 @@
 extjs
+package-installs
 ssh
--- a/elements/hadoop-cloudera/package-installs.yaml
+++ b/elements/hadoop-cloudera/package-installs.yaml
@ -0,0 +1,2 @@
+wget:
+  phase: pre-install.d
--- a/elements/hadoop-hdp/element-deps
+++ b/elements/hadoop-hdp/element-deps
@ -1,5 +1,6 @@
 disable-firewall
 disable-selinux
 java
+package-installs
 sahara-version
 source-repositories
--- a/elements/hadoop-hdp/install.d/30-init-hdp-install
+++ b/elements/hadoop-hdp/install.d/30-init-hdp-install
@ -26,14 +26,6 @@ fi
 set -eu
 set -o pipefail

-install-packages wget
-install-packages ntp
-
-install-packages bind-utils
-
-#install cloud-inint which is necessary for all images
-install-packages cloud-init
-
 # change the cloud-init user to root
 sed "s/ec2-user/root/" -i /etc/cloud/cloud.cfg
 sed "s/disable_root: 1/disable_root: 0/" -i /etc/cloud/cloud.cfg
--- a/elements/hadoop-hdp/install.d/40-install-hdp
+++ b/elements/hadoop-hdp/install.d/40-install-hdp
@ -150,9 +150,6 @@ function installHDP_2_0 {

 # Start of Main HDP Install Element

-# Make sure wget is installed
-install-packages wget
-
 # Call version-specific script to install the desired version of HDP
 if [[ $DIB_HDP_VERSION == "1.3"  ]]; then
    echo "Installing HDP Version $DIB_HDP_VERSION..."
--- a/elements/hadoop-hdp/package-installs.yaml
+++ b/elements/hadoop-hdp/package-installs.yaml
@ -0,0 +1,5 @@
+wget:
+ntp:
+bind-utils:
+# install cloud-init which is necessary for all images
+cloud-init:
--- a/elements/hadoop-mapr/element-deps
+++ b/elements/hadoop-mapr/element-deps
@ -1,3 +1,4 @@
 extjs
 java
 mysql
+package-installs
--- a/elements/hadoop-mapr/install.d/43-mapr-core-repository
+++ b/elements/hadoop-mapr/install.d/43-mapr-core-repository
@ -8,8 +8,6 @@ set -o pipefail

 echo "START: installing MapR core repository"

-install-packages curl
-
 MAPR_REPO_DIR=/opt/mapr-repository/core

 if [ "$DISTRO_NAME" = "ubuntu" ]; then
--- a/elements/hadoop-mapr/package-installs.yaml
+++ b/elements/hadoop-mapr/package-installs.yaml
@ -0,0 +1,2 @@
+curl:
+wget:
--- a/elements/hadoop/element-deps
+++ b/elements/hadoop/element-deps
@ -1,4 +1,5 @@
 cache-url
 java
+package-installs
 sahara-version
 ssh
--- a/elements/hadoop/package-installs.yaml
+++ b/elements/hadoop/package-installs.yaml
@ -0,0 +1,4 @@
+wget:
+  phase: post-install.d
+tar:
+  phase: post-install.d
--- a/elements/hadoop/post-install.d/40-setup-hadoop
+++ b/elements/hadoop/post-install.d/40-setup-hadoop
@ -13,8 +13,6 @@ fi
 set -eu
 set -o pipefail

-install-packages wget tar
-
 function install_hadoop_v1 {
    case "$DISTRO_NAME" in
        ubuntu )
--- a/elements/hive/element-deps
+++ b/elements/hive/element-deps
@ -1,3 +1,4 @@
 cache-url
 hadoop
 mysql
+package-installs
--- a/elements/hive/package-installs.yaml
+++ b/elements/hive/package-installs.yaml
@ -0,0 +1,2 @@
+tar:
+  phase: post-install.d
--- a/elements/hive/post-install.d/60-hive
+++ b/elements/hive/post-install.d/60-hive
@ -6,8 +6,6 @@ fi
 set -eu
 set -o pipefail

-install-packages tar
-
 tmp_dir=/tmp/hive
 cd $tmp_dir

--- a/elements/oozie/element-deps
+++ b/elements/oozie/element-deps
@ -1,2 +1,3 @@
 cache-url
+package-installs
 sahara-version
--- a/elements/oozie/install.d/50-setup-oozie
+++ b/elements/oozie/install.d/50-setup-oozie
@ -8,8 +8,6 @@ set -o pipefail

 echo "Oozie setup"

-install-packages zip unzip tar
-
 tmp_dir=/tmp/oozie

 if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
--- a/elements/oozie/package-installs.yaml
+++ b/elements/oozie/package-installs.yaml
@ -0,0 +1,5 @@
+zip:
+unzip:
+tar:
+wget:
+  phase: post-install.d
--- a/elements/oozie/post-install.d/90-oozie-war
+++ b/elements/oozie/post-install.d/90-oozie-war
@ -6,8 +6,6 @@ fi
 set -eu
 set -o pipefail

-install-packages wget
-
 ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/libtools/mysql.jar
 ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar

--- a/elements/redhat-lsb/element-deps
+++ b/elements/redhat-lsb/element-deps
@ -0,0 +1 @@
+package-installs
--- a/elements/redhat-lsb/package-installs.yaml
+++ b/elements/redhat-lsb/package-installs.yaml
@ -0,0 +1,2 @@
+redhat-lsb:
+  phase: pre-install.d
--- a/elements/redhat-lsb/pre-install.d/02-lsb
+++ b/elements/redhat-lsb/pre-install.d/02-lsb
@ -1,9 +0,0 @@
-#!/bin/bash
-
-if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
-install-packages redhat-lsb
--- a/elements/root-passwd/element-deps
+++ b/elements/root-passwd/element-deps
@ -0,0 +1 @@
+package-installs
--- a/elements/root-passwd/package-installs.yaml
+++ b/elements/root-passwd/package-installs.yaml
@ -0,0 +1,6 @@
+augeas-tools:
+  phase: post-install.d
+openssh-server:
+  phase: post-install.d
+openssh-client:
+  phase: post-install.d
--- a/elements/root-passwd/post-install.d/99-setup
+++ b/elements/root-passwd/post-install.d/99-setup
@ -11,7 +11,6 @@ if [ -z "$DIB_PASSWORD" ]; then
    exit 1
 fi
 sed -i "s/disable_root: true/disable_root: false/" /etc/cloud/cloud.cfg
-install-packages augeas-tools openssh-server openssh-client
 augtool -s set /files/etc/ssh/sshd_config/PasswordAuthentication yes
 augtool -s set /files/etc/ssh/sshd_config/PermitRootLogin yes
 augtool -s set /files/etc/ssh/ssh_config/PasswordAuthentication yes
--- a/elements/spark/element-deps
+++ b/elements/spark/element-deps
@ -1,2 +1,3 @@
 cache-url
 hadoop-cdh
+package-installs
--- a/elements/spark/install.d/60-spark
+++ b/elements/spark/install.d/60-spark
@ -8,8 +8,6 @@ fi
 set -eu
 set -o pipefail

-install-packages tar
-
 tmp_dir=/tmp/spark
 pushd $tmp_dir

--- a/elements/spark/package-installs.yaml
+++ b/elements/spark/package-installs.yaml
@ -0,0 +1 @@
+tar:
--- a/elements/ssh/element-deps
+++ b/elements/ssh/element-deps
@ -0,0 +1 @@
+package-installs
--- a/elements/ssh/install.d/33-ssh
+++ b/elements/ssh/install.d/33-ssh
@ -7,12 +7,6 @@ set -u
 set -o pipefail
 echo "Begin: install and configure SSH"

-# /etc/ssh/sshd_config is provided by openssh-server
-# /etc/ssh/ssh_config is provided by openssh-client
-# Note: You need diskimage-builder w/ SHA 82eacdec (11 July 2013) for
-#       this install to work on Fedora - https://review.openstack.org/#/c/36739/
-install-packages augeas-tools openssh-server openssh-client
-
 augtool -s set /files/etc/ssh/sshd_config/GSSAPIAuthentication no
 augtool -s set /files/etc/ssh/sshd_config/UseDNS no
 augtool -s set /files/etc/ssh/sshd_config/PermitTunnel yes
--- a/elements/ssh/package-installs.yaml
+++ b/elements/ssh/package-installs.yaml
@ -0,0 +1,5 @@
+augeas-tools:
+# /etc/ssh/sshd_config is provided by openssh-server
+openssh-server:
+# /etc/ssh/ssh_config is provided by openssh-client
+openssh-client:
--- a/elements/storm/element-deps
+++ b/elements/storm/element-deps
@ -1,2 +1,3 @@
 cache-url
+package-installs
 zookeeper
--- a/elements/storm/install.d/60-storm
+++ b/elements/storm/install.d/60-storm
@ -8,8 +8,6 @@ fi
 set -u
 set -o pipefail

-install-packages wget unzip build-essential uuid-dev git libtool autoconf pkg-config tar
-
 tmp_dir=/tmp/storm

 echo "Creating Storm User"
--- a/elements/storm/package-installs.yaml
+++ b/elements/storm/package-installs.yaml
@ -0,0 +1,8 @@
+unzip:
+build-essential:
+uuid-dev:
+git:
+libtool:
+autoconf:
+pkg-config:
+tar:
--- a/elements/swift_hadoop/element-deps
+++ b/elements/swift_hadoop/element-deps
@ -0,0 +1 @@
+package-installs
--- a/elements/swift_hadoop/package-installs.yaml
+++ b/elements/swift_hadoop/package-installs.yaml
@ -0,0 +1,2 @@
+wget:
+  phase: post-install.d
--- a/elements/swift_hadoop/post-install.d/81-add-jar
+++ b/elements/swift_hadoop/post-install.d/81-add-jar
@ -6,8 +6,6 @@ fi
 set -eu
 set -o pipefail

-install-packages wget
-
 HDFS_LIB_DIR=${DIB_HDFS_LIB_DIR:-"/usr/share/hadoop/lib"}
 SWIFT_LIB_URI="http://sahara-files.mirantis.com/hadoop-swift/hadoop-swift-latest.jar"

--- a/elements/zookeeper/element-deps
+++ b/elements/zookeeper/element-deps
@ -0,0 +1 @@
+package-installs
--- a/elements/zookeeper/install.d/40-zookeeper
+++ b/elements/zookeeper/install.d/40-zookeeper
@ -8,8 +8,6 @@ fi
 set -eu
 set -o pipefail

-install-packages wget
-
 echo "Downloading Zookeeper"

 mkdir -p /opt/zookeeper
--- a/elements/zookeeper/package-installs.yaml
+++ b/elements/zookeeper/package-installs.yaml
@ -0,0 +1 @@
+wget: