Make install-distro-packages.sh more robust

Introduce a retry loop in install-distro-packages.sh which will
attempt index updates and package installation up to 3 times if the
bindep list is unsatisfied, so as to mitigate intermittent network
failures. Add a clear error message after the last try indicating
the list of packages which were not installed.

Change-Id: I595aedbc57623ee1b30bd5880abf699f95e9eb63

jenkins/scripts/install-distro-packages.sh

@@ -15,13 +15,29 @@ else
     export PACKAGES=/usr/local/jenkins/common_data/bindep-fallback.txt
 fi
 
-# install all requested packages from the appropriate bindep list
-if apt-get -v >/dev/null ; then
-    sudo apt-get update
-    sudo PATH=/usr/sbin:/sbin:$PATH DEBIAN_FRONTEND=noninteractive \
-        apt-get --option "Dpkg::Options::=--force-confold" \
-        --assume-yes install `$BINDEP -b -f $PACKAGES || true`
-else
-    sudo PATH=/usr/sbin:/sbin:$PATH yum install -y \
-        `$BINDEP -b -f $PACKAGES || true`
-fi
+# an install loop, retrying to check that all requested packages are obtained
+try=0
+until $BINDEP -b -f $PACKAGES ; do
+    if [ $try -gt 2 ] ; then
+        set +x
+        echo -e "\nERROR: These requested packages were not installed:\n" \
+            "\n`$BINDEP -b -f $PACKAGES`\n" 1>&2
+        set -x
+        exit 1
+    fi
+
+    # don't abort inside the loop, we check for the desired outcome
+    set +e
+    if apt-get -v >/dev/null ; then
+        sudo apt-get update
+        sudo PATH=/usr/sbin:/sbin:$PATH DEBIAN_FRONTEND=noninteractive \
+            apt-get --option "Dpkg::Options::=--force-confold" \
+            --assume-yes install `$BINDEP -b -f $PACKAGES`
+    else
+        sudo PATH=/usr/sbin:/sbin:$PATH yum install -y \
+            `$BINDEP -b -f $PACKAGES`
+    fi
+    set -e
+
+    try=$(( $try+1 ))
+done