Fix apt-get update and retries

Decrease amount of expensive apt operations and
wrap in retries.
Use jessie backports instead of testing that causes
key/signing errors.
Drop testing mirror/pinning from packer scripts

Change-Id: I5e0f1a64c8b68997d20e4b8f02733d1034f01a77
Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

utils/jenkins/files/debian_backports_repo.list

@@ -0,0 +1 @@
+deb http://httpredir.debian.org/debian jessie-backports main

utils/jenkins/files/debian_pinning

@@ -1,15 +1,15 @@
 Package: ansible
-Pin: release a=testing
+Pin: release a=jessie-backports
 Pin-Priority: 1001
 
 Package: python-setuptools
-Pin: release a=testing
+Pin: release a=jessie-backports
 Pin-Priority: 1001
 
 Package: python-pkg-resources
-Pin: release a=testing
+Pin: release a=jessie-backports
 Pin-Priority: 1001
 
 Package: *
-Pin: release a=testing
+Pin: release a=jessie-backports
 Pin-Priority: 100

utils/jenkins/files/debian_testing_repo.list

@@ -1,2 +0,0 @@
-deb http://http.us.debian.org/debian testing main
-deb-src http://http.us.debian.org/debian testing main

utils/jenkins/kargo_deploy.sh

@@ -47,6 +47,19 @@ function exit_gracefully {
     exit $exit_code
 }
 
+function with_retries {
+    set +e
+    local retries=3
+    for try in 1..$retries; do
+        ${@}
+        [ $? -eq 0 ] && break
+        if [[ "$try" == "$retries" ]]; then
+            exit 1
+        fi
+    done
+    set -e
+}
+
 mkdir -p tmp logs
 
 
@@ -115,8 +128,6 @@ for slaveip in ${SLAVE_IPS[@]}; do
     # Workaround to disable ipv6 dns which can cause docker pull to fail
     echo "precedence ::ffff:0:0/96  100" | ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo sh -c 'cat - >> /etc/gai.conf'"
 
-    # Requirements for ansible
-    ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python-netaddr"
 
     # Workaround to fix DNS search domain: https://github.com/kubespray/kargo/issues/322
     ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo DEBIAN_FRONTEND=noninteractive apt-get remove -y resolvconf"
@@ -132,35 +143,27 @@ for slaveip in ${SLAVE_IPS[@]}; do
     ((current_slave++))
 done
 
-echo "Setting up required dependencies..."
-ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP sudo apt-get update
-ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP sudo apt-get install -y git vim software-properties-common
-
-echo "Setting up ansible..."
+echo "Setting up required dependencies and ansible..."
 case $NODE_BASE_OS in
     ubuntu)
-        set +e
-        ppa_retries=3
-        for try in 1..$ppa_retries; do
-            ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP 'sudo sh -c "apt-add-repository -y ppa:ansible/ansible; apt-get update"' && break
-            if [[ "$try" == "$ppa_retries" ]]; then
-                exit 1
-            fi
-        done
-        set -e
+        with_retries ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP -- sudo apt-add-repository -y ppa:ansible/ansible
+        with_retries ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP -- sudo apt-get update
     ;;
     debian)
         for slaveip in ${SLAVE_IPS[@]}; do
-            scp $SSH_OPTIONS ${BASH_SOURCE%/*}/files/debian_testing_repo.list $ADMIN_USER@$slaveip:/tmp/testing.list
-            ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo cp -f /tmp/testing.list /etc/apt/sources.list.d/testing.list"
-            scp $SSH_OPTIONS ${BASH_SOURCE%/*}/files/debian_pinning $ADMIN_USER@$slaveip:/tmp/testing
-            ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo cp -f /tmp/testing /etc/apt/preferences.d/testing"
-            echo "Upgrading setuptools"
-            ssh $SSH_OPTIONS $ADMIN_USER@$slaveip 'sudo sh -c "apt-get update; apt-get -y install --only-upgrade python-setuptools"'
+            scp $SSH_OPTIONS ${BASH_SOURCE%/*}/files/debian_backports_repo.list $ADMIN_USER@$slaveip:/tmp/backports.list
+            ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo cp -f /tmp/backports.list /etc/apt/sources.list.d/backports.list"
+            scp $SSH_OPTIONS ${BASH_SOURCE%/*}/files/debian_pinning $ADMIN_USER@$slaveip:/tmp/backports
+            ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo cp -f /tmp/backports /etc/apt/preferences.d/backports"
+            with_retries ssh $SSH_OPTIONS $ADMIN_USER@$slaveip -- sudo apt-get update
+            with_retries ssh $SSH_OPTIONS $ADMIN_USER@$slaveip -- sudo apt-get -y install --only-upgrade python-setuptools
         done
     ;;
 esac
-ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP 'sudo sh -c "apt-get install -y ansible"'
+for slaveip in ${SLAVE_IPS[@]}; do
+    with_retries ssh $SSH_OPTIONS $ADMIN_USER@$slaveip -- sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python-netaddr
+done
+with_retries ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP -- sudo apt-get install -y git vim software-properties-common ansible
 
 echo "Checking out kargo playbook..."
 ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP git clone $KARGO_REPO

utils/packer/scripts/debian/packages.sh

@@ -1,28 +1,4 @@
 #!/bin/bash -eux
-#FIXME(bogdando) switch to jessie-backports
-cat > /etc/apt/preferences.d/testing << EOF
-Package: ansible
-Pin: release a=testing
-Pin-Priority: 1001
-
-Package: python-setuptools
-Pin: release a=testing
-Pin-Priority: 1001
-
-Package: python-pkg-resources
-Pin: release a=testing
-Pin-Priority: 1001
-
-Package: *
-Pin: release a=testing
-Pin-Priority: 100
-EOF
-
-cat > /etc/apt/sources.list.d/testing.list << EOF
-deb http://http.us.debian.org/debian testing main
-deb-src http://http.us.debian.org/debian testing main
-EOF
-
 apt-get -y update
 apt-get -y dist-upgrade
 
@@ -49,7 +25,7 @@ python-setuptools
 "
 
 echo "==> Installing packages"
-apt-get -y --allow-unauthenticated install $PACKAGES
+apt-get -y install $PACKAGES
 
 # Upgrading pip
 pip install --upgrade pip