Update packer build for debian-8.5

Add cgroups setup for grub.
Do Docker/CM prerequisites and versions pinning in the
deploy scripts, not the image build time.
Use regular partitioning for the image.

Change-Id: I326c8cfc155d517bbaaf2ffb1f9611519dec7151
Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

.gitreview

@@ -1,4 +1,4 @@
 [gerrit]
-host=review.fuel-infra.org
+host=review.openstack.org
 port=29418
-project=nextgen/mcpinstaller.git
+project=openstack/fuel-ccp-installer.git

utils/jenkins/files/debian_pinning

@@ -0,0 +1,15 @@
+Package: ansible
+Pin: release a=testing
+Pin-Priority: 1001
+
+Package: python-setuptools
+Pin: release a=testing
+Pin-Priority: 1001
+
+Package: python-pkg-resources
+Pin: release a=testing
+Pin-Priority: 1001
+
+Package: *
+Pin: release a=testing
+Pin-Priority: 100

utils/jenkins/files/debian_testing_repo.list

@@ -0,0 +1,2 @@
+deb http://http.us.debian.org/debian testing main
+deb-src http://http.us.debian.org/debian testing main

utils/jenkins/kargo_deploy.sh

@@ -15,6 +15,9 @@ else
 fi
 
 IMAGE_PATH=${IMAGE_PATH:-bootstrap/output-qemu/ubuntu1404}
+# detect OS type from the image name, assume debian by default
+NODE_BASE_OS=$(basename ${IMAGE_PATH} | grep -io -e ubuntu -e debian)
+NODE_BASE_OS="${NODE_BASE_OS:-debian}"
 DEPLOY_TIMEOUT=${DEPLOY_TIMEOUT:-60}
 
 SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
@@ -100,12 +103,27 @@ done
 
 echo "Setting up required dependencies..."
 ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP sudo apt-get update
-ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP sudo apt-get install -y git python-setuptools python-dev python-pip gcc libssl-dev libffi-dev vim software-properties-common
-ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP "sudo pip install 'cryptography>=1.3.2'"
-ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP "sudo pip install 'cffi>=1.6.0'"
+ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP sudo apt-get install -y git python-dev python-pip gcc libssl-dev libffi-dev vim software-properties-common
+ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP "sudo easy_install setuptools"
+ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP "sudo pip install 'cryptography>=1.3.2 'cffi>=1.6.0'"
 
 echo "Setting up ansible..."
-ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP 'sudo sh -c "apt-add-repository -y ppa:ansible/ansible;apt-get update;apt-get install -y ansible"'
+case $NODE_BASE_OS in
+    ubuntu)
+        ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP 'sudo sh -c "apt-add-repository -y ppa:ansible/ansible; apt-get update"'
+    ;;
+    debian)
+        for slaveip in ${SLAVE_IPS[@]}; do
+            scp $SSH_OPTIONS ${BASH_SOURCE%/*}/files/debian_testing_repo.list $ADMIN_USER@$slaveip:/tmp/testing.list
+            ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo cp -f /tmp/testing.list /etc/apt/sources.list.d/testing.list"
+            scp $SSH_OPTIONS ${BASH_SOURCE%/*}/files/debian_pinning $ADMIN_USER@$slaveip:/tmp/testing
+            ssh $SSH_OPTIONS $ADMIN_USER@$slaveip "sudo cp -f /tmp/testing /etc/apt/preferences.d/testing"
+            echo "Upgrading setuptools"
+            ssh $SSH_OPTIONS $ADMIN_USER@$slaveip 'sudo sh -c "apt-get update; apt-get -y install --only-upgrade python-setuptools"'
+        done
+    ;;
+esac
+ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP 'sudo sh -c "apt-get install -y ansible"'
 
 echo "Setting up kargo-cli..."
 ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP git clone https://github.com/kubespray/kargo-cli.git
@@ -166,7 +184,7 @@ if [ "$VLAN_BRIDGE" ] && [ "${deploy_res}" -eq "0" ] && [ "${DONT_DESTROY_ON_SUC
         bridged_iface_mac="`ENV_NAME=${ENV_NAME} python ${BASH_SOURCE%/*}/env.py get_bridged_iface_mac $IP`"
 
         sshpass -p ${ADMIN_PASSWORD} ssh ${SSH_OPTIONS} ${ADMIN_USER}@${IP} bash -s <<EOF >>VLAN_IPS
-bridged_iface=\$(ifconfig -a|awk -v mac="$bridged_iface_mac" '\$0 ~ mac {print \$1}' 'RS=\n\n')
+bridged_iface=\$(/sbin/ifconfig -a|awk -v mac="$bridged_iface_mac" '\$0 ~ mac {print \$1}' 'RS=\n\n')
 sudo ip route del default
 sudo dhclient "\${bridged_iface}"
 echo \$(ip addr list |grep ${bridged_iface_mac} -A 1 |grep 'inet ' |cut -d' ' -f6| cut -d/ -f1)

utils/packer/debian8.5/debian-8.5-amd64.json

@@ -1,5 +1,6 @@
 {
     "variables": {
+        "name": "debian-85-jessie",
         "user": "vagrant",
         "password": "vagrant",
         "disk_size": "100000"
@@ -8,7 +9,7 @@
     "builders":
     [
         {
-            "name": "debian-85-jessie",
+            "name": "{{user `name`}}",
 
             "iso_checksum_type": "none",
             "iso_url": "http://cdimage.debian.org/cdimage/release/8.5.0/amd64/iso-cd/debian-8.5.0-amd64-netinst.iso",
@@ -60,5 +61,12 @@
                 "scripts/cleanup.sh"
             ]
         }
+    ],
+
+    "post-processors": [
+        {
+            "type": "shell-local",
+            "inline": ["qemu-img convert -c -f qcow2 -O qcow2 -o cluster_size=2M ./output-{{user `name`}}/packer-{{user `name`}} ./output-{{user `name`}}/packer-{{user `name`}}.qcow2"]
+        }
     ]
 }

utils/packer/debian8.5/http/preseed_jessie.cfg

@@ -14,15 +14,19 @@ d-i mirror/http/hostname string http.debian.net
 d-i mirror/http/proxy string
 d-i partman-auto-lvm/guided_size string max
 d-i partman-auto/choose_recipe select atomic
-d-i partman-auto/method string lvm
-d-i partman-lvm/confirm boolean true
-d-i partman-lvm/confirm boolean true
-d-i partman-lvm/confirm_nooverwrite boolean true
+d-i partman-auto/method string regular
 d-i partman-lvm/device_remove_lvm boolean true
+d-i partman-md/device_remove_md boolean true
+d-i partman-lvm/confirm boolean true
 d-i partman/choose_partition select finish
 d-i partman/confirm boolean true
 d-i partman/confirm_nooverwrite boolean true
 d-i partman/confirm_write_new_label boolean true
+d-i partman-partitioning/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i partman-basicfilesystems/no_mount_point yes
 d-i passwd/root-login boolean false
 d-i passwd/root-password-again password vagrant
 d-i passwd/root-password password vagrant
@@ -31,14 +35,10 @@ d-i passwd/user-uid string 900
 d-i passwd/user-password password vagrant
 d-i passwd/user-password-again password vagrant
 d-i passwd/username string vagrant
-d-i pkgsel/include string openssh-server sudo bzip2 acpid cryptsetup zlib1g-dev wget curl dkms make nfs-common
+d-i pkgsel/include string openssh-server sudo
 d-i pkgsel/install-language-support boolean false
 d-i pkgsel/update-policy select unattended-upgrades
 d-i pkgsel/upgrade select full-upgrade
-# Prevent packaged version of VirtualBox Guest Additions being installed:
-d-i preseed/early_command string sed -i \
-  '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \
-  /usr/lib/pre-pkgsel.d/20install-hwpackages
 d-i time/zone string UTC
 d-i user-setup/allow-password-weak boolean true
 d-i user-setup/encrypt-home boolean false

utils/packer/debian8.5/scripts/cleanup.sh

@@ -1,5 +1,28 @@
 #!/bin/bash -eux
 
-# Apt cleanup.
-apt autoremove
-apt update
+apt-get -y autoremove --purge
+find /var/cache -type f -exec rm -rf {} \;
+find /var/lib/apt -type f | xargs rm -f
+
+rm -rf /dev/.udev/
+rm -f /lib/udev/rules.d/75-persistent-net-generator.rules
+rm -f /etc/udev/rules.d/70-persistent-net.rules
+mkdir -p /etc/udev/rules.d/70-persistent-net.rules
+
+if [ -d "/var/lib/dhcp" ]; then
+    rm -f /var/lib/dhcp/*
+fi
+
+rm -rf /tmp/*
+
+unset HISTFILE
+rm -f /root/.bash_history
+rm -f /home/vagrant/.bash_history
+
+find /var/log -type f | while read f; do echo -ne '' > $f; done;
+
+>/var/log/lastlog
+>/var/log/wtmp
+>/var/log/btmp
+
+sync

utils/packer/debian8.5/scripts/setup.sh

@@ -3,9 +3,11 @@
 # configure serial console:
 cat >> /etc/default/grub <<EOF
 GRUB_TERMINAL=serial
-GRUB_CMDLINE_LINUX='console=tty0 console=ttyS0,19200n8'
+# Cgroups fix for debian 8
+GRUB_CMDLINE_LINUX='console=tty0 console=ttyS0,19200n8 cgroup_enable=memory swapaccount=1'
 GRUB_SERIAL_COMMAND="serial --speed=19200 --unit=0 --word=8 --parity=no --stop=1"
 EOF
+update-grub2
 
 # Set up sudo
 echo 'vagrant ALL=NOPASSWD:ALL' > /etc/sudoers.d/vagrant
@@ -62,4 +64,4 @@ exit 0
 EOF
 
 # Set vim as a default editor
-update-alternatives --set editor /usr/bin/vim.basic
+update-alternatives --set editor /usr/bin/vim.basic