Fail Fuel Master deployment if puppet or check fails

Puppet apply for Fuel Master deployment runs with
--detailed-exitcodes in order to capture any failures
and fails if any resource fails to apply, including
docker containers.
Cleaned up some obsolete lines about hiera and astute.yaml.

Fixed automated environment deployment scripts to detect
failed deployments and fail as well.

Change-Id: I29238a27f1335d63a6c82274a6dab9b25e1ecf0a
Related-Bug: #1390129

fuelweb_test/models/environment.py

@@ -449,10 +449,15 @@ class EnvironmentModel(object):
         wait(
             lambda: not
             self.get_admin_remote().execute(
-                "grep 'Fuel node deployment complete' '%s'" % log_path
+                "grep 'Fuel node deployment' '%s'" % log_path
             )['exit_code'],
             timeout=(float(settings.PUPPET_TIMEOUT))
         )
+        result = self.get_admin_remote().execute("grep 'Fuel node deployment "
+                                                 "complete' '%s'" % log_path
+                                                 )['exit_code']
+        if result != 0:
+            raise Exception('Fuel node deployment failed.')
 
     def dhcrelay_check(self):
         admin_remote = self.get_admin_remote()

iso/bootstrap_admin_node.docker.sh

@@ -9,6 +9,11 @@ function countdown() {
   done
 }
 
+function fail() {
+  echo "ERROR: Fuel node deployment FAILED! Check /var/log/puppet/bootstrap_admin_node.log for details" 1>&2
+  exit 1
+}
+# LANG variable is a workaround for puppet-3.4.2 bug. See LP#1312758 for details
 export LANG=en_US.UTF8
 showmenu="no"
 if [ -f /root/.showfuelmenu ]; then
@@ -46,8 +51,6 @@ fi
 . /etc/sysconfig/network
 hostname "$HOSTNAME"
 
-# ruby21-hiera RPM does not include /var/lib/hiera/ directory which may cause errors
-
 ### docker stuff
 images_dir="/var/www/nailgun/docker/images"
 
@@ -69,14 +72,13 @@ for image in $images_dir/*tar ; do
     rm -f "$image"
 done
 
-#TODO(mattymo,LP#1313288) Write astute.yaml to /etc/fuel from fuelmenu
-cp -a /etc/astute.yaml /etc/fuel/astute.yaml
-
 # apply puppet
-# LANG variable is a workaround for puppet-3.4.2 bug. See LP#1312758 for details
-puppet apply -d -v /etc/puppet/modules/nailgun/examples/host-only.pp
+puppet apply --detailed-exitcodes -d -v /etc/puppet/modules/nailgun/examples/host-only.pp
+if [ $? -ge 4 ];then
+  fail
+fi
 rmdir /var/log/remote && ln -s /var/log/docker-logs/remote /var/log/remote
 
-dockerctl check
+dockerctl check || fail
 bash /etc/rc.local
 echo "Fuel node deployment complete!"

virtualbox/actions/master-node-create-and-install.sh

@@ -52,3 +52,5 @@ enable_outbound_network_for_product_vm $vm_master_ip $vm_master_username $vm_mas
 echo
 echo "Master node has been installed."
 
+#Sleep 10s to wait for Cobbler to settle
+sleep 10

virtualbox/functions/product.sh

@@ -1,4 +1,4 @@
-#!/bin/bash 
+#!/bin/bash
 
 #    Copyright 2013 Mirantis, Inc.
 #
@@ -27,7 +27,7 @@ is_product_vm_operational() {
     prompt=$4
 
     # Log in into the VM, see if Puppet has completed its run
-    # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line 
+    # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line
     result=$(
         expect << ENDOFEXPECT
         spawn ssh $ssh_options $username@$ip
@@ -35,7 +35,7 @@ is_product_vm_operational() {
         expect "*?assword:*"
         send "$password\r"
         expect "$prompt"
-        send "grep -o 'Fuel node deployment complete' /var/log/puppet/bootstrap_admin_node.log\r"
+        send "grep 'Fuel node deployment' /var/log/puppet/bootstrap_admin_node.log\r"
         expect "$prompt"
 ENDOFEXPECT
     )
@@ -51,10 +51,14 @@ ENDOFEXPECT
 
     for line in $result; do
         IFS="${OIFS}"
-        if [[ $line == Fuel* ]]; then
-	    IFS="${NIFS}"
+        if [[ "$line" == Fuel*complete* ]]; then
+            IFS="${NIFS}"
             return 0;
-        fi    
+        elif [[ "$line" == Fuel*FAILED* ]]; then
+            IFS="${NIFS}"
+            echo "$line" 1>&2
+            exit 1
+        fi
         IFS="${NIFS}"
     done
 
@@ -70,8 +74,14 @@ wait_for_product_vm_to_install() {
     echo "Waiting for product VM to install. Please do NOT abort the script..."
 
     # Loop until master node gets successfully installed
+    maxdelay=3000
     while ! is_product_vm_operational $ip $username $password "$prompt"; do
         sleep 5
+        ((waited += 5))
+        if (( waited >= maxdelay )); then
+          echo "Installation timed out! ($maxdelay seconds)" 1>&2
+          exit 1
+        fi
     done
 }
 
@@ -110,12 +120,12 @@ enable_outbound_network_for_product_vm() {
     else
       echo "OK"
     fi
-    
+
     # Enable internet access on inside the VMs
     echo -n "Enabling outbound network/internet access for the product VM... "
 
     # Log in into the VM, configure and bring up the NAT interface, set default gateway, check internet connectivity
-    # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line 
+    # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line
     result=$(
         expect << ENDOFEXPECT
         spawn ssh $ssh_options $username@$ip