Validate NTP servers

Adds a validation to ensure at least one NTP source
is available.

Misconfigured or inaccessible NTP servers is a
common source of erratic behavior and failures. This
validation will ensure a NTP source is available
or fail with debug output from ntpdate.

The heat boolean: ValidateNtp can be set to disable
this check.

Change-Id: Ie93f943b53bf3a1b60a536df4a28ae203d98988f

all-nodes-validation.yaml

@@ -14,6 +14,10 @@ parameters:
     default: false
     description: Optional validation to ensure FQDN as set by Nova matches the name set in /etc/hosts.
     type: boolean
+  ValidateNtp:
+    default: true
+    description: Validation to ensure at least one time source is accessible.
+    type: boolean
 
 resources:
   AllNodesValidationsImpl:
@@ -25,6 +29,8 @@ resources:
           default: {get_param: PingTestIps}
         - name: validate_fqdn
           default: {get_param: ValidateFqdn}
+        - name: validate_ntp
+          default: {get_param: ValidateNtp}
       config: {get_file: ./validation-scripts/all-nodes.sh}
 
 outputs:

validation-scripts/all-nodes.sh

@@ -82,8 +82,38 @@ function fqdn_check() {
   echo "SUCCESS"
 }
 
+# Verify at least one time source is available.
+function ntp_check() {
+  NTP_SERVERS=$(hiera ntp::servers nil |tr -d '[],"')
+  if [[ "$NTP_SERVERS" != "nil" ]];then
+    echo -n "Testing NTP..."
+    NTP_SUCCESS=0
+    for NTP_SERVER in $NTP_SERVERS; do
+      set +e
+      NTPDATE_OUT=$(ntpdate -qud $NTP_SERVER 2>&1)
+      NTPDATE_EXIT=$?
+      set -e
+      if [[ "$NTPDATE_EXIT" == "0" ]];then
+        NTP_SUCCESS=1
+        break
+      else
+        NTPDATE_OUT_FULL="$NTPDATE_OUT_FULL $NTPDATE_OUT"
+      fi
+    done
+    if  [[ "$NTP_SUCCESS" == "0" ]];then
+      echo "FAILURE"
+      echo "$NTPDATE_OUT_FULL"
+      exit 1
+    fi
+    echo "SUCCESS"
+  fi
+}
+
 ping_controller_ips "$ping_test_ips"
 ping_default_gateways
 if [[ $validate_fqdn == "True" ]];then
   fqdn_check
 fi
+if [[ $validate_ntp == "True" ]];then
+  ntp_check
+fi