Don't spam postci with deployment time loop

We have a lot of resources in the overcloud Heat stack now, which
is causing a lot of noise in the logs.  This patch does a couple
of things to clean that up:

-Shuts off tracing while we're processing deployment times.  A
 couple of log messages should suffice to let people know what is
 going on at that time.
-Limits the number of resources we send to graphite to the top 50.
 There's no need to be logging times for resources that only take
 a few seconds, and sending the full list to graphite causes the
 UI to be really slow loading the list of data points.

Change-Id: I07c648761d81350d611257769c1d33803a8f7553

scripts/common_functions.sh

@@ -216,12 +216,20 @@ function postci(){
         # post metrics
         scp $SSH_OPTIONS root@${SEED_IP}:${METRICS_DATA_FILE} /tmp/seed-metrics
         cat /tmp/seed-metrics >> ${METRICS_DATA_FILE}
+        # This spams the postci output with largely uninteresting trace output
+        set +x
+        echo -n 'Recording Heat deployment times...'
+        # We're only interested in the resources that take the most time
+        long_times=/tmp/long-deploy-times.log
+        head -n 50 $WORKSPACE/logs/undercloud/var/log/heat-deploy-times.log > $long_times
         while read line; do
             # $line should look like "ResourceName 123.0", so concatenating all
             # of this together we should end up with a call that looks like:
             # record_metric tripleo.overcloud.ha.resources.ResourceName 123.0
             record_metric tripleo.overcloud.${TOCI_JOBTYPE}.resources.${line}
-        done <$WORKSPACE/logs/undercloud/var/log/heat-deploy-times.log
+        done <$long_times
+        echo 'Finished'
+        set -x
         metrics_to_graphite "23.253.94.71" #Dan's temp graphite server
         if [ -z "${LEAVE_RUNNING:-}" ] && [ -n "${HOST_IP:-}" ] ; then
             destroy_vms &> $WORKSPACE/logs/destroy_vms.log

scripts/metrics.bash

@@ -48,6 +48,9 @@ function stop_metric {
 }
 
 function metrics_to_graphite {
+    # With a lot of metrics, the trace output gets pretty spammy
+    set +x
+    echo "Sending metrics to graphite"
     local SERVER=$1
     local PORT=${2:-2003} # default port for graphite data
 
@@ -60,10 +63,12 @@ function metrics_to_graphite {
         METRIC_NAME=$(echo $X | cut -d ":" -f 1)
         METRIC_VAL=$(echo $X | cut -d ":" -f 2)
         DTS=$(echo $X | cut -d ":" -f 3)
+        echo "Sending $METRIC_NAME to Graphite"
         echo "$METRIC_NAME $METRIC_VAL $DTS" | nc ${SERVER} ${PORT}
     done
     set -e
     # reset the existing data file and start times
     echo "" > METRICS_START_TIMES
     echo "" > METRICS_DATA_FILE
+    set -x
 }