mirror-update: stats for vos release of mirrors

It's difficult to know if a release process is running too long when
we don't have a history of how long it should run for.

This is mostly the stats function from run_all.sh that has been
sending stats about runtimes there.  Wrap it in a vos_release function
with some minor refactoring, and update the scripts.

As noted inline, there's already release timer stats going to
afs.release.<volume> for the periodic release of docs/tarballs etc.

Change-Id: I3d79d1a0997af8977050b7f6e7cf3b7578cc8491

playbooks/roles/mirror-update/files/centos-mirror-update

@@ -93,7 +93,7 @@ done
 
 date --iso-8601=ns | $K5START tee $BASE/timestamp.txt
 echo "rsync completed successfully, running vos release."
-$VOS_RELEASE $MIRROR_VOLUME
+vos_release $MIRROR_VOLUME
 
 date --iso-8601=ns
 echo "Done."

playbooks/roles/mirror-update/files/epel-mirror-update

@@ -90,7 +90,7 @@ echo "$SHA1SUM $BASE/$REPO/$REPOMD" | sha1sum -c -
 
 date --iso-8601=ns | $K5START tee $BASE/timestamp.txt
 echo "rsync completed successfully, running vos release."
-$VOS_RELEASE $MIRROR_VOLUME
+vos_release $MIRROR_VOLUME
 
 date --iso-8601=ns
 echo "Done."

playbooks/roles/mirror-update/files/fedora-mirror-update

@@ -153,7 +153,7 @@ sleep $(( 20 * 60 ))
 date --iso-8601=ns | $K5START tee $BASE/timestamp.txt
 
 echo_ts "Running vos release."
-$VOS_RELEASE $MIRROR_VOLUME | \
+vos_release $MIRROR_VOLUME | \
     while IFS= read -r line; do echo_ts "$line"; done
 echo_ts "... done"
 

playbooks/roles/mirror-update/files/functions.sh

@@ -2,4 +2,47 @@
 # Common definitions and functions for mirror-update scripts
 #
 
-VOS_RELEASE="ssh -i /root/.ssh/id_vos_release vos_release@afs01.dfw.openstack.org vos release "
+AFS_SERVER=afs01.dfw.openstack.org
+VOS_RELEASE="ssh -i /root/.ssh/id_vos_release vos_release@${AFS_SERVER} vos release "
+
+# Send a timer stat to statsd
+#  send_timer metric [start_time]
+# * uses timer metric afs.release.<$1> normalised for stats
+# * time will be taken from last call of start_timer, or $2 if set
+function send_timer {
+    # Only send stats under cron conditions
+    if [[ ${UNDER_CRON} != 1 ]]; then
+        return
+    fi
+
+    local current
+    current=$(date '+%s')
+    local name
+    # "." is a separator, replace with _
+    name=${1//./_}
+    local start
+    start=${2-$_START_TIME}
+    local elapsed_ms
+    elapsed_ms=$(( (current - start) * 1000 ))
+
+    # See also the release-volumes.py script which sends stats to the
+    # same place for doc, etc. volumes.
+    echo "afs.release.${name}:${elapsed_ms}|ms" | nc -w 1 -u graphite.opendev.org 8125
+    echo "End timer for $name"
+}
+
+# See send_timer
+function start_timer {
+    _START_TIME=$(date '+%s')
+}
+
+# Run vos release via ssh on the AFS server, and report the timing
+# back to graphite
+function vos_release {
+    local name
+    name=$1
+
+    start_timer
+    $VOS_RELEASE ${name}
+    send_timer ${name}
+}

playbooks/roles/mirror-update/files/opensuse-mirror-update

@@ -133,7 +133,7 @@ $K5START rsync -rlptDvz \
 
 date --iso-8601=ns | $K5START tee $BASE/timestamp.txt
 echo "rsync completed successfully, running vos release."
-$VOS_RELEASE $MIRROR_VOLUME
+vos_release $MIRROR_VOLUME
 
 date --iso-8601=ns
 echo "Done."

playbooks/roles/mirror-update/files/yum-puppetlabs-mirror-update

@@ -60,7 +60,7 @@ $K5START rsync -rlptDvz \
 
 date --iso-8601=ns | $K5START tee $BASE/timestamp.txt
 echo "rsync completed successfully, running vos release."
-$VOS_RELEASE $MIRROR_VOLUME
+vos_release $MIRROR_VOLUME
 
 date --iso-8601=ns
 echo "Done."

playbooks/roles/mirror-update/tasks/rsync.yaml

@@ -55,6 +55,12 @@
     mode: '0755'
   loop: '{{ rsync_update_scripts }}'
 
+- name: Set cron flag to enable reporting stats
+  cron:
+    name: UNDER_CRON
+    env: yes
+    job: '1'
+
 - name: Install update cron jobs
   cron:
     name: '{{ item }} mirror sync'
@@ -70,4 +76,4 @@
   vars:
     logrotate_file_name: '/var/log/rsync-mirrors/{{ item }}.log'
     logrotate_frequency: 'weekly'
-  loop: '{{ rsync_update_scripts }}'
+  loop: '{{ rsync_update_scripts }}'