From 9ee3e76a66a9f5fd535de8031a7585dff6ea6f77 Mon Sep 17 00:00:00 2001
From: Jeremy Stanley <fungi@yuggoth.org>
Date: Thu, 18 Feb 2016 21:09:16 +0000
Subject: [PATCH] Retry twine uploads in a loop

Now that we have a mechanism to check whether twine uploaded
successfully, we can also safely retry it when it doesn't. Try to
upload to PyPI up to three times, waiting progressively longer
between retries.

Change-Id: Id16909501d8ed2387daae71f75cf80de28297baf
---
 jenkins/scripts/pypi-tarball-upload.sh | 21 ++++++++++++++++++---
 jenkins/scripts/pypi-wheel-upload.sh   | 21 ++++++++++++++++++---
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/jenkins/scripts/pypi-tarball-upload.sh b/jenkins/scripts/pypi-tarball-upload.sh
index ac482ee876..cbcfc232cb 100755
--- a/jenkins/scripts/pypi-tarball-upload.sh
+++ b/jenkins/scripts/pypi-tarball-upload.sh
@@ -34,6 +34,21 @@ curl --fail -o $FILENAME http://$TARBALL_SITE/$PROJECT/$FILENAME
 file -b $FILENAME | grep gzip
 
 # Uploads may claim to fail but actually succeed so we check if we
-# can download after upload to determine success.
-twine upload -r pypi $FILENAME || true
-curl --head --silent --fail "https://pypi.python.org/simple/$PROJECT/$FILENAME" >/dev/null 2>&1
+# can download after upload to determine success. They can also fail
+# intermittently, so retrying in a delayed loop helps improve
+# robustness.
+TRY=0
+RETVAL=255
+set +e
+while [[ $TRY -lt 3 ]] && [[ $RETVAL -ne 0 ]]; do
+    twine upload -r pypi $FILENAME
+    curl --head --silent --fail \
+        "https://pypi.python.org/simple/$PROJECT/$FILENAME" >/dev/null 2>&1
+    RETVAL=$?
+    (( TRY++ ))
+    if [[ $TRY -lt 3 ]] && [[ $RETVAL -ne 0 ]]; then
+        echo "Upload failed, retrying in $TRY seconds." >&2
+        sleep $TRY
+    fi
+done
+exit $RETVAL
diff --git a/jenkins/scripts/pypi-wheel-upload.sh b/jenkins/scripts/pypi-wheel-upload.sh
index bf1bbeb11a..572e9d9229 100755
--- a/jenkins/scripts/pypi-wheel-upload.sh
+++ b/jenkins/scripts/pypi-wheel-upload.sh
@@ -36,6 +36,21 @@ curl --fail -o $FILENAME http://$TARBALL_SITE/$PROJECT/$FILENAME
 file -b $FILENAME | grep -i zip
 
 # Uploads may claim to fail but actually succeed so we check if we
-# can download after upload to determine success.
-twine upload -r pypi $FILENAME || true
-curl --head --silent --fail "https://pypi.python.org/simple/$PROJECT/$FILENAME" >/dev/null 2>&1
+# can download after upload to determine success. They can also fail
+# intermittently, so retrying in a delayed loop helps improve
+# robustness.
+TRY=0
+RETVAL=255
+set +e
+while [[ $TRY -lt 3 ]] && [[ $RETVAL -ne 0 ]]; do
+    twine upload -r pypi $FILENAME
+    curl --head --silent --fail \
+        "https://pypi.python.org/simple/$PROJECT/$FILENAME" >/dev/null 2>&1
+    RETVAL=$?
+    (( TRY++ ))
+    if [[ $TRY -lt 3 ]] && [[ $RETVAL -ne 0 ]]; then
+        echo "Upload failed, retrying in $TRY seconds." >&2
+        sleep $TRY
+    fi
+done
+exit $RETVAL