From 46537e0c4b5486688592a3f71edbd2f7e8a7bf20 Mon Sep 17 00:00:00 2001
From: Christian Schwede <cschwede@redhat.com>
Date: Fri, 1 Sep 2017 10:54:59 +0200
Subject: [PATCH] Retry Swift ring up-/downloads on failures

The current way never retries up- or downloading the rings from the
undercloud. This is risky, for example when there are temporary network
issues.

This patch adds retries to the curl commands. It does so in two ways:

1. curl retries up to 3 times if there is an HTTP error. This could
happen for example if the proxy is accessible, but the proxy itself
can't connect to a backend server. Note that curl returns an exit code
of 0 if there is an HTTP error, so curl itself needs to retry in this
case.

2. If curl fails hard (for example due to an network error, proxy being
down etc) the whole command will be re-executed up to 3 times by Puppet
itself. The default timeout has been set to 30 seconds instead of the
default timeout of 300 seconds.

Change-Id: I21f732c1afa9d472f4a2fb840b6ddad9b8d65d07
---
 manifests/profile/base/swift/ringbuilder.pp            | 10 +++++++---
 .../notes/swift-ring-curl-retry-1c329d1808b7f02c.yaml  |  8 ++++++++
 2 files changed, 15 insertions(+), 3 deletions(-)
 create mode 100644 releasenotes/notes/swift-ring-curl-retry-1c329d1808b7f02c.yaml

diff --git a/manifests/profile/base/swift/ringbuilder.pp b/manifests/profile/base/swift/ringbuilder.pp
index 27cf4fe15..3c7b8df90 100644
--- a/manifests/profile/base/swift/ringbuilder.pp
+++ b/manifests/profile/base/swift/ringbuilder.pp
@@ -92,8 +92,10 @@ class tripleo::profile::base::swift::ringbuilder (
   if $step >= 2 and $swift_ring_get_tempurl != '' {
     exec{'fetch_swift_ring_tarball':
       path    => ['/usr/bin'],
-      command => "curl --insecure --silent '${swift_ring_get_tempurl}' -o /tmp/swift-rings.tar.gz",
-      returns => [0, 3]
+      command => "curl --insecure --silent --retry 3 '${swift_ring_get_tempurl}' -o /tmp/swift-rings.tar.gz",
+      returns => [0, 3],
+      timeout => 30,
+      tries   => 3,
     }
     ~> exec{'extract_swift_ring_tarball':
       path    => ['/bin'],
@@ -155,9 +157,11 @@ class tripleo::profile::base::swift::ringbuilder (
     }
     exec{'upload_swift_ring_tarball':
       path        => ['/usr/bin'],
-      command     => "curl --insecure --silent -X PUT '${$swift_ring_put_tempurl}' --data-binary @/tmp/swift-rings.tar.gz",
+      command     => "curl --insecure --silent --retry 3 -X PUT '${$swift_ring_put_tempurl}' --data-binary @/tmp/swift-rings.tar.gz",
       require     => Exec['create_swift_ring_tarball'],
       refreshonly => true,
+      timeout     => 30,
+      tries       => 3,
     }
 
     Exec['rebalance_account'] ~> Exec['create_swift_ring_tarball']
diff --git a/releasenotes/notes/swift-ring-curl-retry-1c329d1808b7f02c.yaml b/releasenotes/notes/swift-ring-curl-retry-1c329d1808b7f02c.yaml
new file mode 100644
index 000000000..9d3e2bc49
--- /dev/null
+++ b/releasenotes/notes/swift-ring-curl-retry-1c329d1808b7f02c.yaml
@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    Retry Swift ring up-/downloads on failures to improve overall
+    stability during deployments when there are temporary errors.
+    Retries are executed in case of HTTP errors (for example due to a
+    temporary issue between the proxy and backend servers) as well as
+    connection issue to the proxy itself.