zuul
/
zuul-jobs
Code Issues Proposed changes

Retry more operations 

This adds a generic retry handler that takes a callable, and uses
that not only for the exsting retries around the POST call, but
also the actions when creating the container.

Change-Id: I910b8e886f107d4fe38a9334ba836f010f92557c
This commit is contained in:
James E. Blair 2019-08-14 13:51:42 -07:00
parent e5897bdd44
commit 8595026f46
1 changed files with 50 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
roles/upload-logs-swift/library/zuul_swift_upload.py
View File

@ -141,6 +141,18 @@ def get_cloud(cloud):
return openstack.connect(cloud=cloud) return openstack.connect(cloud=cloud)
def retry_function(func):
for attempt in range(1, POST_ATTEMPTS + 1):
try:
return func()
except Exception:
if attempt >= POST_ATTEMPTS:
raise
else:
logging.exception("Error on attempt %d" % attempt)
time.sleep(attempt * 10)
def sizeof_fmt(num, suffix='B'): def sizeof_fmt(num, suffix='B'):
# From http://stackoverflow.com/questions/1094841/ # From http://stackoverflow.com/questions/1094841/
# reusable-library-to-get-human-readable-version-of-file-size # reusable-library-to-get-human-readable-version-of-file-size
@ -491,30 +503,36 @@ class Uploader():
cdn_url = None cdn_url = None
if not self.cloud.get_container(self.container): if not self.cloud.get_container(self.container):
self.cloud.create_container(name=self.container, public=public) retry_function(
self.cloud.update_container( lambda: self.cloud.create_container(
name=self.container, name=self.container, public=public))
headers={'X-Container-Meta-Web-Index': 'index.html', headers = {'X-Container-Meta-Web-Index': 'index.html',
'X-Container-Meta-Access-Control-Allow-Origin': '*'}) 'X-Container-Meta-Access-Control-Allow-Origin': '*'}
retry_function(
lambda: self.cloud.update_container(
name=self.container,
headers=headers))
# 'X-Container-Meta-Web-Listings': 'true' # 'X-Container-Meta-Web-Listings': 'true'
# The ceph radosgw swift implementation requires an # The ceph radosgw swift implementation requires an
# index.html at the root in order for any other indexes to # index.html at the root in order for any other indexes to
# work. # work.
index_headers = {'access-control-allow-origin': '*'} index_headers = {'access-control-allow-origin': '*'}
self.cloud.create_object(self.container, retry_function(
name='index.html', lambda: self.cloud.create_object(self.container,
data='', name='index.html',
content_type='text/html', data='',
**index_headers) content_type='text/html',
**index_headers))
# Enable the CDN in rax # Enable the CDN in rax
if cdn_url: if cdn_url:
self.cloud.session.put(cdn_url) retry_function(lambda: self.cloud.session.put(cdn_url))
if cdn_url: if cdn_url:
endpoint = (self.cloud.session.head(cdn_url) endpoint = retry_function(
.headers['X-Cdn-Ssl-Uri']) lambda: self.cloud.session.head(
cdn_url).headers['X-Cdn-Ssl-Uri'])
container = endpoint container = endpoint
else: else:
endpoint = self.cloud.object_store.get_endpoint() endpoint = self.cloud.object_store.get_endpoint()
@ -546,7 +564,7 @@ class Uploader():
logging.debug("%s: processing job %s", logging.debug("%s: processing job %s",
threading.current_thread(), threading.current_thread(),
file_detail) file_detail)
self._post_file(file_detail) retry_function(lambda: self._post_file(file_detail))
except requests.exceptions.RequestException: except requests.exceptions.RequestException:
# Do our best to attempt to upload all the files # Do our best to attempt to upload all the files
logging.exception("Error posting file after multiple attempts") logging.exception("Error posting file after multiple attempts")
@ -584,32 +602,24 @@ class Uploader():
# This is required for Rackspace CDN # This is required for Rackspace CDN
headers['access-control-allow-origin'] = '*' headers['access-control-allow-origin'] = '*'
for attempt in range(1, POST_ATTEMPTS + 1): if not file_detail.folder:
try: if (file_detail.encoding is None and
if not file_detail.folder: self._is_text_type(file_detail.mimetype)):
if (file_detail.encoding is None and headers['content-encoding'] = 'deflate'
self._is_text_type(file_detail.mimetype)): data = DeflateFilter(open(file_detail.full_path, 'rb'))
headers['content-encoding'] = 'deflate' else:
data = DeflateFilter(open(file_detail.full_path, 'rb')) if file_detail.encoding:
else: headers['content-encoding'] = file_detail.encoding
if file_detail.encoding: data = open(file_detail.full_path, 'rb')
headers['content-encoding'] = file_detail.encoding else:
data = open(file_detail.full_path, 'rb') data = ''
else: relative_path = relative_path.rstrip('/')
data = '' if relative_path == '':
relative_path = relative_path.rstrip('/') relative_path = '/'
if relative_path == '': self.cloud.create_object(self.container,
relative_path = '/' name=relative_path,
self.cloud.create_object(self.container, data=data,
name=relative_path, **headers)
data=data,
**headers)
break
except requests.exceptions.RequestException:
logging.exception(
"File posting error on attempt %d" % attempt)
if attempt >= POST_ATTEMPTS:
raise
def run(cloud, container, files, def run(cloud, container, files,