openstack/nova
Code Issues Proposed changes

XenAPI: Check image status before uploading data

Browse Source 
Currently, the Xen API plugin sends the data payload to Glance
without checking the image status. In consequence, if the image is
not in queued status Glance will not let the image upload succeed.
This is by design, please refer [0].

The issue here being; buffering of the data will happen on the Glance
server. Consequently, the bound to be failed upload operation will not
get a 409 until after a large chunk of data has already been streamed
to the Glance server. This may result into client side wastage of
bandwidth and wait time.

This patch proposes a HEAD call be made from the glance plugin to
Glance API; thus in turn checking the valid image state for the
data transfer.

[0] http://docs.openstack.org/developer/glance/statuses.html

Change-Id: I24c1222a3b1ac809353724500117e58d50615ac8
This commit is contained in:
Nikhil Komawar
2014-11-21 19:54:58 -05:00
parent 43e89abeb2
commit 9211822923
1 changed files with 118 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
View File

@@ -157,6 +157,9 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port,
raise RetryableError(error)
try:
validate_image_status_before_upload(conn, glance_host, glance_port,
image_id, url, extra_headers)
try:
# NOTE(sirp): httplib under python2.4 won't accept
# a file-like object to request
@@ -227,60 +230,125 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port,
"Response Status: %i, Response body: %s"
% (url, resp.status, resp.read()))
# Note(Jesse): This branch sorts errors into those that are permanent,
# those that are ephemeral, and those that are unexpected.
if resp.status in (httplib.BAD_REQUEST, # 400
httplib.UNAUTHORIZED, # 401
httplib.PAYMENT_REQUIRED, # 402
httplib.FORBIDDEN, # 403
httplib.NOT_FOUND, # 404
httplib.METHOD_NOT_ALLOWED, # 405
httplib.NOT_ACCEPTABLE, # 406
httplib.PROXY_AUTHENTICATION_REQUIRED, # 407
httplib.CONFLICT, # 409
httplib.GONE, # 410
httplib.LENGTH_REQUIRED, # 411
httplib.PRECONDITION_FAILED, # 412
httplib.REQUEST_ENTITY_TOO_LARGE, # 413
httplib.REQUEST_URI_TOO_LONG, # 414
httplib.UNSUPPORTED_MEDIA_TYPE, # 415
httplib.REQUESTED_RANGE_NOT_SATISFIABLE, # 416
httplib.EXPECTATION_FAILED, # 417
httplib.UNPROCESSABLE_ENTITY, # 422
httplib.LOCKED, # 423
httplib.FAILED_DEPENDENCY, # 424
httplib.UPGRADE_REQUIRED, # 426
httplib.NOT_IMPLEMENTED, # 501
httplib.HTTP_VERSION_NOT_SUPPORTED, # 505
httplib.NOT_EXTENDED, # 510
):
raise PluginError("Got Permanent Error response [%i] while "
"uploading image [%s] to glance host [%s:%s]"
% (resp.status, image_id,
glance_host, glance_port))
elif resp.status in (httplib.REQUEST_TIMEOUT, # 408
httplib.INTERNAL_SERVER_ERROR, # 500
httplib.BAD_GATEWAY, # 502
httplib.SERVICE_UNAVAILABLE, # 503
httplib.GATEWAY_TIMEOUT, # 504
httplib.INSUFFICIENT_STORAGE, # 507
):
raise RetryableError("Got Ephemeral Error response [%i] while "
"uploading image [%s] to glance host [%s:%s]"
% (resp.status, image_id,
glance_host, glance_port))
else:
# Note(Jesse): Assume unexpected errors are retryable. If you are
# seeing this error message, the error should probably be added
# to either the ephemeral or permanent error list.
raise RetryableError("Got Unexpected Error response [%i] while "
"uploading image [%s] to glance host [%s:%s]"
% (resp.status, image_id,
glance_host, glance_port))
check_resp_status_and_retry(resp, image_id, glance_host, glance_port)
finally:
conn.close()
def check_resp_status_and_retry(resp, image_id, glance_host, glance_port):
# Note(Jesse): This branch sorts errors into those that are permanent,
# those that are ephemeral, and those that are unexpected.
if resp.status in (httplib.BAD_REQUEST, # 400
httplib.UNAUTHORIZED, # 401
httplib.PAYMENT_REQUIRED, # 402
httplib.FORBIDDEN, # 403
httplib.NOT_FOUND, # 404
httplib.METHOD_NOT_ALLOWED, # 405
httplib.NOT_ACCEPTABLE, # 406
httplib.PROXY_AUTHENTICATION_REQUIRED, # 407
httplib.CONFLICT, # 409
httplib.GONE, # 410
httplib.LENGTH_REQUIRED, # 411
httplib.PRECONDITION_FAILED, # 412
httplib.REQUEST_ENTITY_TOO_LARGE, # 413
httplib.REQUEST_URI_TOO_LONG, # 414
httplib.UNSUPPORTED_MEDIA_TYPE, # 415
httplib.REQUESTED_RANGE_NOT_SATISFIABLE, # 416
httplib.EXPECTATION_FAILED, # 417
httplib.UNPROCESSABLE_ENTITY, # 422
httplib.LOCKED, # 423
httplib.FAILED_DEPENDENCY, # 424
httplib.UPGRADE_REQUIRED, # 426
httplib.NOT_IMPLEMENTED, # 501
httplib.HTTP_VERSION_NOT_SUPPORTED, # 505
httplib.NOT_EXTENDED, # 510
):
raise PluginError("Got Permanent Error response [%i] while "
"uploading image [%s] to glance host [%s:%s]"
% (resp.status, image_id,
glance_host, glance_port))
# NOTE(nikhil): Only a sub-set of the 500 errors are retryable. We
# optimistically retry on 500 errors below.
elif resp.status in (httplib.REQUEST_TIMEOUT, # 408
httplib.INTERNAL_SERVER_ERROR, # 500
httplib.BAD_GATEWAY, # 502
httplib.SERVICE_UNAVAILABLE, # 503
httplib.GATEWAY_TIMEOUT, # 504
httplib.INSUFFICIENT_STORAGE, # 507
):
raise RetryableError("Got Ephemeral Error response [%i] while "
"uploading image [%s] to glance host [%s:%s]"
% (resp.status, image_id,
glance_host, glance_port))
else:
# Note(Jesse): Assume unexpected errors are retryable. If you are
# seeing this error message, the error should probably be added
# to either the ephemeral or permanent error list.
raise RetryableError("Got Unexpected Error response [%i] while "
"uploading image [%s] to glance host [%s:%s]"
% (resp.status, image_id,
glance_host, glance_port))
def validate_image_status_before_upload(conn, glance_host, glance_port,
image_id, url, extra_headers):
try:
# NOTE(nikhil): Attempt to determine if the Image has a status
# of 'queued'. Because data will continued to be sent to Glance
# until it has a chance to check the Image state, discover that
# it is not 'active' and send back a 409. Hence, the data will be
# unnecessarily buffered by Glance. This wastes time and bandwidth.
# LP bug #1202785
conn.request('HEAD', '/v1/images/%s' % image_id,
headers=extra_headers)
head_resp = conn.getresponse()
# NOTE(nikhil): read the response to re-use the conn object.
body_data = head_resp.read(8192)
if len(body_data) > 8:
err_msg = ('Cannot upload data for image %(image_id)s as the '
'HEAD call had more than 8192 bytes of data in '
'the response body.' % {'image_id': image_id})
raise PluginError("Got Permanent Error while uploading image "
"[%s] to glance host [%s:%s]. "
"Message: %s" % (image_id, glance_host,
glance_port, err_msg))
else:
head_resp.read()
except Exception, error:
logging.exception('Failed to HEAD the image %(image_id)s while '
'checking image status before attempting to '
'upload %(url)s' % {'image_id': image_id,
'url': url})
raise RetryableError(error)
if head_resp.status != httplib.OK:
logging.error("Unexpected response while doing a HEAD call "
"to image %s , url = %s , Response Status: "
"%i" % (image_id, url, head_resp.status))
check_resp_status_and_retry(head_resp, image_id, glance_host,
glance_port)
else:
image_status = head_resp.getheader('x-image-meta-status')
if image_status not in ('queued', ):
err_msg = ('Cannot upload data for image %(image_id)s as the '
'image status is %(image_status)s' %
{'image_id': image_id, 'image_status': image_status})
logging.exception(err_msg)
raise PluginError("Got Permanent Error while uploading image "
"[%s] to glance host [%s:%s]. "
"Message: %s" % (image_id, glance_host,
glance_port, err_msg))
else:
logging.info('Found image %(image_id)s in status '
'%(image_status)s. Attempting to '
'upload.' % {'image_id': image_id,
'image_status': image_status})
def download_vhd(session, image_id, glance_host, glance_port, glance_use_ssl,
uuid_stack, sr_path, extra_headers):
"""Download an image from Glance, unbundle it, and then deposit the VHDs