Merge "Adds retry on upload_vhd for xapi glance plugin"

This commit is contained in:
Jenkins 2013-03-04 16:54:13 +00:00 committed by Gerrit Code Review
commit b3e097a0f4
3 changed files with 69 additions and 24 deletions
nova
exception.py
virt/xenapi/imageupload
plugins/xenserver/xenapi/etc/xapi.d/plugins

@ -1089,6 +1089,10 @@ class CouldNotFetchImage(NovaException):
message = _("Could not fetch image %(image_id)s") message = _("Could not fetch image %(image_id)s")
class CouldNotUploadImage(NovaException):
message = _("Could not upload image %(image_id)s")
class TaskAlreadyRunning(NovaException): class TaskAlreadyRunning(NovaException):
message = _("Task %(task_name)s is already running on host %(host)s") message = _("Task %(task_name)s is already running on host %(host)s")

@ -13,8 +13,11 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import time
from oslo.config import cfg from oslo.config import cfg
from nova import exception
from nova.image import glance from nova.image import glance
import nova.openstack.common.log as logging import nova.openstack.common.log as logging
from nova.virt.xenapi import vm_utils from nova.virt.xenapi import vm_utils
@ -22,6 +25,7 @@ from nova.virt.xenapi import vm_utils
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
CONF = cfg.CONF CONF = cfg.CONF
CONF.import_opt('glance_num_retries', 'nova.image.glance')
class GlanceStore(object): class GlanceStore(object):
@ -32,24 +36,49 @@ class GlanceStore(object):
""" """
# NOTE(sirp): Currently we only support uploading images as VHD, there # NOTE(sirp): Currently we only support uploading images as VHD, there
# is no RAW equivalent (yet) # is no RAW equivalent (yet)
LOG.debug(_("Asking xapi to upload to glance %(vdi_uuids)s as" max_attempts = CONF.glance_num_retries + 1
" ID %(image_id)s"), locals(), instance=instance) sleep_time = 0.5
glance_api_servers = glance.get_api_servers() glance_api_servers = glance.get_api_servers()
glance_host, glance_port, glance_use_ssl = glance_api_servers.next()
properties = { properties = {
'auto_disk_config': instance['auto_disk_config'], 'auto_disk_config': instance['auto_disk_config'],
'os_type': instance['os_type'] or CONF.default_os_type, 'os_type': instance['os_type'] or CONF.default_os_type,
} }
params = {'vdi_uuids': vdi_uuids, for attempt_num in xrange(1, max_attempts + 1):
'image_id': image_id,
'glance_host': glance_host,
'glance_port': glance_port,
'glance_use_ssl': glance_use_ssl,
'sr_path': vm_utils.get_sr_path(session),
'auth_token': getattr(context, 'auth_token', None),
'properties': properties}
session.call_plugin_serialized('glance', 'upload_vhd', **params) (glance_host,
glance_port,
glance_use_ssl) = glance_api_servers.next()
try:
params = {'vdi_uuids': vdi_uuids,
'image_id': image_id,
'glance_host': glance_host,
'glance_port': glance_port,
'glance_use_ssl': glance_use_ssl,
'sr_path': vm_utils.get_sr_path(session),
'auth_token': getattr(context, 'auth_token', None),
'properties': properties}
LOG.debug(_("Asking xapi to upload to glance %(vdi_uuids)s as"
" ID %(image_id)s"
" glance server: %(glance_host)s:%(glance_port)d"
" attempt %(attempt_num)d/%(max_attempts)d"),
locals(), instance=instance)
return session.call_plugin_serialized('glance',
'upload_vhd',
**params)
except session.XenAPI.Failure as exc:
_type, _method, error = exc.details[:3]
if error == 'RetryableError':
LOG.error(_('upload_vhd failed: %r') %
(exc.details[3:],))
else:
raise
time.sleep(sleep_time)
sleep_time = min(2 * sleep_time, 15)
raise exception.CouldNotUploadImage(image_id=image_id)

@ -126,14 +126,18 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port,
url = '%s://%s:%s/v1/images/%s' % (scheme, glance_host, glance_port, url = '%s://%s:%s/v1/images/%s' % (scheme, glance_host, glance_port,
image_id) image_id)
logging.info("Writing image data to %s" % url) logging.info("Writing image data to %s" % url)
if glance_use_ssl:
conn = httplib.HTTPSConnection(glance_host, glance_port)
else:
conn = httplib.HTTPConnection(glance_host, glance_port)
# NOTE(sirp): httplib under python2.4 won't accept a file-like object try:
# to request if glance_use_ssl:
conn.putrequest('PUT', '/v1/images/%s' % image_id) conn = httplib.HTTPSConnection(glance_host, glance_port)
else:
conn = httplib.HTTPConnection(glance_host, glance_port)
# NOTE(sirp): httplib under python2.4 won't accept a file-like object
# to request
conn.putrequest('PUT', '/v1/images/%s' % image_id)
except Exception, error:
raise RetryableError(error)
# NOTE(sirp): There is some confusion around OVF. Here's a summary of # NOTE(sirp): There is some confusion around OVF. Here's a summary of
# where we currently stand: # where we currently stand:
@ -172,12 +176,18 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port,
def send_chunked_transfer_encoded(chunk): def send_chunked_transfer_encoded(chunk):
chunk_len = len(chunk) chunk_len = len(chunk)
callback_data['bytes_written'] += chunk_len callback_data['bytes_written'] += chunk_len
conn.send("%x\r\n%s\r\n" % (chunk_len, chunk)) try:
conn.send("%x\r\n%s\r\n" % (chunk_len, chunk))
except Exception, error:
raise RetryableError(error)
utils.create_tarball( utils.create_tarball(
None, staging_path, callback=send_chunked_transfer_encoded) None, staging_path, callback=send_chunked_transfer_encoded)
conn.send("0\r\n\r\n") # Chunked-Transfer terminator try:
conn.send("0\r\n\r\n") # Chunked-Transfer terminator
except Exception, error:
raise RetryableError(error)
bytes_written = callback_data['bytes_written'] bytes_written = callback_data['bytes_written']
logging.info("Wrote %d bytes to %s" % (bytes_written, url)) logging.info("Wrote %d bytes to %s" % (bytes_written, url))
@ -187,9 +197,11 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port,
logging.error("Unexpected response while writing image data to %s: " logging.error("Unexpected response while writing image data to %s: "
"Response Status: %i, Response body: %s" "Response Status: %i, Response body: %s"
% (url, resp.status, resp.read())) % (url, resp.status, resp.read()))
raise Exception("Unexpected response [%i] while uploading image [%s] " raise RetryableError("Unexpected response [%i] while uploading "
"image [%s] "
"to glance host [%s:%s]" "to glance host [%s:%s]"
% (resp.status, image_id, glance_host, glance_port)) % (resp.status, image_id, glance_host, glance_port))
conn.close() conn.close()