Adding support for retrying glance image downloads.
Change-Id: Ifff40d90f7dc61a6d41ae2d6908d6e1e6f0aea7e
This commit is contained in:
parent
5e9e3873e5
commit
1b7fba648a
|
@ -297,6 +297,8 @@ DEFINE_integer('glance_port', 9292, 'default glance port')
|
|||
DEFINE_list('glance_api_servers',
|
||||
['%s:%d' % (FLAGS.glance_host, FLAGS.glance_port)],
|
||||
'list of glance api servers available to nova (host:port)')
|
||||
DEFINE_integer('glance_num_retries', 0,
|
||||
'The number of times to retry downloading an image from glance')
|
||||
DEFINE_integer('s3_port', 3333, 's3 port')
|
||||
DEFINE_string('s3_host', '$my_ip', 's3 host (for infrastructure)')
|
||||
DEFINE_string('s3_dmz', '$my_ip', 's3 dmz ip (for instances)')
|
||||
|
|
|
@ -23,6 +23,7 @@ import copy
|
|||
import datetime
|
||||
import json
|
||||
import random
|
||||
import time
|
||||
from urlparse import urlparse
|
||||
|
||||
from glance.common import exception as glance_exception
|
||||
|
@ -231,11 +232,18 @@ class GlanceImageService(object):
|
|||
|
||||
def get(self, context, image_id, data):
|
||||
"""Calls out to Glance for metadata and data and writes data."""
|
||||
try:
|
||||
num_retries = FLAGS.glance_num_retries
|
||||
for count in xrange(1 + num_retries):
|
||||
client = self._get_client(context)
|
||||
image_meta, image_chunks = client.get_image(image_id)
|
||||
except glance_exception.NotFound:
|
||||
raise exception.ImageNotFound(image_id=image_id)
|
||||
try:
|
||||
image_meta, image_chunks = client.get_image(image_id)
|
||||
break
|
||||
except glance_exception.NotFound:
|
||||
raise exception.ImageNotFound(image_id=image_id)
|
||||
except Exception:
|
||||
if count == num_retries:
|
||||
raise
|
||||
time.sleep(1)
|
||||
|
||||
for chunk in image_chunks:
|
||||
data.write(chunk)
|
||||
|
|
|
@ -508,6 +508,35 @@ class TestGlanceImageService(test.TestCase):
|
|||
self.assertEqual(image_meta['created_at'], self.NOW_DATETIME)
|
||||
self.assertEqual(image_meta['updated_at'], self.NOW_DATETIME)
|
||||
|
||||
def test_get_with_retries(self):
|
||||
tries = [0]
|
||||
|
||||
class GlanceBusyException(Exception):
|
||||
pass
|
||||
|
||||
class MyGlanceStubClient(glance_stubs.StubGlanceClient):
|
||||
"""A client that fails the first time, then succeeds."""
|
||||
def get_image(self, image_id):
|
||||
if tries[0] == 0:
|
||||
tries[0] = 1
|
||||
raise GlanceBusyException()
|
||||
else:
|
||||
return {}, []
|
||||
|
||||
client = MyGlanceStubClient()
|
||||
service = glance.GlanceImageService(client=client)
|
||||
image_id = 1 # doesn't matter
|
||||
writer = NullWriter()
|
||||
|
||||
# When retries are disabled, we should get an exception
|
||||
self.flags(glance_num_retries=0)
|
||||
self.assertRaises(GlanceBusyException, service.get, self.context,
|
||||
image_id, writer)
|
||||
|
||||
# Now lets enable retries. No exception should happen now.
|
||||
self.flags(glance_num_retries=1)
|
||||
service.get(self.context, image_id, writer)
|
||||
|
||||
def test_glance_client_image_id(self):
|
||||
fixture = self._make_fixture(name='test image')
|
||||
image_id = self.service.create(self.context, fixture)['id']
|
||||
|
|
|
@ -582,6 +582,7 @@ w
|
|||
'glance_port': glance_port,
|
||||
'uuid_stack': uuid_stack,
|
||||
'sr_path': cls.get_sr_path(session),
|
||||
'num_retries': FLAGS.glance_num_retries,
|
||||
'auth_token': getattr(context, 'auth_token', None)}
|
||||
|
||||
kwargs = {'params': pickle.dumps(params)}
|
||||
|
|
|
@ -33,6 +33,7 @@ import shlex
|
|||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import XenAPIPlugin
|
||||
|
||||
|
@ -67,7 +68,7 @@ def _copy_kernel_vdi(dest, copy_args):
|
|||
|
||||
|
||||
def _download_tarball(sr_path, staging_path, image_id, glance_host,
|
||||
glance_port, auth_token):
|
||||
glance_port, auth_token, num_retries):
|
||||
"""Download the tarball image from Glance and extract it into the staging
|
||||
area.
|
||||
"""
|
||||
|
@ -77,12 +78,17 @@ def _download_tarball(sr_path, staging_path, image_id, glance_host,
|
|||
headers['x-auth-token'] = auth_token
|
||||
|
||||
conn = httplib.HTTPConnection(glance_host, glance_port)
|
||||
conn.request('GET', '/v1/images/%s' % image_id, headers=headers)
|
||||
resp = conn.getresponse()
|
||||
if resp.status == httplib.NOT_FOUND:
|
||||
raise Exception("Image '%s' not found in Glance" % image_id)
|
||||
elif resp.status != httplib.OK:
|
||||
raise Exception("Unexpected response from Glance %i" % resp.status)
|
||||
|
||||
for count in xrange(1 + num_retries):
|
||||
conn.request('GET', '/v1/images/%s' % image_id, headers=headers)
|
||||
resp = conn.getresponse()
|
||||
if resp.status == httplib.OK:
|
||||
break
|
||||
elif resp.status == httplib.NOT_FOUND:
|
||||
raise Exception("Image '%s' not found in Glance" % image_id)
|
||||
elif count == num_retries:
|
||||
raise Exception("Unexpected response from Glance %i" % resp.status)
|
||||
time.sleep(1)
|
||||
|
||||
tar_cmd = "tar -zx --directory=%(staging_path)s" % locals()
|
||||
tar_proc = _make_subprocess(tar_cmd, stderr=True, stdin=True)
|
||||
|
@ -404,11 +410,12 @@ def download_vhd(session, args):
|
|||
uuid_stack = params["uuid_stack"]
|
||||
sr_path = params["sr_path"]
|
||||
auth_token = params["auth_token"]
|
||||
num_retries = params["num_retries"]
|
||||
|
||||
staging_path = _make_staging_area(sr_path)
|
||||
try:
|
||||
_download_tarball(sr_path, staging_path, image_id, glance_host,
|
||||
glance_port, auth_token)
|
||||
glance_port, auth_token, num_retries)
|
||||
# Right now, it's easier to return a single string via XenAPI,
|
||||
# so we'll json encode the list of VHDs.
|
||||
return json.dumps(_import_vhds(sr_path, staging_path, uuid_stack))
|
||||
|
|
Loading…
Reference in New Issue