Browse Source

Poll for final state on test_copy_image_revert_lifecycle()

This test currently simulates failure by pre-deleting the store
directory for 'file3' which is the second of a two-store import
operation. The goal is to assert that the later failure reverts
the import of the earlier 'file2' store. However, the way the
polling loop works is that we break out once 'file2' has completed,
and then assume that 'file3' has already failed and reverted.
This is a race, and one we're losing consistently in CI.

This patch waits for the failure of 'file3' to be reported, as
well as the revert of 'file2' to occur before exiting the polling
loop and checking the final state of things to ensure that the
revert has actually happened. This addresses the non-determinism
inherent in the original test.

Change-Id: I11c7edaefc96236d2757acfb70d9c338c0f51348
(cherry picked from commit 6c96319eeb)
Dan Smith 1 month ago
2 changed files with 54 additions and 13 deletions
  1. +36
  2. +18

+ 36
- 0
glance/tests/functional/ View File

@@ -17,6 +17,7 @@ import six
import time

from oslo_serialization import jsonutils
from oslo_utils import timeutils
import requests
from six.moves import http_client as http

@@ -129,3 +130,38 @@ def wait_for_copying(request_path, request_headers, stores=[],
entity_id = request_path.rsplit('/', 1)[1]
msg = "Entity {0} failed to copy image to stores '{1}' within {2} sec"
raise Exception(msg.format(entity_id, ",".join(stores), max_sec))

def poll_entity(url, headers, callback, max_sec=10, delay_sec=0.2,
"""Poll a given URL passing the parsed entity to a callback.

This is a utility method that repeatedly GETs a URL, and calls
a callback with the result. The callback determines if we should
keep polling by returning True (up to the timeout).

:param url: The url to fetch
:param headers: The request headers to use for the fetch
:param callback: A function that takes the parsed entity and is expected
to return True if we should keep polling
:param max_sec: The overall timeout before we fail
:param delay_sec: The time between fetches
:param require_success: Assert resp_code is http.OK each time before
calling the callback

timer = timeutils.StopWatch(max_sec)

while not timer.expired():
resp = requests.get(url, headers=headers)
if require_success and resp.status_code != http.OK:
raise Exception(
'Received %i response from server' % resp.status_code)
entity = resp.json()
keep_polling = callback(entity)
if keep_polling is not True:
return keep_polling

raise Exception('Poll timeout if %i seconds exceeded!' % max_sec)

+ 18
- 13
glance/tests/functional/v2/ View File

@@ -5588,19 +5588,24 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest):
response =, headers=headers, data=data)
self.assertEqual(http.ACCEPTED, response.status_code)

# Verify image is copied
# NOTE(abhishekk): As import is a async call we need to provide
# some timelap to complete the call.
path = self._url('/v2/images/%s' % image_id)

# Ensure data is not deleted from existing stores on failure
def poll_callback(image):
# NOTE(danms): We need to wait for the specific
# arrangement we're expecting, which is that file3 has
# failed, nothing else is importing, and file2 has been
# removed from stores by the revert.
return not (image['os_glance_importing_to_stores'] == '' and
image['os_glance_failed_import'] == 'file3' and
image['stores'] == 'file1')

func_utils.poll_entity(self._url('/v2/images/%s' % image_id),

# Here we check that the failure of 'file3' caused 'file2' to
# be removed from image['stores'], and that 'file3' is reported
# as failed in the appropriate status list. Since the import
# started with 'store1' being populated, that should remain,
# but 'store2' should be reverted/removed.
path = self._url('/v2/images/%s' % image_id)
response = requests.get(path, headers=self._headers())
self.assertEqual(http.OK, response.status_code)