Poll for final state on test_copy_image_revert_lifecycle()

This test currently simulates failure by pre-deleting the store
directory for 'file3' which is the second of a two-store import
operation. The goal is to assert that the later failure reverts
the import of the earlier 'file2' store. However, the way the
polling loop works is that we break out once 'file2' has completed,
and then assume that 'file3' has already failed and reverted.
This is a race, and one we're losing consistently in CI.

This patch waits for the failure of 'file3' to be reported, as
well as the revert of 'file2' to occur before exiting the polling
loop and checking the final state of things to ensure that the
revert has actually happened. This addresses the non-determinism
inherent in the original test.

Change-Id: I11c7edaefc96236d2757acfb70d9c338c0f51348
(cherry picked from commit 6c96319eeb)
This commit is contained in:
Dan Smith 2020-08-12 09:22:30 -07:00
parent 541c0fd61e
commit ebeb31e636
2 changed files with 53 additions and 12 deletions

View File

@ -17,6 +17,7 @@ import six
import time import time
from oslo_serialization import jsonutils from oslo_serialization import jsonutils
from oslo_utils import timeutils
import requests import requests
from six.moves import http_client as http from six.moves import http_client as http
@ -129,3 +130,38 @@ def wait_for_copying(request_path, request_headers, stores=[],
entity_id = request_path.rsplit('/', 1)[1] entity_id = request_path.rsplit('/', 1)[1]
msg = "Entity {0} failed to copy image to stores '{1}' within {2} sec" msg = "Entity {0} failed to copy image to stores '{1}' within {2} sec"
raise Exception(msg.format(entity_id, ",".join(stores), max_sec)) raise Exception(msg.format(entity_id, ",".join(stores), max_sec))
def poll_entity(url, headers, callback, max_sec=10, delay_sec=0.2,
require_success=True):
"""Poll a given URL passing the parsed entity to a callback.
This is a utility method that repeatedly GETs a URL, and calls
a callback with the result. The callback determines if we should
keep polling by returning True (up to the timeout).
:param url: The url to fetch
:param headers: The request headers to use for the fetch
:param callback: A function that takes the parsed entity and is expected
to return True if we should keep polling
:param max_sec: The overall timeout before we fail
:param delay_sec: The time between fetches
:param require_success: Assert resp_code is http.OK each time before
calling the callback
"""
timer = timeutils.StopWatch(max_sec)
timer.start()
while not timer.expired():
resp = requests.get(url, headers=headers)
if require_success and resp.status_code != http.OK:
raise Exception(
'Received %i response from server' % resp.status_code)
entity = resp.json()
keep_polling = callback(entity)
if keep_polling is not True:
return keep_polling
time.sleep(delay_sec)
raise Exception('Poll timeout if %i seconds exceeded!' % max_sec)

View File

@ -5588,19 +5588,24 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest):
response = requests.post(path, headers=headers, data=data) response = requests.post(path, headers=headers, data=data)
self.assertEqual(http.ACCEPTED, response.status_code) self.assertEqual(http.ACCEPTED, response.status_code)
# Verify image is copied def poll_callback(image):
# NOTE(abhishekk): As import is a async call we need to provide # NOTE(danms): We need to wait for the specific
# some timelap to complete the call. # arrangement we're expecting, which is that file3 has
path = self._url('/v2/images/%s' % image_id) # failed, nothing else is importing, and file2 has been
func_utils.wait_for_copying(request_path=path, # removed from stores by the revert.
request_headers=self._headers(), return not (image['os_glance_importing_to_stores'] == '' and
stores=['file2'], image['os_glance_failed_import'] == 'file3' and
max_sec=10, image['stores'] == 'file1')
delay_sec=0.2,
start_delay_sec=1,
failure_scenario=True)
# Ensure data is not deleted from existing stores on failure func_utils.poll_entity(self._url('/v2/images/%s' % image_id),
self._headers(),
poll_callback)
# Here we check that the failure of 'file3' caused 'file2' to
# be removed from image['stores'], and that 'file3' is reported
# as failed in the appropriate status list. Since the import
# started with 'store1' being populated, that should remain,
# but 'store2' should be reverted/removed.
path = self._url('/v2/images/%s' % image_id) path = self._url('/v2/images/%s' % image_id)
response = requests.get(path, headers=self._headers()) response = requests.get(path, headers=self._headers())
self.assertEqual(http.OK, response.status_code) self.assertEqual(http.OK, response.status_code)