openstack/trove-tempest-plugin
Code Issues Proposed changes

Add expect_op_status for wait_for_instance_status function.

Browse Source 
We should wait until the instance’s status becomes healthy before
sending api requests.

Change-Id: I8f818b6278499ef420588b6bdcfd1734c5cf41d6
This commit is contained in:
wu.chunyang 2023-07-21 09:19:11 +08:00
parent 826cd69b85
commit 9122baa272
5 changed files with 46 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
trove_tempest_plugin/tests/base.py
View File

@ -237,7 +237,8 @@ class BaseTroveTest(test.BaseTestCase):
instance = cls.create_instance(create_user=cls.create_user) instance = cls.create_instance(create_user=cls.create_user)
cls.instance_id = instance['id'] cls.instance_id = instance['id']
cls.wait_for_instance_status(cls.instance_id) cls.wait_for_instance_status(cls.instance_id,
expected_op_status=["HEALTHY"])
cls.instance = cls.client.get_resource( cls.instance = cls.client.get_resource(
"instances", cls.instance_id)['instance'] "instances", cls.instance_id)['instance']
cls.instance_ip = cls.get_instance_ip(cls.instance) cls.instance_ip = cls.get_instance_ip(cls.instance)
@ -412,11 +413,13 @@ class BaseTroveTest(test.BaseTestCase):
{"restart": {}}, {"restart": {}},
expected_status_code=202, expected_status_code=202,
need_response=False) need_response=False)
cls.wait_for_instance_status(instance_id) cls.wait_for_instance_status(instance_id,
expected_op_status=["HEALTHY"])
@classmethod @classmethod
def wait_for_instance_status(cls, id, def wait_for_instance_status(cls, id,
expected_status=["HEALTHY", "ACTIVE"], expected_status=["ACTIVE"],
expected_op_status=[],
need_delete=False, need_delete=False,
timeout=CONF.database.database_build_timeout): timeout=CONF.database.database_build_timeout):
def _wait(): def _wait():
@ -431,7 +434,13 @@ class BaseTroveTest(test.BaseTestCase):
if cur_status in expected_status: if cur_status in expected_status:
LOG.info('Instance %s becomes %s', id, cur_status) LOG.info('Instance %s becomes %s', id, cur_status)
raise loopingcall.LoopingCallDone() if expected_op_status:
op_status = res["instance"]["operating_status"]
if op_status in expected_op_status:
raise loopingcall.LoopingCallDone()
else:
raise loopingcall.LoopingCallDone()
elif "ERROR" not in expected_status and cur_status == "ERROR": elif "ERROR" not in expected_status and cur_status == "ERROR":
# If instance status goes to ERROR but is not expected, stop # If instance status goes to ERROR but is not expected, stop
# waiting # waiting
@ -475,6 +484,9 @@ class BaseTroveTest(test.BaseTestCase):
if type(expected_status) != list: if type(expected_status) != list:
expected_status = [expected_status] expected_status = [expected_status]
if type(expected_op_status) != list:
expected_op_status = [expected_op_status]
if need_delete: if need_delete:
# If resource already removed, return # If resource already removed, return
try: try:
@ -633,7 +645,8 @@ class BaseTroveTest(test.BaseTestCase):
f"mgmt/instances/{instance_id}/action", f"mgmt/instances/{instance_id}/action",
rebuild_req, expected_status_code=202, rebuild_req, expected_status_code=202,
need_response=False) need_response=False)
cls.wait_for_instance_status(instance_id) cls.wait_for_instance_status(instance_id,
expected_op_status=["HEALTHY"])
@classmethod @classmethod
def create_config(cls, name, values, datastore, datastore_version): def create_config(cls, name, values, datastore, datastore_version):

12
trove_tempest_plugin/tests/scenario/base_actions.py
View File

@ -67,7 +67,8 @@ class TestInstanceActionsBase(trove_base.BaseTroveTest):
instance = self.create_instance(name=name, instance = self.create_instance(name=name,
datastore_version=ds_version, datastore_version=ds_version,
create_user=self.create_user) create_user=self.create_user)
self.wait_for_instance_status(instance['id']) self.wait_for_instance_status(instance['id'],
expected_op_status=["HEALTHY"])
instance = self.client.get_resource( instance = self.client.get_resource(
"instances", instance['id'])['instance'] "instances", instance['id'])['instance']
instance_ip = self.get_instance_ip(instance) instance_ip = self.get_instance_ip(instance)
@ -83,7 +84,8 @@ class TestInstanceActionsBase(trove_base.BaseTroveTest):
f"{new_version}") f"{new_version}")
body = {"instance": {"datastore_version": new_version}} body = {"instance": {"datastore_version": new_version}}
self.client.patch_resource('instances', instance['id'], body) self.client.patch_resource('instances', instance['id'], body)
self.wait_for_instance_status(instance['id']) self.wait_for_instance_status(instance['id'],
expected_op_status=["HEALTHY"])
LOG.info(f"Getting database version on {instance_ip}") LOG.info(f"Getting database version on {instance_ip}")
actual = self.get_db_version(instance_ip) actual = self.get_db_version(instance_ip)
@ -110,7 +112,8 @@ class TestInstanceActionsBase(trove_base.BaseTroveTest):
self.client.create_resource(f"instances/{self.instance_id}/action", self.client.create_resource(f"instances/{self.instance_id}/action",
resize_flavor, expected_status_code=202, resize_flavor, expected_status_code=202,
need_response=False) need_response=False)
self.wait_for_instance_status(self.instance_id) self.wait_for_instance_status(self.instance_id,
expected_op_status=["HEALTHY"])
# Verify Trove flavor # Verify Trove flavor
ret = self.client.get_resource('instances', self.instance_id) ret = self.client.get_resource('instances', self.instance_id)
@ -140,7 +143,8 @@ class TestInstanceActionsBase(trove_base.BaseTroveTest):
self.client.create_resource(f"instances/{self.instance_id}/action", self.client.create_resource(f"instances/{self.instance_id}/action",
resize_volume, expected_status_code=202, resize_volume, expected_status_code=202,
need_response=False) need_response=False)
self.wait_for_instance_status(self.instance_id) self.wait_for_instance_status(self.instance_id,
expected_op_status=["HEALTHY"])
# Verify Trove volume # Verify Trove volume
ret = self.client.get_resource('instances', self.instance_id) ret = self.client.get_resource('instances', self.instance_id)

2
trove_tempest_plugin/tests/scenario/base_backup.py
View File

@ -70,6 +70,7 @@ class TestBackupBase(trove_base.BaseTroveTest):
) )
self.wait_for_instance_status( self.wait_for_instance_status(
restore_instance['id'], restore_instance['id'],
expected_op_status=["HEALTHY"],
timeout=CONF.database.database_restore_timeout) timeout=CONF.database.database_restore_timeout)
if self.enable_root: if self.enable_root:
@ -116,6 +117,7 @@ class TestBackupBase(trove_base.BaseTroveTest):
) )
self.wait_for_instance_status( self.wait_for_instance_status(
restore_instance['id'], restore_instance['id'],
expected_op_status=["HEALTHY"],
timeout=CONF.database.database_restore_timeout) timeout=CONF.database.database_restore_timeout)
if self.enable_root: if self.enable_root:

17
trove_tempest_plugin/tests/scenario/base_replication.py
View File

@ -54,6 +54,7 @@ class TestReplicationBase(trove_base.BaseTroveTest):
need_delete=True, expected_status='DELETED') need_delete=True, expected_status='DELETED')
self.wait_for_instance_status( self.wait_for_instance_status(
replica1_id, replica1_id,
expected_op_status=["HEALTHY"],
timeout=CONF.database.database_build_timeout * 2) timeout=CONF.database.database_build_timeout * 2)
replica1 = self.client.get_resource( replica1 = self.client.get_resource(
"instances", replica1_id)['instance'] "instances", replica1_id)['instance']
@ -108,6 +109,7 @@ class TestReplicationBase(trove_base.BaseTroveTest):
need_delete=True, expected_status='DELETED') need_delete=True, expected_status='DELETED')
self.wait_for_instance_status( self.wait_for_instance_status(
replica2_id, replica2_id,
expected_op_status=["HEALTHY"],
timeout=CONF.database.database_build_timeout * 2) timeout=CONF.database.database_build_timeout * 2)
replica2 = self.client.get_resource( replica2 = self.client.get_resource(
"instances", replica2_id)['instance'] "instances", replica2_id)['instance']
@ -144,9 +146,12 @@ class TestReplicationBase(trove_base.BaseTroveTest):
self.client.create_resource(f"instances/{self.instance_id}/action", self.client.create_resource(f"instances/{self.instance_id}/action",
req_body, expected_status_code=202, req_body, expected_status_code=202,
need_response=False) need_response=False)
self.wait_for_instance_status(self.instance_id) self.wait_for_instance_status(self.instance_id,
self.wait_for_instance_status(replica1_id) expected_op_status=["HEALTHY"])
self.wait_for_instance_status(replica2_id) self.wait_for_instance_status(replica1_id,
expected_op_status=["HEALTHY"])
self.wait_for_instance_status(replica2_id,
expected_op_status=["HEALTHY"])
# Verify the volumes of all the replicas are also resized to 2G # Verify the volumes of all the replicas are also resized to 2G
replica1 = self.client.get_resource('instances', replica1_id) replica1 = self.client.get_resource('instances', replica1_id)
@ -162,7 +167,8 @@ class TestReplicationBase(trove_base.BaseTroveTest):
self.client.create_resource(f"instances/{replica1_id}/action", self.client.create_resource(f"instances/{replica1_id}/action",
promote_primary, expected_status_code=202, promote_primary, expected_status_code=202,
need_response=False) need_response=False)
self.wait_for_instance_status(replica1_id) self.wait_for_instance_status(replica1_id,
expected_op_status=["HEALTHY"])
# Make sure to delete replicas first for clean up, in case failure # Make sure to delete replicas first for clean up, in case failure
# happens when replica1 is still the primary. # happens when replica1 is still the primary.
@ -203,7 +209,8 @@ class TestReplicationBase(trove_base.BaseTroveTest):
} }
self.client.put_resource(f'/instances/{self.instance_id}', self.client.put_resource(f'/instances/{self.instance_id}',
detach_replica) detach_replica)
self.wait_for_instance_status(self.instance_id) self.wait_for_instance_status(self.instance_id,
expected_op_status=["HEALTHY"])
# Verify original primary # Verify original primary
ret = self.client.get_resource('instances', self.instance_id) ret = self.client.get_resource('instances', self.instance_id)

8
trove_tempest_plugin/tests/scenario/base_update_access.py
View File

@ -41,7 +41,9 @@ class TestInstanceUpdateAccessBase(trove_base.BaseTroveTest):
} }
} }
self.client.put_resource(f'instances/{self.instance_id}', body) self.client.put_resource(f'instances/{self.instance_id}', body)
self.wait_for_instance_status(self.instance_id, timeout=30) self.wait_for_instance_status(self.instance_id,
expected_op_status=["HEALTHY"],
timeout=30)
instance = self.client.get_resource( instance = self.client.get_resource(
"instances", self.instance_id)['instance'] "instances", self.instance_id)['instance']
@ -59,7 +61,9 @@ class TestInstanceUpdateAccessBase(trove_base.BaseTroveTest):
} }
} }
self.client.put_resource(f'instances/{self.instance_id}', body) self.client.put_resource(f'instances/{self.instance_id}', body)
self.wait_for_instance_status(self.instance_id, timeout=30) self.wait_for_instance_status(self.instance_id,
expected_op_status=["HEALTHY"],
timeout=30)
@decorators.idempotent_id("c907cc80-36b4-11eb-b177-00224d6b7bc1") @decorators.idempotent_id("c907cc80-36b4-11eb-b177-00224d6b7bc1")
def test_instance_update_access(self): def test_instance_update_access(self):