openstack
/
masakari
Code Issues Proposed changes

use looping call with timeout 

When period call, use loopingcall with timeout instead of etimeout.

Change-Id: I5c7d77a0578bbf96ba9d13f6b2784eb912fc548a
This commit is contained in:
suzhengwei 2023-03-20 18:01:56 +08:00
parent 0b32d34379
commit b2f2a58e3c
3 changed files with 32 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
masakari/engine/drivers/taskflow/host_failure.py
View File

@ -15,7 +15,6 @@
import eventlet
from eventlet import greenpool
from eventlet import timeout as etimeout
from oslo_config import cfg
from oslo_log import log as logging
@ -176,23 +175,20 @@ class EvacuateInstancesTask(base.MasakariTask):
if new_vm_state == 'stopped':
raise loopingcall.LoopingCallDone()
periodic_call_stopped = loopingcall.FixedIntervalLoopingCall(
_wait_for_stop_confirmation)
try:
self.novaclient.stop_server(context, instance.id)
# confirm instance is stopped after recovery
periodic_call_stopped.start(interval=CONF.verify_interval)
etimeout.with_timeout(
CONF.wait_period_after_power_off,
periodic_call_stopped.wait)
except etimeout.Timeout:
self.novaclient.stop_server(context, instance.id)
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_for_stop_confirmation)
timer.start(interval=CONF.verify_interval,
timeout=CONF.wait_period_after_power_off).wait()
except loopingcall.LoopingCallTimeOut:
with excutils.save_and_reraise_exception():
msg = ("Instance '%(uuid)s' is successfully evacuated but "
"timeout to stop.") % {'uuid': instance.id}
LOG.warning(msg)
finally:
periodic_call_stopped.stop()
timer.stop()
def _evacuate_and_confirm(self, context, vmove,
reserved_host=None):
@ -242,16 +238,13 @@ class EvacuateInstancesTask(base.MasakariTask):
raise loopingcall.LoopingCallDone()
def _wait_for_evacuation():
periodic_call = loopingcall.FixedIntervalLoopingCall(
_wait_for_evacuation_confirmation)
try:
# add a timeout to the periodic call.
periodic_call.start(interval=CONF.verify_interval)
etimeout.with_timeout(
CONF.wait_period_after_evacuation,
periodic_call.wait)
except etimeout.Timeout:
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_for_evacuation_confirmation)
timer.start(interval=CONF.verify_interval,
timeout=CONF.wait_period_after_evacuation).wait()
except loopingcall.LoopingCallTimeOut:
with excutils.save_and_reraise_exception():
msg = ("Timeout for instance '%(uuid)s' evacuation."
% {'uuid': instance.id})
@ -259,7 +252,7 @@ class EvacuateInstancesTask(base.MasakariTask):
finally:
# stop the periodic call, in case of exceptions or
# Timeout.
periodic_call.stop()
timer.stop()
try:
vm_state = getattr(instance, "OS-EXT-STS:vm_state")
@ -318,7 +311,7 @@ class EvacuateInstancesTask(base.MasakariTask):
vmove,
status=fields.VMoveStatus.SUCCEEDED,
dest_host=dest_host)
except etimeout.Timeout:
except loopingcall.LoopingCallTimeOut:
# Instance is not stop in the expected time_limit.
msg = "Failed reason: timeout."
_update_vmove(

30
masakari/engine/drivers/taskflow/instance_failure.py
View File

@ -13,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from eventlet import timeout as etimeout
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
@ -93,17 +91,15 @@ class StopInstanceTask(base.MasakariTask):
if vm_state == 'stopped':
raise loopingcall.LoopingCallDone()
periodic_call = loopingcall.FixedIntervalLoopingCall(
_wait_for_power_off)
try:
# add a timeout to the periodic call.
periodic_call.start(interval=CONF.verify_interval)
etimeout.with_timeout(CONF.wait_period_after_power_off,
periodic_call.wait)
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_for_power_off)
timer.start(interval=CONF.verify_interval,
timeout=CONF.wait_period_after_power_off).wait()
msg = "Stopped instance: '%s'" % instance_uuid
self.update_details(msg, 1.0)
except etimeout.Timeout:
except loopingcall.LoopingCallTimeOut:
msg = "Failed to stop instance %(instance)s" % {
'instance': instance.id
}
@ -112,7 +108,7 @@ class StopInstanceTask(base.MasakariTask):
message=msg)
finally:
# stop the periodic call, in case of exceptions or Timeout.
periodic_call.stop()
timer.stop()
class StartInstanceTask(base.MasakariTask):
@ -159,20 +155,18 @@ class ConfirmInstanceActiveTask(base.MasakariTask):
if vm_state == 'active':
raise loopingcall.LoopingCallDone()
periodic_call = loopingcall.FixedIntervalLoopingCall(
_wait_for_active)
try:
msg = "Confirming instance '%s' vm_state is ACTIVE" % instance_uuid
self.update_details(msg)
# add a timeout to the periodic call.
periodic_call.start(interval=CONF.verify_interval)
etimeout.with_timeout(CONF.wait_period_after_power_on,
periodic_call.wait)
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_for_active)
timer.start(interval=CONF.verify_interval,
timeout=CONF.wait_period_after_power_on).wait()
msg = "Confirmed instance '%s' vm_state is ACTIVE" % instance_uuid
self.update_details(msg, 1.0)
except etimeout.Timeout:
except loopingcall.LoopingCallTimeOut:
msg = "Failed to start instance %(instance)s" % {
'instance': instance_uuid
}
@ -181,7 +175,7 @@ class ConfirmInstanceActiveTask(base.MasakariTask):
message=msg)
finally:
# stop the periodic call, in case of exceptions or Timeout.
periodic_call.stop()
timer.stop()
def get_instance_recovery_flow(context, novaclient, process_what):

16
masakari/engine/drivers/taskflow/process_failure.py
View File

@ -13,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from eventlet import timeout as etimeout
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
@ -70,23 +68,21 @@ class ConfirmComputeNodeDisabledTask(base.MasakariTask):
if service_disabled:
raise loopingcall.LoopingCallDone()
periodic_call = loopingcall.FixedIntervalLoopingCall(
_wait_for_disable)
try:
msg = "Confirming compute service is disabled on host: '%s'" % (
host_name)
self.update_details(msg)
# add a timeout to the periodic call.
periodic_call.start(interval=CONF.verify_interval)
etimeout.with_timeout(
CONF.wait_period_after_service_update,
periodic_call.wait)
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_for_disable)
timer.start(interval=CONF.verify_interval,
timeout=CONF.wait_period_after_service_update).wait()
msg = "Confirmed compute service is disabled on host: '%s'" % (
host_name)
self.update_details(msg, 1.0)
except etimeout.Timeout:
except loopingcall.LoopingCallTimeOut:
msg = "Failed to disable service %(process_name)s" % {
'process_name': process_name
}
@ -95,7 +91,7 @@ class ConfirmComputeNodeDisabledTask(base.MasakariTask):
message=msg)
finally:
# stop the periodic call, in case of exceptions or Timeout.
periodic_call.stop()
timer.stop()
def get_compute_process_recovery_flow(context, novaclient, process_what):