Fix event listeners
PyMI subscribes to event providers, setting a callback that will be invoked when an event arrives. When called, the event watcher will block until an event is retrieved. A threading.Event is used internally by PyMI for the event waiting workflow. The issue is that the threading module will be monkey patched by eventlet. The event callback is performed from a different thread, so attempting to set the event for which the event watcher waits will some times raise the "error: cannot switch to a different thread" greenlet exception, thus breaking the event listener. This issue can be fixed at the os-win level, without PyMI having to know about eventlet monkey patching. We can make sure that PyMI uses the unpatched threading module, also ensuring that the blocking calls are run in a different thread, so that we don't block the calling thread. eventlet.tpool can be used to easily achieve this. Closes-Bug: #1568824 Change-Id: I9be6c1a1b144bdf565f25517c52439a5fb10cb51
This commit is contained in:
parent
c6cc06ee96
commit
53a8d6d0b3
@ -14,8 +14,16 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
|
||||
from eventlet import patcher
|
||||
import pbr.version
|
||||
|
||||
|
||||
__version__ = pbr.version.VersionInfo(
|
||||
'os_win').version_string()
|
||||
|
||||
if sys.platform == 'win32':
|
||||
import wmi
|
||||
# We need to make sure that WMI uses the unpatched threading module.
|
||||
wmi.threading = patcher.original('threading')
|
||||
|
@ -296,33 +296,37 @@ class ClusterUtilsTestCase(test_base.OsWinBaseTestCase):
|
||||
self._FAKE_HOST,
|
||||
[self._clusterutils._LIVE_MIGRATION_TYPE])
|
||||
|
||||
def test_monitor_vm_failover_no_vm(self):
|
||||
@mock.patch.object(clusterutils, 'tpool')
|
||||
def test_monitor_vm_failover_no_vm(self, mock_tpool):
|
||||
self._clusterutils._watcher = mock.MagicMock()
|
||||
fake_prev = mock.MagicMock(OwnerNode=self._FAKE_PREV_HOST)
|
||||
fake_wmi_object = mock.MagicMock(OwnerNode=self._FAKE_HOST,
|
||||
Name='Virtual Machine',
|
||||
previous=fake_prev)
|
||||
self._clusterutils._watcher.return_value = fake_wmi_object
|
||||
mock_tpool.execute.return_value = fake_wmi_object
|
||||
fake_callback = mock.MagicMock()
|
||||
|
||||
self._clusterutils.monitor_vm_failover(fake_callback)
|
||||
|
||||
self._clusterutils._watcher.assert_called_once_with(
|
||||
mock_tpool.execute.assert_called_once_with(
|
||||
self._clusterutils._watcher,
|
||||
self._clusterutils._WMI_EVENT_TIMEOUT_MS)
|
||||
fake_callback.assert_not_called()
|
||||
|
||||
def test_monitor_vm_failover(self):
|
||||
@mock.patch.object(clusterutils, 'tpool')
|
||||
def test_monitor_vm_failover(self, mock_tpool):
|
||||
self._clusterutils._watcher = mock.MagicMock()
|
||||
fake_prev = mock.MagicMock(OwnerNode=self._FAKE_PREV_HOST)
|
||||
fake_wmi_object = mock.MagicMock(OwnerNode=self._FAKE_HOST,
|
||||
Name=self._FAKE_RESOURCEGROUP_NAME,
|
||||
previous=fake_prev)
|
||||
self._clusterutils._watcher.return_value = fake_wmi_object
|
||||
mock_tpool.execute.return_value = fake_wmi_object
|
||||
fake_callback = mock.MagicMock()
|
||||
|
||||
self._clusterutils.monitor_vm_failover(fake_callback)
|
||||
|
||||
self._clusterutils._watcher.assert_called_once_with(
|
||||
mock_tpool.execute.assert_called_once_with(
|
||||
self._clusterutils._watcher,
|
||||
self._clusterutils._WMI_EVENT_TIMEOUT_MS)
|
||||
fake_callback.assert_called_once_with(self._FAKE_VM_NAME,
|
||||
self._FAKE_PREV_HOST,
|
||||
|
@ -942,7 +942,9 @@ class VMUtilsTestCase(test_base.OsWinBaseTestCase):
|
||||
self.assertEqual(watcher.return_value, listener)
|
||||
|
||||
@mock.patch('time.sleep')
|
||||
def test_vm_power_state_change_event_handler(self, mock_sleep):
|
||||
@mock.patch.object(vmutils, 'tpool')
|
||||
def test_vm_power_state_change_event_handler(self, mock_tpool,
|
||||
mock_sleep):
|
||||
self._mock_wmi.x_wmi_timed_out = exceptions.HyperVException
|
||||
|
||||
enabled_state = constants.HYPERV_VM_STATE_ENABLED
|
||||
@ -953,9 +955,9 @@ class VMUtilsTestCase(test_base.OsWinBaseTestCase):
|
||||
|
||||
fake_listener = (
|
||||
self._vmutils._conn.Msvm_ComputerSystem.watch_for.return_value)
|
||||
fake_listener.side_effect = (self._mock_wmi.x_wmi_timed_out,
|
||||
fake_event, Exception,
|
||||
KeyboardInterrupt)
|
||||
mock_tpool.execute.side_effect = (self._mock_wmi.x_wmi_timed_out,
|
||||
fake_event, Exception,
|
||||
KeyboardInterrupt)
|
||||
|
||||
handler = self._vmutils.get_vm_power_state_change_listener(
|
||||
get_handler=True)
|
||||
@ -965,7 +967,8 @@ class VMUtilsTestCase(test_base.OsWinBaseTestCase):
|
||||
|
||||
fake_callback.assert_called_once_with(mock.sentinel.vm_name,
|
||||
enabled_state)
|
||||
fake_listener.assert_has_calls(
|
||||
mock_tpool.execute.assert_has_calls(
|
||||
fake_listener,
|
||||
[mock.call(self._vmutils._DEFAULT_EVENT_TIMEOUT_MS)] * 4)
|
||||
mock_sleep.assert_called_once_with(
|
||||
self._vmutils._DEFAULT_EVENT_TIMEOUT_MS / 1000)
|
||||
|
@ -143,14 +143,16 @@ class NetworkUtilsTestCase(test_base.OsWinBaseTestCase):
|
||||
mock.sentinel.switch_port_name)
|
||||
self.assertEqual(mock.sentinel.mac_address, actual_mac_address)
|
||||
|
||||
@mock.patch.object(networkutils.tpool, 'execute')
|
||||
@mock.patch.object(networkutils, 'wmi', create=True)
|
||||
@mock.patch.object(networkutils.NetworkUtils, '_get_event_wql_query')
|
||||
def test_get_vnic_event_listener(self, mock_get_event_query, mock_wmi):
|
||||
def test_get_vnic_event_listener(self, mock_get_event_query, mock_wmi,
|
||||
mock_execute):
|
||||
mock_wmi.x_wmi_timed_out = ValueError
|
||||
event = mock.MagicMock()
|
||||
port_class = self.netutils._conn.Msvm_SyntheticEthernetPortSettingData
|
||||
wmi_event_listener = port_class.watch_for.return_value
|
||||
wmi_event_listener.side_effect = [mock_wmi.x_wmi_timed_out, event]
|
||||
mock_execute.side_effect = [mock_wmi.x_wmi_timed_out, event]
|
||||
|
||||
# callback will raise an exception in order to stop iteration in the
|
||||
# listener.
|
||||
@ -166,8 +168,9 @@ class NetworkUtilsTestCase(test_base.OsWinBaseTestCase):
|
||||
timeframe=2)
|
||||
port_class.watch_for.assert_called_once_with(
|
||||
mock_get_event_query.return_value)
|
||||
wmi_event_listener.assert_has_calls(
|
||||
[mock.call(self.netutils._VNIC_LISTENER_TIMEOUT_MS)] * 2)
|
||||
mock_execute.assert_has_calls(
|
||||
[mock.call(wmi_event_listener,
|
||||
self.netutils._VNIC_LISTENER_TIMEOUT_MS)] * 2)
|
||||
callback.assert_called_once_with(event.ElementName)
|
||||
|
||||
def test_get_event_wql_query(self):
|
||||
|
@ -23,7 +23,9 @@ import sys
|
||||
if sys.platform == 'win32':
|
||||
import wmi
|
||||
|
||||
from eventlet import tpool
|
||||
from oslo_log import log as logging
|
||||
|
||||
from os_win._i18n import _, _LE
|
||||
from os_win import exceptions
|
||||
from os_win.utils import baseutils
|
||||
@ -209,7 +211,8 @@ class ClusterUtils(baseutils.BaseUtils):
|
||||
new_host = None
|
||||
try:
|
||||
# wait for new event for _WMI_EVENT_TIMEOUT_MS miliseconds.
|
||||
wmi_object = self._watcher(self._WMI_EVENT_TIMEOUT_MS)
|
||||
wmi_object = tpool.execute(self._watcher,
|
||||
self._WMI_EVENT_TIMEOUT_MS)
|
||||
old_host = wmi_object.previous.OwnerNode
|
||||
new_host = wmi_object.OwnerNode
|
||||
# wmi_object.Name field is of the form:
|
||||
|
@ -27,6 +27,7 @@ import uuid
|
||||
if sys.platform == 'win32':
|
||||
import wmi
|
||||
|
||||
from eventlet import tpool
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
from oslo_utils import uuidutils
|
||||
@ -819,7 +820,12 @@ class VMUtils(baseutils.BaseUtilsVirt):
|
||||
try:
|
||||
# Retrieve one by one all the events that occurred in
|
||||
# the checked interval.
|
||||
event = listener(event_timeout)
|
||||
#
|
||||
# We use eventlet.tpool for retrieving the events in
|
||||
# order to avoid issues caused by greenthread/thread
|
||||
# communication. Note that PyMI must use the unpatched
|
||||
# threading module.
|
||||
event = tpool.execute(listener, event_timeout)
|
||||
|
||||
vm_name = event.ElementName
|
||||
vm_state = event.EnabledState
|
||||
|
@ -21,6 +21,7 @@ Hyper-V Server / Windows Server 2012.
|
||||
import re
|
||||
|
||||
from eventlet import greenthread
|
||||
from eventlet import tpool
|
||||
import sys
|
||||
|
||||
if sys.platform == 'win32':
|
||||
@ -219,7 +220,8 @@ class NetworkUtils(baseutils.BaseUtilsVirt):
|
||||
# Retrieve one by one all the events that occurred in
|
||||
# the checked interval.
|
||||
try:
|
||||
event = listener(self._VNIC_LISTENER_TIMEOUT_MS)
|
||||
event = tpool.execute(listener,
|
||||
self._VNIC_LISTENER_TIMEOUT_MS)
|
||||
callback(event.ElementName)
|
||||
except wmi.x_wmi_timed_out:
|
||||
# no new event published.
|
||||
|
Loading…
Reference in New Issue
Block a user