watchdog

1 #watchdog.py 2 #A part of NonVisual Desktop Access (NVDA) 3 #Copyright (C) 2008-2011 NV Access Inc 4 #This file is covered by the GNU General Public License. 5 #See the file COPYING for more details. 6 7 import sys 8 import traceback 9 import time 10 import threading 11 import inspect 12 from ctypes import windll, oledll 13 import ctypes.wintypes 14 import comtypes 15 import winUser 16 import winKernel 17 from logHandler import log 18 19 #settings 20 #: How often to check whether the core is alive 21 CHECK_INTERVAL=0.1 22 #: How long to wait for the core to be alive under normal circumstances 23 NORMAL_CORE_ALIVE_TIMEOUT=10 24 #: The minimum time to wait for the core to be alive 25 MIN_CORE_ALIVE_TIMEOUT=0.5 26 #: How long to wait between recovery attempts 27 RECOVER_ATTEMPT_INTERVAL = 0.05 28 #: The amount of time before the core should be considered severely frozen and a warning logged. 29 FROZEN_WARNING_TIMEOUT = 15 30 31 safeWindowClassSet=set([ 32 'Internet Explorer_Server', 33 '_WwG', 34 'EXCEL7', 35 ]) 36 37 isRunning=False 38 isAttemptingRecovery = False 39 40 _coreAliveEvent = threading.Event() 41 _resumeEvent = threading.Event() 42 _coreThreadID=windll.kernel32.GetCurrentThreadId() 43 _watcherThread=None

44 45 -class CallCancelled(Exception):

46 """Raised when a call is cancelled. 47 """

48

49 -def alive():

50 """Inform the watchdog that the core is alive. 51 """ 52 global _coreAliveEvent 53 _coreAliveEvent.set()

54

55 -def _watcher():

56 global isAttemptingRecovery 57 while isRunning: 58 # If the watchdog is suspended, wait until it is resumed. 59 _resumeEvent.wait() 60 61 # Wait for the core to be alive. 62 # Wait a maximum of NORMAL_CORE_ALIVE_TIMEOUT, but shorten this to a minimum of MIN_CORE_ALIVE_TIMEOUT under special circumstances. 63 waited = 0 64 while True: 65 # Wait MIN_CORE_ALIVE_TIMEOUT, unless there is less than that time remaining for NORMAL_CORE_ALIVE_TIMEOUT. 66 timeout = min(MIN_CORE_ALIVE_TIMEOUT, NORMAL_CORE_ALIVE_TIMEOUT - waited) 67 if timeout <= 0: 68 # Timeout elapsed. 69 break 70 _coreAliveEvent.wait(timeout) 71 waited += timeout 72 if _coreAliveEvent.isSet() or _shouldRecoverAfterMinTimeout(): 73 break 74 if log.isEnabledFor(log.DEBUGWARNING) and not _coreAliveEvent.isSet(): 75 log.debugWarning("Trying to recover from freeze, core stack:\n%s"% 76 "".join(traceback.format_stack(sys._current_frames()[_coreThreadID]))) 77 lastTime=time.time() 78 while not _coreAliveEvent.isSet(): 79 curTime=time.time() 80 if curTime-lastTime>FROZEN_WARNING_TIMEOUT: 81 lastTime=curTime 82 log.warning("Core frozen in stack:\n%s"% 83 "".join(traceback.format_stack(sys._current_frames()[_coreThreadID]))) 84 # The core is dead, so attempt recovery. 85 isAttemptingRecovery = True 86 _recoverAttempt() 87 _coreAliveEvent.wait(RECOVER_ATTEMPT_INTERVAL) 88 isAttemptingRecovery = False 89 90 # At this point, the core is alive. 91 _coreAliveEvent.clear() 92 # Wait a bit to avoid excessive resource consumption. 93 time.sleep(CHECK_INTERVAL)

94

95 -def _shouldRecoverAfterMinTimeout():

96 info=winUser.getGUIThreadInfo(0) 97 if not info.hwndFocus: 98 # The foreground thread is frozen or there is no foreground thread (probably due to a freeze elsewhere). 99 return True 100 # Import late to avoid circular import. 101 import api 102 #If a system menu has been activated but NVDA's focus is not yet in the menu then use min timeout 103 if info.flags&winUser.GUI_SYSTEMMENUMODE and info.hwndMenuOwner and api.getFocusObject().windowClassName!='#32768': 104 return True 105 if winUser.getClassName(info.hwndFocus) in safeWindowClassSet: 106 return False 107 if not winUser.isDescendantWindow(info.hwndActive, api.getFocusObject().windowHandle): 108 # The foreground window has changed. 109 return True 110 newHwnd=info.hwndFocus 111 newThreadID=winUser.getWindowThreadProcessID(newHwnd)[1] 112 return newThreadID!=api.getFocusObject().windowThreadID

113

114 -def _recoverAttempt():

115 try: 116 oledll.ole32.CoCancelCall(_coreThreadID,0) 117 except: 118 pass 119 import NVDAHelper 120 NVDAHelper.localLib.cancelSendMessage()

121

122 @ctypes.WINFUNCTYPE(ctypes.wintypes.LONG, ctypes.c_void_p) 123 -def _crashHandler(exceptionInfo):

124 # An exception might have been set for this thread. 125 # Clear it so that it doesn't get raised in this function. 126 ctypes.pythonapi.PyThreadState_SetAsyncExc(threading.currentThread().ident, None) 127 import core 128 core.restart() 129 return 1 # EXCEPTION_EXECUTE_HANDLER

130

131 @ctypes.WINFUNCTYPE(None) 132 -def _notifySendMessageCancelled():

133 caller = inspect.currentframe().f_back 134 if not caller: 135 return 136 # Set a profile function which will raise an exception when returning from the calling frame. 137 def sendMessageCallCanceller(frame, event, arg): 138 if frame == caller: 139 # Raising an exception will also cause the profile function to be deactivated. 140 raise CallCancelled

141 sys.setprofile(sendMessageCallCanceller) 142 143 RPC_E_CALL_CANCELED = -2147418110 144 _orig_COMError_init = comtypes.COMError.__init__

145 -def _COMError_init(self, hresult, text, details):

146 if hresult == RPC_E_CALL_CANCELED: 147 raise CallCancelled 148 _orig_COMError_init(self, hresult, text, details)

149

150 -def initialize():

151 """Initialize the watchdog. 152 """ 153 global _watcherThread, isRunning 154 if isRunning: 155 raise RuntimeError("already running") 156 isRunning=True 157 # Catch application crashes. 158 windll.kernel32.SetUnhandledExceptionFilter(_crashHandler) 159 oledll.ole32.CoEnableCallCancellation(None) 160 # Handle cancelled SendMessage calls. 161 import NVDAHelper 162 NVDAHelper._setDllFuncPointer(NVDAHelper.localLib, "_notifySendMessageCancelled", _notifySendMessageCancelled) 163 # Monkey patch comtypes to specially handle cancelled COM calls. 164 comtypes.COMError.__init__ = _COMError_init 165 _coreAliveEvent.set() 166 _resumeEvent.set() 167 _watcherThread=threading.Thread(target=_watcher) 168 _watcherThread.start()

169

170 -def terminate():

171 """Terminate the watchdog. 172 """ 173 global isRunning 174 if not isRunning: 175 return 176 isRunning=False 177 oledll.ole32.CoDisableCallCancellation(None) 178 comtypes.COMError.__init__ = _orig_COMError_init 179 _resumeEvent.set() 180 _coreAliveEvent.set() 181 _watcherThread.join()

182

183 -class Suspender(object):

184 """A context manager to temporarily suspend the watchdog for a block of code. 185 """ 186

187 - def __enter__(self):

188 _resumeEvent.clear()

189

190 - def __exit__(self,*args):

191 _resumeEvent.set()

192

193 -class CancellableCallThread(threading.Thread):

194 """A worker thread used to execute a call which must be made cancellable. 195 If the call is cancelled, this thread must be abandoned. 196 """ 197

198 - def __init__(self):

199 super(CancellableCallThread, self).__init__() 200 self.daemon = True 201 self._executeEvent = threading.Event() 202 self._executionDoneEvent = ctypes.windll.kernel32.CreateEventW(None, False, False, None) 203 self.isUsable = True

204

205 - def execute(self, func, args, kwargs, pumpMessages=True):

206 # Don't even bother making the call if the core is already dead. 207 if isAttemptingRecovery: 208 raise CallCancelled 209 210 self._func = func 211 self._args = args 212 self._kwargs = kwargs 213 self._result = None 214 self._exc_info = None 215 self._executeEvent.set() 216 217 if pumpMessages: 218 waitHandles = (ctypes.wintypes.HANDLE * 1)(self._executionDoneEvent) 219 waitIndex = ctypes.wintypes.DWORD() 220 timeout = int(1000 * CHECK_INTERVAL) 221 while True: 222 if pumpMessages: 223 try: 224 oledll.ole32.CoWaitForMultipleHandles(0, timeout, 1, waitHandles, ctypes.byref(waitIndex)) 225 break 226 except WindowsError: 227 pass 228 else: 229 if windll.kernel32.WaitForSingleObject(self._executionDoneEvent, timeout) != winKernel.WAIT_TIMEOUT: 230 break 231 if isAttemptingRecovery: 232 self.isUsable = False 233 raise CallCancelled 234 235 exc = self._exc_info 236 if exc: 237 raise exc[0], exc[1], exc[2] 238 return self._result

239

240 - def run(self):

241 comtypes.CoInitializeEx(comtypes.COINIT_MULTITHREADED) 242 while self.isUsable: 243 self._executeEvent.wait() 244 self._executeEvent.clear() 245 try: 246 self._result = self._func(*self._args, **self._kwargs) 247 except: 248 self._exc_info = sys.exc_info() 249 ctypes.windll.kernel32.SetEvent(self._executionDoneEvent) 250 ctypes.windll.kernel32.CloseHandle(self._executionDoneEvent)

251 252 cancellableCallThread = None

253 -def cancellableExecute(func, *args, **kwargs):

254 """Execute a function in the main thread, making it cancellable. 255 @param func: The function to execute. 256 @type func: callable 257 @param ccPumpMessages: Whether to pump messages while waiting. 258 @type ccPumpMessages: bool 259 @param args: Positional arguments for the function. 260 @param kwargs: Keyword arguments for the function. 261 @raise CallCancelled: If the call was cancelled. 262 """ 263 global cancellableCallThread 264 pumpMessages = kwargs.pop("ccPumpMessages", True) 265 if not isRunning or not _resumeEvent.isSet() or not isinstance(threading.currentThread(), threading._MainThread): 266 # Watchdog is not running or this is a background thread, 267 # so just execute the call. 268 return func(*args, **kwargs) 269 if not cancellableCallThread or not cancellableCallThread.isUsable: 270 # The thread hasn't yet been created or is not usable. 271 # Create a new one. 272 cancellableCallThread = CancellableCallThread() 273 cancellableCallThread.start() 274 return cancellableCallThread.execute(func, args, kwargs, pumpMessages=pumpMessages)

275

276 -def cancellableSendMessage(hwnd, msg, wParam, lParam, flags=0, timeout=60000):

277 """Send a window message, making the call cancellable. 278 The C{timeout} and C{flags} arguments should usually be left at their default values. 279 The call will still be cancelled if appropriate even if the specified timeout has not yet been reached. 280 @raise CallCancelled: If the call was cancelled. 281 """ 282 import NVDAHelper 283 result = ctypes.wintypes.DWORD() 284 NVDAHelper.localLib.cancellableSendMessageTimeout(hwnd, msg, wParam, lParam, flags, timeout, ctypes.byref(result)) 285 return result.value

286

Source Code for Module watchdog