UnhookWindowsHookEx produces ERROR_INVALID_HOOK_HANDLE? - wpf

Question

I'm trying to implement a background application in WPF that does some things only when right-clicking the top of the screen, so I tried hooking WH_MOUSE_LL, which works great. The problem is unhooking the callback when exiting the application, which works perfectly fine in C, but gives me ERROR_INVALID_HOOK_HANDLE in C# even though the hook handle passed to UnhookWindowsHookEx is exactly the same handle I got from SetWindowsHookEx (examples below).
EDIT: I just created a bare bones C# console application registering the same hook and instantly unregistering it and it works fine too, so I think WPF might cause the problems.
Here is the C# code that does not work, problem is in the destructor:
using System;
using System.Runtime.InteropServices;
public class LowLevelMouseHook {
#region Win32 API
private const int WH_MOUSE_LL = 14;
private const ulong WM_RBUTTONDOWN = 0x0204;
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private delegate long LowLevelMouseProc (int nCode, UIntPtr wParam, IntPtr lParam);
private struct POINT {
long x;
long y;
}
private struct MSLLHOOKSTRUCT {
POINT pt;
int mouseData;
int flags;
int time;
UIntPtr dwExtraInfo;
}
[DllImport("User32.dll", EntryPoint = "CallNextHookEx", SetLastError = true)]
private static extern long CallNextHookEx (IntPtr hHook, int nCode, UIntPtr wParam, IntPtr lParam);
[DllImport("Kernel32.dll", EntryPoint = "GetLastError", SetLastError = true)]
private static extern uint GetLastError ();
[DllImport("Kernel32.dll", EntryPoint = "LoadLibraryW", CharSet = CharSet.Unicode, SetLastError = true)]
private static extern IntPtr LoadLibrary (string lpFileName);
[DllImport("User32.dll", EntryPoint = "SetWindowsHookExW", SetLastError = true)]
private static extern IntPtr SetWindowsHookEx (int idHook, Delegate lpfn, IntPtr hMod, uint dwThreadId);
[DllImport("User32.dll", EntryPoint = "UnhookWindowsHookEx", SetLastError = true)]
private static extern byte UnhookWindowsHookEx (IntPtr hHook);
#endregion
// There shall only be one instance of this class
private static LowLevelMouseHook instance = null;
private IntPtr hHook;
private LowLevelMouseProc hProc; // If I don't store the delegate, I get some CallbackOnCollectedDelegate exception
private LowLevelMouseHook () {
this.hProc = new LowLevelMouseProc(MouseProc);
this.hHook = SetWindowsHookEx(WH_MOUSE_LL, this.hProc, LoadLibrary("User32.dll"), 0);
}
~LowLevelMouseHook () {
//
// PROBLEM:
// UnhookWindowsHookEx always returns 0 and GetLastError returns ERROR_INVALID_HOOK_HANDLE (1404)
// even though this.hHook is still the same value SetWindowsHookEx returned.
//
if (UnhookWindowsHookEx(this.hHook) == 0)
System.Windows.Forms.MessageBox.Show($"Error {GetLastError()} unhooking WH_MOUSE_LL", "UnhookWindowsHookEx Error");
LowLevelMouseHook.instance = null;
}
// Create new LowLevelMouseHook instance.
// This is called during the Startup-Event of Application.
public static void Init () {
LowLevelMouseHook.instance = new LowLevelMouseHook();
}
private static long MouseProc (int nCode, UIntPtr wParam, IntPtr lParam) {
if ((ulong)wParam == WM_RBUTTONDOWN) {
; // Things to do; that works fine
}
return CallNextHookEx(LowLevelMouseHook.instance.hHook, nCode, wParam, lParam);
}
}
Here is the C code that works perfectly fine:
#include <stdio.h>
#include <Windows.h>
FILE *file;
HHOOK hHook;
LRESULT CALLBACK LowLevelMouseProc_Test (int nCode, WPARAM wParam, LPARAM lParam) {
PMSLLHOOKSTRUCT pHookStruct = (PMSLLHOOKSTRUCT)lParam;
fprintf(file, "MouseProc_Test(nCode = %i, wParam = %lu, lParam->pt = (%i, %i))\n", nCode, wParam, pHookStruct->pt.x, pHookStruct->pt.y);
return CallNextHookEx(hHook, nCode, wParam, lParam);
}
VOID CALLBACK TimerProc_Test (HWND hWnd, UINT uMsg, UINT_PTR idEvent, DWORD dwTime) {
PostQuitMessage(0);
}
int WINAPI WinMain (HINSTANCE hInstance, HINSTANCE hPrevInstance, PSTR szCmdLine, int nShow) {
fopen_s(&file, "wm_mouse_ll.txt", "w");
hHook = SetWindowsHookExW(WH_MOUSE_LL, LowLevelMouseProc_Test, LoadLibrary("User32.dll"), 0);
if (hHook != 0) {
fprintf(file, "SetWindowsHookExW Success (hHook = 0x%p)\n", hHook);
SetTimer(NULL, 0, 5000, TimerProc_Test);
MSG msg = {0};
while (GetMessageW(&msg, NULL, 0, 0) != 0) {
TranslateMessage(&msg);
DispatchMessageW(&msg);
}
if (UnhookWindowsHookEx(hHook) != 0)
fputs("UnhookWindowsHookEx Success\n", file);
else
fprintf(file, "UnhookWindowsHookEx Error %u\n", GetLastError());
} else {
fprintf(file, "SetWindowsHookExW Error %u\n", GetLastError());
}
fclose(file);
return 0;
}

Answer 1

Ok, I simply restructured LowLevelMouseHook a little bit and added a custom Main() method that looks like the one below. UnhookWindowsHookEx now succeeds.
public static class EntryPoint {
public static void Main () {
LowLevelMouseHook.Begin() // Here SetWindowsHookEx is called
App.Main() // Original entrypoint
LowLevelMouseHook.End() // Here UnhookWindowsHookEx is called
}
}