Now QuickLook handles preview invoked from Commom Dialogs (yet only matched bitness)

QuickLook.Native.Shell32/QuickLook.Native.Shell32.cpp

@@ -1,4 +1,3 @@
-// dllmain.cpp : Defines the entry point for the DLL application.
 #include "stdafx.h"
 
 #include "Shell32.h"

QuickLook.Native.Shell32/Shell32.cpp

@@ -1,7 +1,6 @@
 #include "stdafx.h"
 
 #include "Shell32.h"
-#include <atlcomcli.h>
 
 using namespace std;
 
@@ -28,6 +27,13 @@ Shell32::FocusedWindowType Shell32::GetFocusedWindowType()
 		{
 			type = EXPLORER;
 		}
+		else if (wcscmp(classBuffer, L"#32770") == 0)
+		{
+			if (FindWindowEx(hwndfg, nullptr, L"DUIViewWndClassName", nullptr) != nullptr)
+			{
+				type = DIALOG;
+			}
+		}
 	}
 	delete[] classBuffer;
 
@@ -38,11 +44,14 @@ void Shell32::GetCurrentSelection(PWCHAR buffer)
 {
 	switch (GetFocusedWindowType())
 	{
+	case DESKTOP:
+		getSelectedFromDesktop(buffer);
+		break;
 	case EXPLORER:
 		getSelectedFromExplorer(buffer);
 		break;
-	case DESKTOP:
-		getSelectedFromDesktop(buffer);
+	case DIALOG:
+		getSelectedFromCommonDialog(buffer);
 		break;
 	default:
 		break;
@@ -57,7 +66,7 @@ void Shell32::getSelectedFromExplorer(PWCHAR buffer)
 	if (FAILED(psw.CoCreateInstance(CLSID_ShellWindows)))
 		return;
 
-	auto hwndFGW = GetForegroundWindow();
+	auto hwndfg = GetForegroundWindow();
 
 	auto count = 0L;
 	psw->get_Count(&count);
@@ -77,11 +86,11 @@ void Shell32::getSelectedFromExplorer(PWCHAR buffer)
 		if (FAILED(pdisp->QueryInterface(IID_IWebBrowserApp, reinterpret_cast<void**>(&pwba))))
 			continue;
 
-		HWND hwndWBA;
-		if (FAILED(pwba->get_HWND(reinterpret_cast<LONG_PTR*>(&hwndWBA))))
+		HWND hwndwba;
+		if (FAILED(pwba->get_HWND(reinterpret_cast<LONG_PTR*>(&hwndwba))))
 			continue;
 
-		if (hwndWBA != hwndFGW || isCursorActivated(hwndWBA))
+		if (hwndwba != hwndfg || isCursorActivated(hwndwba))
 			continue;
 
 		getSelectedInternal(pwba, buffer);
@@ -162,3 +171,79 @@ bool Shell32::isCursorActivated(HWND hwnd)
 	GetGUIThreadInfo(tId, &gui);
 	return gui.flags || gui.hwndCaret;
 }
+
+#pragma region DialogHook
+
+#pragma comment(linker, "/SECTION:.shared,RWS")
+#pragma data_seg(".shared")
+HWND ghMsgWindow = nullptr; // Window handle
+HHOOK ghHook = nullptr; // Hook handle
+UINT WM_HOOK_NOTIFY = 0;
+WCHAR filePathBuffer[MAX_PATH] = {'\0'};
+#pragma data_seg()
+
+void Shell32::getSelectedFromCommonDialog(PWCHAR buffer)
+{
+	auto hwndfg = GetForegroundWindow();
+	auto tid = GetWindowThreadProcessId(hwndfg, nullptr);
+
+	if (WM_HOOK_NOTIFY == 0)
+		WM_HOOK_NOTIFY = RegisterWindowMessage(L"WM_QUICKLOOK_HOOK_NOTIFY_MSG");
+
+	if (ghHook != nullptr)
+		UnhookWindowsHookEx(ghHook);
+	ghHook = SetWindowsHookEx(WH_CALLWNDPROC, static_cast<HOOKPROC>(MsgHookProc), ModuleFromAddress(MsgHookProc), tid);
+	if (ghHook == nullptr)
+		return;
+
+	SendMessage(hwndfg, WM_HOOK_NOTIFY, 0, 0);
+	wcscpy_s(buffer, wcslen(buffer) - 1, filePathBuffer);
+}
+
+HMODULE Shell32::ModuleFromAddress(PVOID pv)
+{
+	MEMORY_BASIC_INFORMATION mbi;
+	return VirtualQuery(pv, &mbi, sizeof mbi) != 0 ? static_cast<HMODULE>(mbi.AllocationBase) : nullptr;
+}
+
+LRESULT Shell32::MsgHookProc(int nCode, WPARAM wParam, LPARAM lParam)
+{
+	if (nCode < 0)
+		goto CONTINUE;
+
+	auto pMSG = reinterpret_cast<CWPSTRUCT*>(lParam);
+
+	// only response to WM_HOOK_NOTIFY
+	if (pMSG->message != WM_HOOK_NOTIFY)
+		goto CONTINUE;
+
+	UnhookWindowsHookEx(ghHook);
+	ghHook = nullptr;
+
+	// get current selected item
+	auto psb = reinterpret_cast<IShellBrowser*>(SendMessage(GetForegroundWindow(), WM_USER + 7, 0, 0));
+	if (psb == nullptr)
+		goto CONTINUE;
+
+	getSelectedInternal2(psb, filePathBuffer);
+
+	return 0;
+
+CONTINUE:
+	return CallNextHookEx(ghHook, nCode, wParam, lParam);
+}
+
+void Shell32::getSelectedInternal2(CComPtr<IShellBrowser> psb, PWCHAR buffer)
+{
+	CComPtr<IShellView> psv;
+	if (FAILED(psb->QueryActiveShellView(&psv)))
+		return;
+
+	CComPtr<IDataObject> dao;
+	if (FAILED(psv->GetItemObject(SVGIO_SELECTION, IID_IDataObject, reinterpret_cast<void**>(&dao))))
+		return;
+
+	return obtainFirstItem(dao, buffer);
+}
+
+#pragma endregion DialogHook

QuickLook.Native.Shell32/Shell32.h

@@ -10,10 +10,12 @@ public:
 		INVALID = 0,
 		DESKTOP = 1,
 		EXPLORER = 2,
+		DIALOG = 3,
 	};
 
 	static FocusedWindowType GetFocusedWindowType();
 	static void GetCurrentSelection(PWCHAR buffer);
+	static LRESULT __stdcall WindowProcedure(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam);
 
 private:
 	static void getSelectedFromDesktop(PWCHAR buffer);
@@ -22,4 +24,11 @@ private:
 	static void getSelectedInternal(CComQIPtr<IWebBrowserApp> pWebBrowserApp, PWCHAR buffer);
 	static void obtainFirstItem(CComPtr<IDataObject> dao, PWCHAR buffer);
 	static bool isCursorActivated(HWND hwndfg);
+
+#pragma region DialogHook
+	static void getSelectedFromCommonDialog(PWCHAR buffer);
+	static HMODULE ModuleFromAddress(PVOID pv);
+	static LRESULT CALLBACK MsgHookProc(int nCode, WPARAM wParam, LPARAM lParam);
+	static void getSelectedInternal2(CComPtr<IShellBrowser> psb, PWCHAR buffer);
+#pragma endregion DialogHook
 };

QuickLook.Native.Shell32/stdafx.h

@@ -19,4 +19,3 @@
 #include<shlguid.h>
 #include<Shlobj.h>
 #include<Shellapi.h>
-#include<vector>

QuickLook.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
-VisualStudioVersion = 15.0.26430.12
+VisualStudioVersion = 15.0.26430.13
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuickLook", "QuickLook\QuickLook.csproj", "{8B4A9CE5-67B5-4A94-81CB-3771F688FDEB}"
 	ProjectSection(ProjectDependencies) = postProject
@@ -133,7 +133,6 @@ Global
 		{AB1270AF-7EB4-4B4F-9E09-6404F1A28EA0}.Release|x86.Build.0 = Release|x86
 		{F0214FC2-EFBE-426C-842D-B42BC37D9525}.Debug|Any CPU.ActiveCfg = Debug|x86
 		{F0214FC2-EFBE-426C-842D-B42BC37D9525}.Debug|x86.ActiveCfg = Debug|x86
-		{F0214FC2-EFBE-426C-842D-B42BC37D9525}.Debug|x86.Build.0 = Debug|x86
 		{F0214FC2-EFBE-426C-842D-B42BC37D9525}.Release|Any CPU.ActiveCfg = Release|x86
 		{F0214FC2-EFBE-426C-842D-B42BC37D9525}.Release|Any CPU.Build.0 = Release|x86
 		{F0214FC2-EFBE-426C-842D-B42BC37D9525}.Release|x86.ActiveCfg = Release|x86

QuickLook/NativeMethods/QuickLook.cs

@@ -6,6 +6,8 @@ namespace QuickLook.NativeMethods
 {
     internal static class QuickLook
     {
+        private const int MaxPath = 260;
+
         [DllImport("QuickLook.Native.Shell32.dll", EntryPoint = "GetFocusedWindowType",
             CallingConvention = CallingConvention.Cdecl)]
         private static extern FocusedWindowType GetFocusedWindowTypeNative_32();
@@ -33,7 +35,7 @@ namespace QuickLook.NativeMethods
             // communicate with COM in a separate thread
             Task.Run(() =>
             {
-                sb = new StringBuilder(255 + 1);
+                sb = new StringBuilder(MaxPath);
                 if (App.Is64Bit)
                     GetCurrentSelectionNative_64(sb);
                 else
@@ -47,7 +49,8 @@ namespace QuickLook.NativeMethods
         {
             Invalid,
             Desktop,
-            Explorer
+            Explorer,
+            Dialog
         }
     }
 }

QuickLook/ViewWindowManager.cs

@@ -70,14 +70,12 @@ namespace QuickLook
 
             // if the current focus is in Desktop or explorer windows, just close the preview window and leave the task to System.
             var focus = NativeMethods.QuickLook.GetFocusedWindowType();
-            if (focus == NativeMethods.QuickLook.FocusedWindowType.Desktop ||
-                focus == NativeMethods.QuickLook.FocusedWindowType.Explorer)
-                if (_path == NativeMethods.QuickLook.GetCurrentSelection())
-                {
-                    StopFocusMonitor();
-                    _currentMainWindow.BeginHide();
-                    return;
-                }
+            if (focus != NativeMethods.QuickLook.FocusedWindowType.Invalid)
+            {
+                StopFocusMonitor();
+                _currentMainWindow.BeginHide();
+                return;
+            }
 
             // if the focus is in the preview window, run it
             if (!WindowHelper.IsForegroundWindowBelongToSelf())