Warn when TTD is used without the WinDbg/TTD package (Fixes #1103)

core/adapters/dbgeng/README.md

@@ -40,7 +40,7 @@ The installer performs these steps:
 
 ## UI Integration
 
-The installer is called from `ui/ui.cpp` in the `GlobalDebuggerUI::installTTD()` function, which:
+The installer is called from `ui/ttdinstall.cpp` in the `TTDInstall::RunInstaller()` function, which:
 - Shows a progress dialog
 - Runs the installer asynchronously using QTimer
 - Displays success/failure messages

core/adapters/dbgengadapter.cpp

@@ -103,6 +103,19 @@ std::string DbgEngAdapter::GetDbgEngPath(const std::string& arch)
 }
 
 
+std::string DbgEngAdapter::GetDbgEngInstallHint()
+{
+	if (!Settings::Instance()->Get<string>("debugger.x64dbgEngPath").empty())
+		return "debugger.x64dbgEngPath is set, but it does not point at a folder that holds the DbgEng DLLs. It "
+			   "must be the amd64 folder of a WinDbg installation.";
+
+	return "The WinDbg/TTD package has not been downloaded. Use \"Debugger\" -> \"Install WinDbg/TTD\" to let "
+		   "Binary Ninja download it, then restart Binary Ninja. A WinDbg installed from the Microsoft Store or "
+		   "through the standalone installer cannot be used, since it is packaged in a form the debugger cannot "
+		   "load from.";
+}
+
+
 static bool LoadOneDLL(const string& path, const string& name, bool strictCheckPath = true, bool forceUnload = true)
 {
 	auto handle = GetModuleHandleA(name.c_str());
@@ -171,8 +184,7 @@ bool DbgEngAdapter::LoadDngEngLibraries()
 	auto enginePath = GetDbgEngPath("amd64");
 	if (enginePath.empty())
 	{
-		LogWarn("The debugger cannot find the path for the DbgEng DLLs. "
-			"If you have set debugger.x64dbgEngPath, check if it valid");
+		LogWarn("The debugger cannot find the path for the DbgEng DLLs. %s", GetDbgEngInstallHint().c_str());
 		return false;
 	}
 	LogDebug("DbgEng libraries in path %s", enginePath.c_str());

core/adapters/dbgengadapter.h

@@ -255,6 +255,10 @@ namespace BinaryNinjaDebugger {
 
 		static std::string GetDbgEngPath(const std::string& arch);
 
+		// Explains, for the user, where the DbgEng DLLs are supposed to come from. Meant to be appended to an
+		// error reported when they could not be found or loaded.
+		static std::string GetDbgEngInstallHint();
+
 		static bool LoadDngEngLibraries();
 
 		std::string GenerateRandomPipeName();

core/adapters/dbgengttdadapter.cpp

@@ -33,10 +33,16 @@ bool DbgEngTTDAdapter::ExecuteWithArgsInternal(const std::string& path, const st
 
     if (!Start()) {
         this->Reset();
+        std::string error = "Failed to initialize DbgEng";
+        // By far the most common reason for this is that the WinDbg/TTD package was never downloaded, so say
+        // what to do about it rather than leaving the user with an opaque failure
+        if (GetModuleHandleA("dbgeng.dll") == nullptr)
+            error += ": the DbgEng DLLs are not loaded. " + DbgEngAdapter::GetDbgEngInstallHint();
+
         DebuggerEvent event;
         event.type = LaunchFailureEventType;
-        event.data.errorData.error = fmt::format("Failed to initialize DbgEng");
-        event.data.errorData.shortError = fmt::format("Failed to initialize DbgEng");
+        event.data.errorData.error = error;
+        event.data.errorData.shortError = "Failed to initialize DbgEng";
         PostDebuggerEvent(event);
         return false;
     }

docs/guide/dbgeng-ttd.md

@@ -16,6 +16,13 @@ If it does not work, for example if your machine cannot connect to the Internet,
 
 The WinDbg installation only needs to be done once.
 
+> **⚠️ A WinDbg you installed yourself is not used**
+>
+> Binary Ninja cannot use a WinDbg installed from the Microsoft Store or through the standalone installer, because
+> those are packaged in a form the debugger cannot load from. You must either let Binary Ninja download its own
+> copy (the first method below), or extract the package yourself and point `debugger.x64dbgEngPath` at it (the
+> second method). Without one of those, TTD will not work.
+
 ### Install WinDbg Automatically
 
 - Open Binary Ninja

ui/CMakeLists.txt

@@ -12,6 +12,8 @@ list(FILTER SOURCES EXCLUDE REGEX qrc_.*)
 if (NOT WIN32)
     list(REMOVE_ITEM SOURCES ${PROJECT_SOURCE_DIR}/ttdrecord.h)
     list(REMOVE_ITEM SOURCES ${PROJECT_SOURCE_DIR}/ttdrecord.cpp)
+    list(REMOVE_ITEM SOURCES ${PROJECT_SOURCE_DIR}/ttdinstall.h)
+    list(REMOVE_ITEM SOURCES ${PROJECT_SOURCE_DIR}/ttdinstall.cpp)
     list(REMOVE_ITEM SOURCES ${PROJECT_SOURCE_DIR}/windbgupdatedialog.h)
     list(REMOVE_ITEM SOURCES ${PROJECT_SOURCE_DIR}/windbgupdatedialog.cpp)
 endif ()

ui/controlswidget.cpp

@@ -31,6 +31,9 @@ limitations under the License.
 #include <filesystem>
 #include <fstream>
 #include <QFileDialog>
+#ifdef WIN32
+	#include "ttdinstall.h"
+#endif
 
 using namespace BinaryNinjaDebuggerAPI;
 using namespace BinaryNinja;
@@ -274,6 +277,14 @@ void DebugControlsWidget::performLaunch()
 			return;
 	}
 
+#ifdef WIN32
+	// Replaying a trace needs the DbgEng DLLs that come with the WinDbg/TTD package. Say so now, instead of
+	// failing with an opaque "Failed to initialize DbgEng" once the launch is under way.
+	if ((m_controller->GetAdapterType() == "DBGENG_TTD")
+		&& !TTDInstall::EnsureComponentAvailable(this, TTDInstall::ReplayEngine))
+		return;
+#endif
+
 	// TODO: we should have the adapter returns this property
 	bool isLocalLaunch = true;
 	auto adapter = m_controller->GetAdapterType();