From d06fbba10401b239289a172e4d1398e31bdb6c5a Mon Sep 17 00:00:00 2001 From: Nemirtingas Date: Fri, 16 Aug 2019 19:10:12 +0200 Subject: [PATCH] Added error handling when hook fails --- overlay_experimental/DX10_Hook.cpp | 4 +- overlay_experimental/DX10_Hook.h | 4 +- overlay_experimental/DX11_Hook.cpp | 4 +- overlay_experimental/DX11_Hook.h | 4 +- overlay_experimental/DX12_Hook.cpp | 7 +-- overlay_experimental/DX12_Hook.h | 4 +- overlay_experimental/DX9_Hook.cpp | 4 +- overlay_experimental/DX9_Hook.h | 4 +- overlay_experimental/Hook_Manager.cpp | 83 +++++++++++++++++++++++---- overlay_experimental/Hook_Manager.h | 15 ++++- overlay_experimental/OpenGL_Hook.cpp | 26 ++++----- overlay_experimental/OpenGL_Hook.h | 6 +- 12 files changed, 122 insertions(+), 43 deletions(-) diff --git a/overlay_experimental/DX10_Hook.cpp b/overlay_experimental/DX10_Hook.cpp index acaa653..a71135a 100644 --- a/overlay_experimental/DX10_Hook.cpp +++ b/overlay_experimental/DX10_Hook.cpp @@ -12,7 +12,7 @@ // This is created by DX10_Hook::Create, and deleted by the Hook_Manager if not used static DX10_Hook* hook; -void DX10_Hook::start_hook() +bool DX10_Hook::start_hook() { if (!_hooked) { @@ -55,10 +55,12 @@ void DX10_Hook::start_hook() else { PRINT_DEBUG("Failed to hook DirectX 10\n"); + return false; } if(pDevice)pDevice->Release(); if(pSwapChain)pSwapChain->Release(); } + return true; } void DX10_Hook::resetRenderState() diff --git a/overlay_experimental/DX10_Hook.h b/overlay_experimental/DX10_Hook.h index d23338a..ecea403 100644 --- a/overlay_experimental/DX10_Hook.h +++ b/overlay_experimental/DX10_Hook.h @@ -22,7 +22,6 @@ private: DX10_Hook(); virtual ~DX10_Hook(); - void start_hook(); void resetRenderState(); void prepareForOverlay(IDXGISwapChain *pSwapChain); @@ -43,7 +42,8 @@ private: //decltype(D3D10CreateDeviceAndSwapChain)* D3D10CreateDeviceAndSwapChain; public: - static void Create(); // Initialize DX10 Hook. + bool start_hook(); + static DX10_Hook* Inst(); void loadFunctions(ID3D10Device *pDevice, IDXGISwapChain *pSwapChain); }; diff --git a/overlay_experimental/DX11_Hook.cpp b/overlay_experimental/DX11_Hook.cpp index 44967f7..951fb23 100644 --- a/overlay_experimental/DX11_Hook.cpp +++ b/overlay_experimental/DX11_Hook.cpp @@ -22,7 +22,7 @@ HRESULT GetDeviceAndCtxFromSwapchain(IDXGISwapChain* pSwapChain, ID3D11Device** return ret; } -void DX11_Hook::start_hook() +bool DX11_Hook::start_hook() { if (!_hooked) { @@ -65,11 +65,13 @@ void DX11_Hook::start_hook() else { PRINT_DEBUG("Failed to hook DirectX 11\n"); + return false; } if(pDevice) pDevice->Release(); if(pSwapChain) pSwapChain->Release(); } + return true; } void DX11_Hook::resetRenderState() diff --git a/overlay_experimental/DX11_Hook.h b/overlay_experimental/DX11_Hook.h index 031ee9c..c238167 100644 --- a/overlay_experimental/DX11_Hook.h +++ b/overlay_experimental/DX11_Hook.h @@ -22,7 +22,6 @@ private: DX11_Hook(); virtual ~DX11_Hook(); - void start_hook(); void resetRenderState(); void prepareForOverlay(IDXGISwapChain* pSwapChain); @@ -43,7 +42,8 @@ private: //decltype(D3D11CreateDeviceAndSwapChain)* D3D11CreateDeviceAndSwapChain; public: - static void Create(); // Initialize DX11 Hook. + bool start_hook(); + static DX11_Hook* Inst(); void loadFunctions(ID3D11Device *pDevice, IDXGISwapChain *pSwapChain); }; diff --git a/overlay_experimental/DX12_Hook.cpp b/overlay_experimental/DX12_Hook.cpp index 924e9fe..af6548f 100644 --- a/overlay_experimental/DX12_Hook.cpp +++ b/overlay_experimental/DX12_Hook.cpp @@ -9,15 +9,14 @@ #include #include -// This is created by DX12_Hook::Create, and deleted by the Hook_Manager if not used -static DX12_Hook* hook; - -void DX12_Hook::start_hook() +bool DX12_Hook::start_hook() { if (!_hooked) { PRINT_DEBUG("Hooked DirectX 12\n"); + return false; } + return true; } void DX12_Hook::resetRenderState() diff --git a/overlay_experimental/DX12_Hook.h b/overlay_experimental/DX12_Hook.h index 555826f..2d07fd0 100644 --- a/overlay_experimental/DX12_Hook.h +++ b/overlay_experimental/DX12_Hook.h @@ -24,7 +24,6 @@ private: DX12_Hook(); virtual ~DX12_Hook(); - void start_hook(); void resetRenderState(); void prepareForOverlay(IDXGISwapChain* pSwapChain); @@ -43,7 +42,8 @@ private: //decltype(D3D12CreateDevice)* D3D12CreateDevice; public: - static void Create(); // Initialize DX11 Hook. + bool start_hook(); + static DX12_Hook* Inst(); void loadFunctions(ID3D12Device *pDevice, IDXGISwapChain *pSwapChain); }; diff --git a/overlay_experimental/DX9_Hook.cpp b/overlay_experimental/DX9_Hook.cpp index 98a6480..64f1cbe 100644 --- a/overlay_experimental/DX9_Hook.cpp +++ b/overlay_experimental/DX9_Hook.cpp @@ -18,7 +18,7 @@ static DX9_Hook* hook; ///////// ///////// ////////////////////////////////////////////////////////////////// -void DX9_Hook::start_hook() +bool DX9_Hook::start_hook() { if (!_hooked) { @@ -56,11 +56,13 @@ void DX9_Hook::start_hook() else { PRINT_DEBUG("Failed to DirectX 9\n"); + return false; } if(pDeviceEx)pDeviceEx->Release(); if(pD3D)pD3D->Release(); } + return true; } void DX9_Hook::resetRenderState() diff --git a/overlay_experimental/DX9_Hook.h b/overlay_experimental/DX9_Hook.h index e20913a..fb1e623 100644 --- a/overlay_experimental/DX9_Hook.h +++ b/overlay_experimental/DX9_Hook.h @@ -21,7 +21,6 @@ private: DX9_Hook(); virtual ~DX9_Hook(); - void start_hook(); void resetRenderState(); void prepareForOverlay(IDirect3DDevice9* pDevice); @@ -44,7 +43,8 @@ private: //decltype(Direct3DCreate9Ex)* Direct3DCreate9Ex; public: - static void Create(); // Initialize DX9 Hook. + bool start_hook(); + static DX9_Hook* Inst(); void loadFunctions(IDirect3DDevice9Ex *pDeviceEx); }; diff --git a/overlay_experimental/Hook_Manager.cpp b/overlay_experimental/Hook_Manager.cpp index 7d8871a..bf23852 100644 --- a/overlay_experimental/Hook_Manager.cpp +++ b/overlay_experimental/Hook_Manager.cpp @@ -34,7 +34,17 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain* if (pDevice) { Hook_Manager::Inst().UnHookAllRendererDetector(); - DX10_Hook::Create(); + DX10_Hook* hook = DX10_Hook::Inst(); + if (!hook->start_hook()) + { + // Hook failed, start over + delete static_cast(hook); + Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay()); + } + else + { + Hook_Manager::Inst().AddHook(hook); + } } else { @@ -42,15 +52,31 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain* if (pDevice) { Hook_Manager::Inst().UnHookAllRendererDetector(); - DX11_Hook::Create(); + DX11_Hook* hook = DX11_Hook::Inst(); + if (!hook->start_hook()) + { + // Hook failed, start over + delete static_cast(hook); + Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay()); + } + else + { + Hook_Manager::Inst().AddHook(hook); + } } else { _this->GetDevice(__uuidof(ID3D12Device), (void**)& pDevice); - if (pDevice) + DX12_Hook* hook = DX12_Hook::Inst(); + if (!hook->start_hook()) { - Hook_Manager::Inst().UnHookAllRendererDetector(); - DX12_Hook::Create(); + // Hook failed, start over + delete static_cast(hook); + Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay()); + } + else + { + Hook_Manager::Inst().AddHook(hook); } } } @@ -62,28 +88,60 @@ HRESULT STDMETHODCALLTYPE Hook_Manager::MyIDXGISwapChain_Present(IDXGISwapChain* HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion) { Hook_Manager::Inst().UnHookAllRendererDetector(); - DX9_Hook::Create(); + DX9_Hook* hook = DX9_Hook::Inst(); + if (!hook->start_hook()) + { + // Hook failed, start over + delete static_cast(hook); + Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay()); + } + else + { + Hook_Manager::Inst().AddHook(hook); + } return (_this->*_IDirect3DDevice9_Present)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion); } HRESULT STDMETHODCALLTYPE Hook_Manager::MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags) { Hook_Manager::Inst().UnHookAllRendererDetector(); - DX9_Hook::Create(); + DX9_Hook* hook = DX9_Hook::Inst(); + if (!hook->start_hook()) + { + // Hook failed, start over + delete static_cast(hook); + Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay()); + } + else + { + Hook_Manager::Inst().AddHook(hook); + } return (_this->*_IDirect3DDevice9Ex_PresentEx)(pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion, dwFlags); } BOOL WINAPI Hook_Manager::MywglMakeCurrent(HDC hDC, HGLRC hGLRC) { Hook_Manager::Inst().UnHookAllRendererDetector(); - OpenGL_Hook::Create(); - return _wglMakeCurrent(hDC, hGLRC);; + OpenGL_Hook* hook = OpenGL_Hook::Inst(); + if (!hook->start_hook()) + { + // Hook failed, start over + delete static_cast(hook); + Hook_Manager::Inst().HookRenderer(Hook_Manager::Inst().GetOverlay()); + } + else + { + Hook_Manager::Inst().AddHook(hook); + } + + return _wglMakeCurrent(hDC, hGLRC); } void Hook_Manager::HookDXGIPresent() { if (!_dxgi_hooked) { + _dxgi_hooked = true; rendererdetect_hook->BeginHook(); rendererdetect_hook->HookFuncs( @@ -98,6 +156,7 @@ void Hook_Manager::HookDX9Present() { if (!_dx9_hooked) { + _dx9_hooked = true; rendererdetect_hook->BeginHook(); rendererdetect_hook->HookFuncs( @@ -113,6 +172,7 @@ void Hook_Manager::HookwglMakeCurrent() { if (!_ogl_hooked) { + _ogl_hooked = true; rendererdetect_hook->BeginHook(); rendererdetect_hook->HookFuncs( @@ -238,7 +298,9 @@ void Hook_Manager::hook_dx12() { if (!_dxgi_hooked && !_renderer_found) { - DX12_Hook::Create(); + DX12_Hook* hook = DX12_Hook::Inst(); + hook->start_hook(); // TODO: Prints to error log about DX12 Implementation status + delete static_cast(hook); } } @@ -350,6 +412,7 @@ void Hook_Manager::UnHookAllRendererDetector() delete rendererdetect_hook; rendererdetect_hook = nullptr; } + _loadlibrary_hooked = false; _ogl_hooked = false; #ifdef STEAM_WIN32 diff --git a/overlay_experimental/Hook_Manager.h b/overlay_experimental/Hook_Manager.h index bffe287..a55188a 100644 --- a/overlay_experimental/Hook_Manager.h +++ b/overlay_experimental/Hook_Manager.h @@ -38,9 +38,9 @@ protected: Hook_Manager(); virtual ~Hook_Manager(); - - void UnHookAllRendererDetector(); + void UnHookAllRendererDetector(); + // Setup opengl device void hook_opengl(); void HookLoadLibrary(); @@ -52,13 +52,21 @@ protected: bool _dx9_hooked; // DX9 Present and PresentEx Hooked ? bool _dxgi_hooked; // DXGI Present is hooked ? (DX10, DX11, DX12) + // DXGIPresent will be used to detect if DX10, DX11 or DX12 should be used for overlay void HookDXGIPresent(); + // DX9 Present and PresentEx will be used to detect if DX9 should be used for overlay void HookDX9Present(); + // wglMakeCurrent will be used to detect if OpenGL3 should be used for overlay void HookwglMakeCurrent(); + // Setup DX9 Device and get vtable void hook_dx9(); + // Setup DX10 Device and get vtable void hook_dx10(); + // Setup DX11 Device and get vtable void hook_dx11(); + // Setup DX12 Device and get vtable void hook_dx12(); + void create_hookA(const char* libname); void create_hookW(const wchar_t* libname); @@ -66,9 +74,12 @@ protected: static HMODULE WINAPI MyLoadLibraryW(LPCWSTR lpLibFileName); static HMODULE WINAPI MyLoadLibraryExA(LPCTSTR lpLibFileName, HANDLE hFile, DWORD dwFlags); static HMODULE WINAPI MyLoadLibraryExW(LPCWSTR lpLibFileName, HANDLE hFile, DWORD dwFlags); + // If this is called, then DX10, DX11 or DX12 will be used to render overlay static HRESULT STDMETHODCALLTYPE MyIDXGISwapChain_Present(IDXGISwapChain* _this, UINT SyncInterval, UINT Flags); + // If any of theses is called, then DX9 will be used to render overlay static HRESULT STDMETHODCALLTYPE MyPresent(IDirect3DDevice9* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion); static HRESULT STDMETHODCALLTYPE MyPresentEx(IDirect3DDevice9Ex* _this, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags); + // If this is called, then OpenGL 3 will be used to render overlay static BOOL WINAPI MywglMakeCurrent(HDC hDC, HGLRC hGLRC); public: diff --git a/overlay_experimental/OpenGL_Hook.cpp b/overlay_experimental/OpenGL_Hook.cpp index bb5fed6..c326ab2 100644 --- a/overlay_experimental/OpenGL_Hook.cpp +++ b/overlay_experimental/OpenGL_Hook.cpp @@ -13,10 +13,9 @@ #include "steam_overlay.h" -// This is created by OpenGL_Hook::Create, and deleted by the Hook_Manager if not used -static OpenGL_Hook* hook; +OpenGL_Hook* OpenGL_Hook::_inst = nullptr; -void OpenGL_Hook::start_hook() +bool OpenGL_Hook::start_hook() { if (!_hooked) { @@ -40,8 +39,10 @@ void OpenGL_Hook::start_hook() PRINT_DEBUG("Failed to hook OpenGL\n"); /* Problem: glewInit failed, something is seriously wrong. */ PRINT_DEBUG("Error: %s\n", glewGetErrorString(err)); + return false; } } + return true; } void OpenGL_Hook::resetRenderState() @@ -106,8 +107,8 @@ void OpenGL_Hook::prepareForOverlay(HDC hDC) ///////////////////////////////////////////////////////////////////////////////////// BOOL WINAPI OpenGL_Hook::MywglSwapBuffers(HDC hDC) { - hook->prepareForOverlay(hDC); - return hook->wglSwapBuffers(hDC); + OpenGL_Hook::Inst()->prepareForOverlay(hDC); + return OpenGL_Hook::Inst()->wglSwapBuffers(hDC); } OpenGL_Hook::OpenGL_Hook(): @@ -138,18 +139,15 @@ OpenGL_Hook::~OpenGL_Hook() ImGui::DestroyContext(); } - hook = nullptr; + _inst = nullptr; } -void OpenGL_Hook::Create() +OpenGL_Hook* OpenGL_Hook::Inst() { - if (hook == nullptr) - { - hook = new OpenGL_Hook; - hook->start_hook(); - // Register the hook to the Hook Manager - Hook_Manager::Inst().AddHook(hook); - } + if (_inst == nullptr) + _inst = new OpenGL_Hook; + + return _inst; } #endif//NO_OVERLAY \ No newline at end of file diff --git a/overlay_experimental/OpenGL_Hook.h b/overlay_experimental/OpenGL_Hook.h index 80b94e0..bae017c 100644 --- a/overlay_experimental/OpenGL_Hook.h +++ b/overlay_experimental/OpenGL_Hook.h @@ -13,6 +13,8 @@ public: //using wglMakeCurrent_t = BOOL(WINAPI*)(HDC, HGLRC); private: + static OpenGL_Hook* _inst; + // Variables bool initialized; @@ -20,7 +22,6 @@ private: OpenGL_Hook(); virtual ~OpenGL_Hook(); - void start_hook(); void resetRenderState(); void prepareForOverlay(HDC hDC); @@ -35,7 +36,8 @@ private: //wglMakeCurrent_t wglMakeCurrent; public: - static void Create(); // Initialize OGL Hook. + bool start_hook(); + inline static OpenGL_Hook* Inst(); }; #endif//NO_OVERLAY