Add DirectX9Ex support

Cargo.toml

@@ -3,7 +3,7 @@ name = "hudhook"
 version = "0.9.2"
 edition = "2021"
 rust-version = "1.85"
-description = "A graphics API hook with dear imgui render loop. Supports DirectX 9, 11, 12, and OpenGL 3."
+description = "A graphics API hook with dear imgui render loop. Supports DirectX 9, 9Ex, 11, 12, and OpenGL 3."
 homepage = "https://github.com/veeenu/hudhook"
 repository = "https://github.com/veeenu/hudhook"
 documentation = "https://veeenu.github.io/hudhook"
@@ -20,8 +20,9 @@ targets = [
 ]
 
 [features]
-default = ["dx9", "dx11", "dx12", "opengl3", "inject"]
+default = ["dx9", "dx9ex", "dx11", "dx12", "opengl3", "inject"]
 dx9 = []
+dx9ex = ["dx9"]
 dx11 = []
 dx12 = []
 opengl3 = ["dep:gl_generator"]
@@ -54,6 +55,10 @@ crate-type = ["cdylib"]
 name = "demo_hook_dx9"
 crate-type = ["cdylib"]
 
+[[example]]
+name = "demo_hook_dx9ex"
+crate-type = ["cdylib"]
+
 [[example]]
 name = "demo_hook_opengl3"
 crate-type = ["cdylib"]

README.md

@@ -11,7 +11,7 @@
 
 A Rust renderer hook library for building [Dear ImGui](https://github.com/ocornut/imgui) overlays.
 
-Currently supports DirectX 9, DirectX 11, DirectX 12 and OpenGL 3. Runs on Windows and Wine/Proton.
+Currently supports DirectX 9, DirectX 9Ex, DirectX 11, DirectX 12 and OpenGL 3. Runs on Windows and Wine/Proton.
 
 ![hello](tests/hello.jpg)
 
@@ -52,6 +52,12 @@ impl ImguiRenderLoop for MyRenderLoop {
     hudhook!(ImguiDx9Hooks, MyRenderLoop);
 }
 
+{
+    // Use this if hooking into a DirectX 9Ex application.
+    use hudhook::hooks::dx9::ex::ImguiDx9ExHooks;
+    hudhook!(ImguiDx9ExHooks, MyRenderLoop);
+}
+
 {
     // Use this if hooking into a DirectX 11 application.
     use hudhook::hooks::dx11::ImguiDx11Hooks;

examples/demo_hook_dx9ex.rs

@@ -0,0 +1,33 @@
+use hudhook::*;
+
+mod support;
+
+/// Entry point created by the `hudhook` library.
+///
+/// # Safety
+///
+/// haha
+#[no_mangle]
+pub unsafe extern "system" fn DllMain(
+    hmodule: ::hudhook::windows::Win32::Foundation::HINSTANCE,
+    reason: u32,
+    _: *mut ::std::ffi::c_void,
+) {
+    if reason == ::hudhook::windows::Win32::System::SystemServices::DLL_PROCESS_ATTACH {
+        support::setup_tracing();
+        ::hudhook::tracing::trace!("DllMain()");
+        let hmodule_raw = hmodule.0 as usize;
+        ::std::thread::spawn(move || {
+            let hmodule = ::hudhook::windows::Win32::Foundation::HINSTANCE(hmodule_raw as _);
+            if let Err(e) = ::hudhook::Hudhook::builder()
+                .with::<hooks::dx9::ex::ImguiDx9ExHooks>(support::HookExample::new())
+                .with_hmodule(hmodule)
+                .build()
+                .apply()
+            {
+                ::hudhook::tracing::error!("Couldn't apply hooks: {e:?}");
+                ::hudhook::eject();
+            }
+        });
+    }
+}

src/hooks/dx9/ex.rs

@@ -0,0 +1,293 @@
+//! Hooks for DirectX 9Ex.
+
+use std::ffi::c_void;
+use std::sync::atomic::Ordering;
+use std::sync::OnceLock;
+use std::{mem, ptr};
+
+use tracing::{error, trace};
+use windows::core::{Interface, BOOL, HRESULT};
+use windows::Win32::Foundation::{HWND, RECT};
+use windows::Win32::Graphics::Direct3D9::{
+    Direct3DCreate9Ex, IDirect3DDevice9Ex, D3DADAPTER_DEFAULT, D3DCREATE_SOFTWARE_VERTEXPROCESSING,
+    D3DDEVTYPE_NULLREF, D3DDISPLAYMODE, D3DDISPLAYMODEEX, D3DFORMAT, D3DPRESENT_PARAMETERS,
+    D3DSWAPEFFECT_DISCARD, D3D_SDK_VERSION,
+};
+use windows::Win32::Graphics::Gdi::RGNDATA;
+
+use super::{render, reset_pipeline, PIPELINE, RENDER_LOOP};
+use crate::hooks::DummyHwnd;
+use crate::mh::MhHook;
+use crate::{perform_eject, util, Hooks, ImguiRenderLoop, EJECT_REQUESTED, HOOK_EJECTION_BARRIER};
+
+type Dx9ExPresentType = unsafe extern "system" fn(
+    this: IDirect3DDevice9Ex,
+    psourcerect: *const RECT,
+    pdestrect: *const RECT,
+    hdestwindowoverride: HWND,
+    pdirtyregion: *const RGNDATA,
+) -> HRESULT;
+
+type Dx9ExPresentExType = unsafe extern "system" fn(
+    this: IDirect3DDevice9Ex,
+    psourcerect: *const RECT,
+    pdestrect: *const RECT,
+    hdestwindowoverride: HWND,
+    pdirtyregion: *const RGNDATA,
+    dwflags: u32,
+) -> HRESULT;
+
+type Dx9ExResetType =
+    unsafe extern "system" fn(this: IDirect3DDevice9Ex, *const D3DPRESENT_PARAMETERS) -> HRESULT;
+
+type Dx9ExResetExType = unsafe extern "system" fn(
+    this: IDirect3DDevice9Ex,
+    *const D3DPRESENT_PARAMETERS,
+    *const D3DDISPLAYMODEEX,
+) -> HRESULT;
+
+struct Trampolines {
+    dx9ex_present: Dx9ExPresentType,
+    dx9ex_present_ex: Dx9ExPresentExType,
+    dx9ex_reset: Dx9ExResetType,
+    dx9ex_reset_ex: Dx9ExResetExType,
+}
+
+static mut TRAMPOLINES: OnceLock<Trampolines> = OnceLock::new();
+
+unsafe extern "system" fn dx9ex_present_impl(
+    device: IDirect3DDevice9Ex,
+    psourcerect: *const RECT,
+    pdestrect: *const RECT,
+    hdestwindowoverride: HWND,
+    pdirtyregion: *const RGNDATA,
+) -> HRESULT {
+    let _hook_ejection_guard = HOOK_EJECTION_BARRIER.acquire_ejection_guard();
+
+    let Trampolines { dx9ex_present, .. } =
+        TRAMPOLINES.get().expect("DirectX 9Ex trampolines uninitialized");
+
+    if let Err(e) = render(&device) {
+        error!("Render error: {e:?}");
+    }
+
+    trace!("Call IDirect3DDevice9Ex::Present trampoline");
+    let result = dx9ex_present(device, psourcerect, pdestrect, hdestwindowoverride, pdirtyregion);
+    if EJECT_REQUESTED.load(Ordering::SeqCst) {
+        perform_eject();
+    }
+    result
+}
+
+unsafe extern "system" fn dx9ex_present_ex_impl(
+    device: IDirect3DDevice9Ex,
+    psourcerect: *const RECT,
+    pdestrect: *const RECT,
+    hdestwindowoverride: HWND,
+    pdirtyregion: *const RGNDATA,
+    dwflags: u32,
+) -> HRESULT {
+    let _hook_ejection_guard = HOOK_EJECTION_BARRIER.acquire_ejection_guard();
+
+    let Trampolines { dx9ex_present_ex, .. } =
+        TRAMPOLINES.get().expect("DirectX 9Ex trampolines uninitialized");
+
+    if let Err(e) = render(&device) {
+        error!("Render error: {e:?}");
+    }
+
+    trace!("Call IDirect3DDevice9Ex::PresentEx trampoline");
+    let result = dx9ex_present_ex(
+        device,
+        psourcerect,
+        pdestrect,
+        hdestwindowoverride,
+        pdirtyregion,
+        dwflags,
+    );
+    if EJECT_REQUESTED.load(Ordering::SeqCst) {
+        perform_eject();
+    }
+    result
+}
+
+unsafe extern "system" fn dx9ex_reset_impl(
+    this: IDirect3DDevice9Ex,
+    present_params: *const D3DPRESENT_PARAMETERS,
+) -> HRESULT {
+    let _hook_ejection_guard = HOOK_EJECTION_BARRIER.acquire_ejection_guard();
+
+    let Trampolines { dx9ex_reset, .. } =
+        TRAMPOLINES.get().expect("DirectX 9Ex trampolines uninitialized");
+
+    reset_pipeline();
+
+    dx9ex_reset(this, present_params)
+}
+
+unsafe extern "system" fn dx9ex_reset_ex_impl(
+    this: IDirect3DDevice9Ex,
+    present_params: *const D3DPRESENT_PARAMETERS,
+    fullscreen_display_mode: *const D3DDISPLAYMODEEX,
+) -> HRESULT {
+    let _hook_ejection_guard = HOOK_EJECTION_BARRIER.acquire_ejection_guard();
+
+    let Trampolines { dx9ex_reset_ex, .. } =
+        TRAMPOLINES.get().expect("DirectX 9Ex trampolines uninitialized");
+
+    reset_pipeline();
+
+    dx9ex_reset_ex(this, present_params, fullscreen_display_mode)
+}
+
+fn get_target_addrs() -> (Dx9ExPresentType, Dx9ExPresentExType, Dx9ExResetType, Dx9ExResetExType) {
+    let d9 = unsafe { Direct3DCreate9Ex(D3D_SDK_VERSION).unwrap() };
+
+    let mut d3d_display_mode =
+        D3DDISPLAYMODE { Width: 0, Height: 0, RefreshRate: 0, Format: D3DFORMAT(0) };
+    unsafe { d9.GetAdapterDisplayMode(D3DADAPTER_DEFAULT, &mut d3d_display_mode).unwrap() };
+
+    let mut present_params = D3DPRESENT_PARAMETERS {
+        Windowed: BOOL(1),
+        SwapEffect: D3DSWAPEFFECT_DISCARD,
+        BackBufferFormat: d3d_display_mode.Format,
+        ..Default::default()
+    };
+
+    let dummy_hwnd = DummyHwnd::new();
+    let device: IDirect3DDevice9Ex = util::try_out_ptr(|v| unsafe {
+        d9.CreateDeviceEx(
+            D3DADAPTER_DEFAULT,
+            D3DDEVTYPE_NULLREF,
+            dummy_hwnd.hwnd(),
+            D3DCREATE_SOFTWARE_VERTEXPROCESSING as u32,
+            &mut present_params,
+            ptr::null_mut(),
+            v,
+        )
+    })
+    .expect("IDirect3D9Ex::CreateDeviceEx: failed to create device");
+
+    let present_ptr = device.vtable().base__.Present;
+    let present_ex_ptr = device.vtable().PresentEx;
+    let reset_ptr = device.vtable().base__.Reset;
+    let reset_ex_ptr = device.vtable().ResetEx;
+
+    unsafe {
+        (
+            mem::transmute::<
+                unsafe extern "system" fn(
+                    *mut c_void,
+                    *const RECT,
+                    *const RECT,
+                    HWND,
+                    *const RGNDATA,
+                ) -> HRESULT,
+                Dx9ExPresentType,
+            >(present_ptr),
+            mem::transmute::<
+                unsafe extern "system" fn(
+                    *mut c_void,
+                    *const RECT,
+                    *const RECT,
+                    HWND,
+                    *const RGNDATA,
+                    u32,
+                ) -> HRESULT,
+                Dx9ExPresentExType,
+            >(present_ex_ptr),
+            mem::transmute::<
+                unsafe extern "system" fn(*mut c_void, *mut D3DPRESENT_PARAMETERS) -> HRESULT,
+                Dx9ExResetType,
+            >(reset_ptr),
+            mem::transmute::<
+                unsafe extern "system" fn(
+                    *mut c_void,
+                    *mut D3DPRESENT_PARAMETERS,
+                    *mut D3DDISPLAYMODEEX,
+                ) -> HRESULT,
+                Dx9ExResetExType,
+            >(reset_ex_ptr),
+        )
+    }
+}
+
+/// Hooks for DirectX 9Ex.
+pub struct ImguiDx9ExHooks([MhHook; 4]);
+
+impl ImguiDx9ExHooks {
+    /// Construct a set of [`MhHook`]s that will render UI via the
+    /// provided [`ImguiRenderLoop`].
+    ///
+    /// The following functions are hooked:
+    /// - `IDirect3DDevice9Ex::Present`
+    /// - `IDirect3DDevice9Ex::PresentEx`
+    /// - `IDirect3DDevice9Ex::Reset`
+    /// - `IDirect3DDevice9Ex::ResetEx`
+    ///
+    /// An application using DirectX9Ex may render through either the
+    /// `Ex` functions or the ones inherited from `IDirect3DDevice9`, so both
+    /// pairs are hooked.
+    ///
+    /// # Safety
+    ///
+    /// yolo
+    pub unsafe fn new<T>(t: T) -> Self
+    where
+        T: ImguiRenderLoop + Send + Sync + 'static,
+    {
+        let (dx9ex_present_addr, dx9ex_present_ex_addr, dx9ex_reset_addr, dx9ex_reset_ex_addr) =
+            get_target_addrs();
+
+        trace!("IDirect3DDevice9Ex::Present = {:p}", dx9ex_present_addr as *const c_void);
+        trace!("IDirect3DDevice9Ex::PresentEx = {:p}", dx9ex_present_ex_addr as *const c_void);
+        let hook_present =
+            MhHook::new(dx9ex_present_addr as *mut c_void, dx9ex_present_impl as *mut c_void)
+                .expect("couldn't create IDirect3DDevice9Ex::Present hook");
+        let hook_present_ex =
+            MhHook::new(dx9ex_present_ex_addr as *mut c_void, dx9ex_present_ex_impl as *mut c_void)
+                .expect("couldn't create IDirect3DDevice9Ex::PresentEx hook");
+        let hook_reset =
+            MhHook::new(dx9ex_reset_addr as *mut c_void, dx9ex_reset_impl as *mut c_void)
+                .expect("couldn't create IDirect3DDevice9Ex::Reset hook");
+        let hook_reset_ex =
+            MhHook::new(dx9ex_reset_ex_addr as *mut c_void, dx9ex_reset_ex_impl as *mut c_void)
+                .expect("couldn't create IDirect3DDevice9Ex::ResetEx hook");
+
+        RENDER_LOOP.get_or_init(|| Box::new(t));
+        TRAMPOLINES.get_or_init(|| Trampolines {
+            dx9ex_present: mem::transmute::<*mut c_void, Dx9ExPresentType>(
+                hook_present.trampoline(),
+            ),
+            dx9ex_present_ex: mem::transmute::<*mut c_void, Dx9ExPresentExType>(
+                hook_present_ex.trampoline(),
+            ),
+            dx9ex_reset: mem::transmute::<*mut c_void, Dx9ExResetType>(hook_reset.trampoline()),
+            dx9ex_reset_ex: mem::transmute::<*mut c_void, Dx9ExResetExType>(
+                hook_reset_ex.trampoline(),
+            ),
+        });
+
+        Self([hook_present, hook_present_ex, hook_reset, hook_reset_ex])
+    }
+}
+
+impl Hooks for ImguiDx9ExHooks {
+    fn from_render_loop<T>(t: T) -> Box<Self>
+    where
+        Self: Sized,
+        T: ImguiRenderLoop + Send + Sync + 'static,
+    {
+        Box::new(unsafe { Self::new(t) })
+    }
+
+    fn hooks(&self) -> &[MhHook] {
+        &self.0
+    }
+
+    unsafe fn unhook(&mut self) {
+        TRAMPOLINES.take();
+        PIPELINE.take().map(|p| p.into_inner().take());
+        RENDER_LOOP.take();
+    }
+}