From 7bb6685b167f221bf4c1bb98aaf1bdf4880038f1 Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:16:17 +0200 Subject: [PATCH 01/19] Fix recorder identity, lifecycle, RNG checks and dust patch --- .github/workflows/tests.yml | 12 + .gitignore | 3 + CHANGELOG.md | 13 + code/fixes.lua | 65 +-- code/native.lua | 36 ++ code/recorder.lua | 972 ++++++++++++++++-------------------- code/ui.lua | 208 ++++---- code/utils.lua | 58 +-- code/validation.lua | 36 ++ definition.yml | 14 +- init.lua | 318 ++++++------ options.yml | 54 +- tests/test_recorder.py | 148 ++++++ 13 files changed, 1042 insertions(+), 895 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 code/native.lua create mode 100644 code/validation.lua create mode 100644 tests/test_recorder.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..8e96e43 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,12 @@ +name: Recorder tests +on: [push, pull_request] +jobs: + lua: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install lupa==2.6 + - run: python -m unittest discover -s tests -v diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..86fff81 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e6e0745 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +## 0.1.0 + +- Fix the player-identity detour losing its returned registers and apply the configured fixed seed. +- Correct the dust-effect patch's stack cleanup and replace the complete original instruction. +- Isolate recorder instances, clear prefetched commands/RNG on stop, close files on failed startup, and preserve the local player when stopping. +- Validate replay metadata and command records before starting playback; limit and clear the native payload buffer. +- Compare both RNG values and indices and retain the first divergence instead of silently skipping it. +- Check every native patch location before installation and reject unsupported or conflicting executables. This stage supports the original SHC 1.41 executable only. +- Correct the UCP definition/options schema version to 1.0.0 and add portable LuaJIT regression tests. + +This is the repair foundation. Existing recordings are not guaranteed compatible with corrected RNG hooks. The legacy manual lobby setup and playback controls are still present; Extreme, settings snapshots and the replay browser follow separately. diff --git a/code/fixes.lua b/code/fixes.lua index efc4b76..464cf8a 100644 --- a/code/fixes.lua +++ b/code/fixes.lua @@ -1,32 +1,33 @@ ---[[ - Fixes which might be needed for stable recordings - Test this -]]-- - -return { - apply = function() - --- faulty randomness line? This one is called extra when loading from the load game menu - -- dust clouds arising from removed keeps... - core.writeCode(0x004fc4a3, {0x90, 0x90, 0x90, 0x90, 0x90}) - core.writeCode(0x004fc627, {core.AssemblyLambda([[sub esp, 48]], {})}) - - --- Mother RNG fix, nopping out. Could also switch with RNG1 to keep the baby cry noise - core.writeCode(0x005474a3, {0x90, 0x90, 0x90, 0x90, 0x90}) - - -- removing all function calls to RNG1 that come from the music thread. - core.writeCode(0x0047a8d5, {0x90, 0x90, 0x90, 0x90, 0x90}) - core.writeCode(0x0047a86b, {0x90, 0x90, 0x90, 0x90, 0x90}) - - core.writeCode(0x0047c348, {0x90, 0x90, 0x90, 0x90, 0x90}) - - --- Right click menu fixes (during pause): - -- Do not process tick when game is paused and pull down terrain or flatten terrain is used - core.writeCode(0x0045ceff, {0x90, 0x90}) - - -- Do not increase match time when game is paused and flatten terrain is used - core.writeCode(0x0045ce34, {0xEB, 0x46}) - - -- Removing mothers and children (TODO test without this) - core.writeCode(0x004582ed, {0xEB}) - end -} \ No newline at end of file +--[[ + Fixes which might be needed for stable recordings + Test this +]]-- + +return { + apply = function() + --- faulty randomness line? This one is called extra when loading from the load game menu + -- Dust spawning normally returns with RET 0x2c (11 arguments). + -- Replace the whole CALL and perform exactly its 44-byte cleanup. + core.writeCode(0x004fc4a3, {0x90, 0x90, 0x90, 0x90, 0x90}) + core.writeCode(0x004fc627, {0x83, 0xC4, 0x2C, 0x90, 0x90}) + + --- Mother RNG fix, nopping out. Could also switch with RNG1 to keep the baby cry noise + core.writeCode(0x005474a3, {0x90, 0x90, 0x90, 0x90, 0x90}) + + -- removing all function calls to RNG1 that come from the music thread. + core.writeCode(0x0047a8d5, {0x90, 0x90, 0x90, 0x90, 0x90}) + core.writeCode(0x0047a86b, {0x90, 0x90, 0x90, 0x90, 0x90}) + + core.writeCode(0x0047c348, {0x90, 0x90, 0x90, 0x90, 0x90}) + + --- Right click menu fixes (during pause): + -- Do not process tick when game is paused and pull down terrain or flatten terrain is used + core.writeCode(0x0045ceff, {0x90, 0x90}) + + -- Do not increase match time when game is paused and flatten terrain is used + core.writeCode(0x0045ce34, {0xEB, 0x46}) + + -- Removing mothers and children (TODO test without this) + core.writeCode(0x004582ed, {0xEB}) + end +} diff --git a/code/native.lua b/code/native.lua new file mode 100644 index 0000000..7a24185 --- /dev/null +++ b/code/native.lua @@ -0,0 +1,36 @@ +-- Original SHC 1.41 patch bytes. Verify the complete set before writing. +local sites = { + {0x004428B5, {0x89, 0x3D, 0xA4, 0x9C, 0xFE, 0x01}}, + {0x00442877, {0xB9, 0xC0, 0x79, 0xA2, 0x01, 0xE8, 0xBF, 0x7E, 0x02, 0x00}}, + {0x0042BF4C, {0x89, 0x1D, 0xA4, 0x9C, 0xFE, 0x01}}, + {0x004428C6, {0xC7, 0x05, 0x94, 0xA0, 0xFE, 0x01, 0x01, 0x00, 0x00, 0x00}}, + {0x0046B358, {0x89, 0x6E, 0x18, 0x89, 0x46, 0x04}}, + {0x00495337, {0x8B, 0x0D, 0x24, 0x66, 0x12, 0x01}}, + {0x00494BA5, {0xB9, 0xB0, 0xC9, 0xF2, 0x00}}, + {0x00490690, {0x83, 0xEC, 0x68, 0xA1, 0x20, 0x42, 0xB9, 0x00}}, + {0x00487C50, {0x81, 0xEC, 0xF4, 0x03, 0x00, 0x00}}, + {0x00480353, {0x8B, 0x8E, 0x24, 0xD8, 0x02, 0x00}}, + {0x0047EAF0, {0x8B, 0x91, 0x18, 0x06, 0x00, 0x00}}, + {0x004876A6, {0x89, 0xB7, 0xA4, 0x06, 0x00, 0x00}}, + {0x0046A800, {0x8B, 0x81, 0x4C, 0x9C, 0x00, 0x00}}, + {0x0046A7D0, {0x8B, 0x81, 0x48, 0x9C, 0x00, 0x00}}, + {0x0046A74A, {0x83, 0xC4, 0x04, 0x89, 0x46, 0x04}}, + {0x0059AB30, {0x68, 0x48, 0x98, 0x5E, 0x00}}, + {0x004FC4A3, {0xE8, 0x28, 0xE3, 0xF6, 0xFF}}, + {0x004FC627, {0xE8, 0xB4, 0x84, 0xF0, 0xFF}}, + {0x005474A3, {0xE8, 0x28, 0x33, 0xF2, 0xFF}}, + {0x0047A8D5, {0xE8, 0x26, 0xFF, 0xFE, 0xFF}}, + {0x0047A86B, {0xE8, 0x90, 0xFF, 0xFE, 0xFF}}, + {0x0047C348, {0xE8, 0xB3, 0xE4, 0xFE, 0xFF}}, + {0x0045CEFF, {0x75, 0x2D}}, + {0x0045CE34, {0x7D, 0x46}}, + {0x004582ED, {0x75}}, +} +return {verify = function() + for _, site in ipairs(sites) do + local actual = core.readBytes(site[1], #site[2]) + for i, expected in ipairs(site[2]) do + assert(actual[i] == expected, string.format("Recorder: unsupported executable or conflicting patch at %08X; no recorder hooks installed", site[1])) + end + end +end} diff --git a/code/recorder.lua b/code/recorder.lua index e0aa1ce..4e8b14c 100644 --- a/code/recorder.lua +++ b/code/recorder.lua @@ -1,540 +1,432 @@ ---[[ -Replay implementation ---]] - -local utils = require("code/utils") - -local Recorder = {} - --- Recorder class -function Recorder:new(params) - local o = { -- TODO delete? - --finishedPlayback = false, - --finishedRecording = false, - } - - setmetatable(o, self) - self.__index = self - - if params.name then - o:setName(params.name) - end - - self.rngLogMethod = params.rngLogMethod - - self.RECORDER_STATES = {NONE = 0, RECORD = 1, PLAYBACK = 2} - - self.mode = "none" - - -- https://stackoverflow.com/questions/2613734/maximum-packet-size-for-a-tcp-connection | size of GameCommand struct - self.MAX_PACKET_SIZE = 65535 -- 1272 - self.commandDataAddress = core.allocate(self.MAX_PACKET_SIZE, true) - - self.nextUpCommandTimeAddress = core.allocate(4) - - self.commandRecorderState = core.allocate(4) - core.writeInteger(self.commandRecorderState, 0) - - self.rngRecorderState = core.allocate(4) - core.writeInteger(self.rngRecorderState, 0) - - self.infoRecorderState = core.allocate(4) - core.writeInteger(self.infoRecorderState, 0) - - self.cachedRNG = nil - - return o -end - -function Recorder:setName(name) - self.name = name - self.commandsFileName = self.name .. "-commands.json" - self.rngFileName = self.name .. "-rng-sync.json" - self.infoFileName = self.name .. "-infself.json" -end - -function Recorder:reset() - self.mode = "none" - core.writeInteger(self.nextUpCommandTimeAddress, 0) - -- Set SEC_CurrentPlayerSlotID back to 1 - print("reset recorder state") - core.writeInteger(0x01a275dc, 1) -- TODO test if this is needed for multiplayer - - core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.NONE) - core.writeInteger(self.rngRecorderState, self.RECORDER_STATES.NONE) - core.writeInteger(self.infoRecorderState, self.RECORDER_STATES.NONE) - - self.commandsFile:close() - self.rngFile:close() - self.infoFile:close() -end - -function Recorder:startRecording() - self.mode = "record" - if io.open(self.commandsFileName, "r") then - error("Cannot overwrite existing recording: " .. self.commandsFileName) - end - if io.open(self.rngFileName, "r") then - error("Cannot overwrite existing recording: " .. self.rngFileName) - end - if io.open(self.infoFileName, "r") then - error("Cannot overwrite existing recording: " .. self.infoFileName) - end - self.commandsFile = io.open(self.commandsFileName, "w") - self.rngFile = io.open(self.rngFileName, "w") - self.infoFile = io.open(self.infoFileName, "w") - - core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.RECORD) - core.writeInteger(self.rngRecorderState, self.RECORDER_STATES.RECORD) - core.writeInteger(self.infoRecorderState, self.RECORDER_STATES.RECORD) -end - -function Recorder:stopRecording() - self:reset() -end - -function Recorder:startPlayback() - self.mode = "play" - -- Makes sure no valid commands during playback from player input - core.writeInteger(0x191de0c, -1) -- TODO move this somewhere in lobby setup - - if not io.open(self.commandsFileName, "r") then - error("Cannot find recording: " .. self.commandsFileName) - end - if not io.open(self.rngFileName, "r") then - error("Cannot find recording: " .. self.rngFileName) - end - if not io.open(self.infoFileName, "r") then - error("Cannot find recording: " .. self.infoFileName) - end - self.commandsFile = io.open(self.commandsFileName, "r") - self.rngFile = io.open(self.rngFileName, "r") - self.infoFile = io.open(self.infoFileName, "r") - - core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.PLAYBACK) - core.writeInteger(self.rngRecorderState, self.RECORDER_STATES.PLAYBACK) - core.writeInteger(self.infoRecorderState, self.RECORDER_STATES.PLAYBACK) -end - -function Recorder:stopPlayback() - self:reset() -end - -function Recorder:discardFiles() - os.remove(self.commandsFileName) - os.remove(self.rngFileName) - os.remove(self.infoFileName) -end - -function Recorder:saveCommand(commandCategory, time, address, size, player) - local data = json:encode({ - commandCategory = commandCategory, - time = time, - data = utils.tableToHex(core.readBytes(address, size)), - size = size, - player = player, - }) - self.commandsFile:write(data .. "\n") - self.commandsFile:flush() -end - -function Recorder:loadCommand() - local data = self.commandsFile:read() -- reads a line - if data == nil then - return nil - end - return json:decode(data) -- note that data is returned, not an address to data -end - -function Recorder:saveRNG(time, index1, rng1, index2, rng2, extra) - local data = json:encode({ - time = time, - index1 = index1, - rng1 = rng1, - index2 = index2, - rng2 = rng2, - extra = extra, - }) - self.rngFile:write(data .. "\n") - self.rngFile:flush() -end - -function Recorder:loadRNG() - local data = self.rngFile:read() - if data == nil then - return nil - end - return json:decode(data) -end - -function Recorder:saveInfo(gameType, mapSeed, matchSeed, RNGvalue1, RNGvalue2, RNGindex1, RNGindex2) -- Saves RNG starting values to replicate game later - local data = json:encode({ - gameType = gameType, - mapSeed = mapSeed, - matchSeed = matchSeed, - RNGvalue1 = RNGvalue1, - RNGvalue2 = RNGvalue2, - RNGindex1 = RNGindex1, - RNGindex2 = RNGindex2, - }) - self.infoFile:write(data .. "\n") - self.infoFile:flush() -end - -function Recorder:loadInfo() - local data = self.infoFile:read() - if data == nil then - return nil - end - self.info = json:decode(data) -- store in this object I suppose. It is state - return self.info -end - -function Recorder:ScheduleCommandWrapper(commandCategory, player, time, address) - self._scheduleCommand(0x191d768, commandCategory, player, time, address) -end - -function Recorder:scheduleCommand(command) -- TODO test - if command.size > self.MAX_PACKET_SIZE then - print("Not enough memory for data") - error("EXCEEDED MAX PACKET SIZE"); - end - - core.writeBytes(self.commandDataAddress, utils.hexToTable(command.data)) - - self:ScheduleCommandWrapper(command.commandCategory, command.player, command.time, self.commandDataAddress) - -end - -function Recorder:peekCommand() - - if self.nextCommand == nil then - self.nextCommand = self:loadCommand() - end - - if self.nextCommand == nil then - -- We have reached the end of file: EOF - core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.NONE) - end - - return self.nextCommand -end - -function Recorder:peekCommandTime() - local c = self:peekCommand() - if c == nil then - return nil - end - return c.time -end - -function Recorder:consumeSavedCommand() - local c = self.nextCommand - - if c == nil then - c = self:peekCommand() - end - - self.nextCommand = nil - - return c -end - --- 0x004428b5 SHC -function Recorder:onStartSkirmish(registers) -- SINGLEPLAYER ONLY (TODO make this work for multiplayer too) - if core.readInteger(self.commandRecorderState) == self.RECORDER_STATES.RECORD then - local gameRNG1index = core.readInteger(0x01a3160c) - local gameRNG2index = core.readInteger(0x01a31608) - local gameRNG1 = core.readSmallInteger(0x01a279c0) - local gameRNG2 = core.readSmallInteger(0x01a279c2) - local mapSeed = self.mapSeed - local matchSeed = core.readInteger(0x01a279c4) - - self:saveInfo(0, mapSeed, matchSeed, gameRNG1, gameRNG2, gameRNG1index, gameRNG2index) - - print("Saved skirmish information:") - print(string.format("Gametype=%d, mapSeed=%d, matchSeed=%d, gameRNG1=%d, gameRNG2=%d, gameRNG1index=%d, gameRNG2index=%d", 0, mapSeed, matchSeed, gameRNG1, gameRNG2, gameRNG1index, gameRNG2index)) - - elseif core.readInteger(self.commandRecorderState) == self.RECORDER_STATES.PLAYBACK then - local skirmishInfo = self:loadInfo() - local populateRNG1040 = core.exposeCode(0x0046a760, 1, 1) - - -- Set SEC_CurrentPlayerSlotID to 0 (Disallows actions during playback) TODO move this - print("recorder in Playback state") - --core.writeInteger(0x01a275dc, 0) - - -- Load mapSeed and fill RNG table - core.writeInteger(0x01a279c4, skirmishInfo.mapSeed) - populateRNG1040(0x01a279c0); - - -- Load matchSeed (populateRNG1040() is called later in LaunchGame() after the map is setup) - core.writeInteger(0x01a279c4, skirmishInfo.matchSeed) - - -- Load starting RNG values - core.writeInteger(0x01a3160c, skirmishInfo.RNGindex1) - core.writeInteger(0x01a31608, skirmishInfo.RNGindex2) - core.writeSmallInteger(0x01a279c0, skirmishInfo.RNGvalue1) - core.writeSmallInteger(0x01a279c2, skirmishInfo.RNGvalue2) - - print("Loaded skirmish information:") - print(string.format("Gametype=%d, mapSeed=%d, matchSeed=%d, gameRNG1=%d, gameRNG2=%d, gameRNG1index=%d, gameRNG2index=%d", skirmishInfo.gameType, skirmishInfo.mapSeed, skirmishInfo.matchSeed, skirmishInfo.RNGvalue1, skirmishInfo.RNGvalue2, skirmishInfo.RNGindex1, skirmishInfo.RNGindex2)) - - end - return registers -end - --- 0x00442877 SHC -function Recorder:onBeforeSetMatchSeed(registers) -- SINGLEPLAYER ONLY (TODO make this work for multiplayer too) - self.mapSeed = core.readInteger(0x01a279c4) - local setTimeBasedSeed = core.exposeCode(0x0046a740, 1, 1) - - -- original code - setTimeBasedSeed(0x01a279c0) - return registers -end - --- 0x0042bf4c SHC -function Recorder:onCustomSkirmishGame(registers) -- SINGLEPLAYER ONLY (TODO test if playerIDs work fine in multiplayer) - -- Make singleplayer skirmishes use real playerID for commands, not -1 - local DAT_QueuedCommandPlayer = 0x191de0c - core.writeInteger(DAT_QueuedCommandPlayer, 01) -- In Singleplayer multiplayerID is always 01 - return registers -end - -function Recorder:scheduleNextCommand(registers) - print("Consuming command") - local c = self:consumeSavedCommand() - if c == nil then - print("... no command left") - --self.finishedPlayback = true -- already done by peekCommand higher up in the call hierarchy - core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.NONE) - return - end - - print(string.format("Matchtime now: %d", core.readInteger(0x01fe7da8))) - print(string.format("Scheduling the command: Command", c.commandCategory, c.time, self.commandDataAddress, c.size, c.player)) - self:scheduleCommand(c) - - print("Peeking at the next command") - local c = self:peekCommand() - if c == nil then - print("... no command left") - --self.finishedPlayback = true -- already done by peekCommand higher up in the call hierarchy - core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.NONE) - return - end - - print(string.format("Setting next trigger time: %d", c.time)) - core.writeInteger(self.nextUpCommandTimeAddress, c.time) -end - -function Recorder:onReceiveAllTransmittedCommandsASM(scheduleNextCommandAddress) - -- Schedules saved commands if commandRecorderState is in PLAYBACK mode - return { - core.AssemblyLambda([[ - startOfFunction: - mov eax, [commandRecorderState] - cmp eax, 2 - jne endOfFunction - - checkStarted: - mov eax, dword [SEC_MatchTime] - cmp eax, 0 - jle endOfFunction - - checkTime: - add eax, 64 - mov edx, dword [nextUpCommandTimeAddress] - cmp eax, edx - jg takeCommand - jmp endOfFunction - - takeCommand: - call scheduleNextCommandAddress - jmp startOfFunction - - endOfFunction: - ]], { - commandRecorderState = self.commandRecorderState, - SEC_MatchTime = 0x01fe7da8, - nextUpCommandTimeAddress = self.nextUpCommandTimeAddress, - scheduleNextCommandAddress = scheduleNextCommandAddress, - }) -} -end - -function Recorder:onCommand(commandCategory, time, address, size, player) - if core.readInteger(self.commandRecorderState) == 1 and time > 0 then - print(string.format("Recording Command", commandCategory, time, address, size, player)) - self:saveCommand(commandCategory, time, address, size, player) - end -end - - -- on sent commands -function Recorder:onTransmitCommand(registers) - local commandCategory = core.readInteger(registers.ESP + 4) - local time = core.readInteger(registers.ESP + 8) - local address = core.readInteger(registers.ESP + 12) - local size = core.readInteger(registers.ESP + 16) - -- local idTo = core.readInteger(registers.ESP + 20) - local player = core.readInteger(0x0191de0c) - print(string.format("Transmitted Command", commandCategory, time, address, size, player)) - - self:onCommand(commandCategory, time, address, size, player) -end - --- on received commands -function Recorder:onScheduleCommand(registers) - local commandCategory = core.readInteger(registers.ESP + 4 + 0x10) - local player = core.readInteger(registers.ESP + 8 + 0x10) - local time = core.readInteger(registers.ESP + 12 + 0x10) - local address = core.readInteger(registers.ESP + 16 + 0x10) - local size = core.readInteger(0x0194af98) - print(string.format("Received Command", commandCategory, time, address, size, player)) - - self:onCommand(commandCategory, time, address, size, player) -end - -function Recorder:fakeMultiplayerIdentities(registers) - if originalInMultiplayer then - registers.EDX = 1 -- multiplayer - else - registers.EDX = core.readInteger(registers.ECX + 0x618) -- original game mode - end - return registers -end - -function Recorder:syncCheck(registers, traceF) - local gameTime = core.readInteger(0x01fe7da8) - local gameRNG1 = core.readSmallInteger(0x01a279c0) - local gameRNG2 = core.readSmallInteger(0x01a279c2) - local gameRNG1index = core.readInteger(0x01a3160c) - local gameRNG2index = core.readInteger(0x01a31608) - - if self.mode == "record" then - - if core.readInteger(self.rngRecorderState) ~= self.RECORDER_STATES.RECORD then - return - end - - -- print("Recording sync data: " .. string.format("%0.16X\t%0.16X\t%0.16X", gameTime, gameRNG1, gameRNG2)) - if self.rngLogMethod == "trace" then - local returnAddress = core.readInteger(registers.ESP) - local returnAddress1 = nil - local returnAddress2 = nil - if traceF == 1 then - returnAddress1 = returnAddress - end - if traceF == 2 then - returnAddress2 = returnAddress - end - self:saveRNG(gameTime, gameRNG1index, gameRNG1, gameRNG2index, gameRNG2, {ra1 = returnAddress1, ra2 = returnAddress2}) - else - self:saveRNG(gameTime, gameRNG1index, gameRNG1, gameRNG2index, gameRNG2) - end - - elseif self.mode == "play" then - - if core.readInteger(self.rngRecorderState) ~= self.RECORDER_STATES.PLAYBACK then - return - end - - if self.cachedRNG == nil then - self.cachedRNG = self:loadRNG() - end - - if self.cachedRNG == nil then - print("No more sync data left") - core.writeInteger(self.rngRecorderState, self.RECORDER_STATES.NONE) - return - end - - if gameTime > self.cachedRNG.time then - while (self.cachedRNG ~= nil) and (gameTime > self.cachedRNG.time) do - print("Game is ahead of rng info, skipping...") - self.cachedRNG = self:loadRNG() - end - end - - if self.cachedRNG == nil then - print("No more sync data left") - core.writeInteger(self.rngRecorderState, self.RECORDER_STATES.NONE) - return - end - - if self.cachedRNG.time > gameTime then - return - end - - local data = self.cachedRNG - self.cachedRNG = nil - - if data.time ~= gameTime then - print("time mismatch between data: " .. tonumber(data.time) .. " and game: " .. tonumber(gameTime)) - return - end - - -- print(string.format("SYNC data at (time, rng1, rng2): %d, %d, %d", data.time, data.rng1, data.rng2)) - - local anyDesync = false - - if traceRNG1 then - if data.rng1 ~= gameRNG1 then - anyDesync = true - print(string.format("DESYNC in RNG1: data = time %d, rng1 %d, rng2 %d; game = time %d, %d, %d", data.time, data.rng1, data.rng2, gameTime, gameRNG1, gameRNG2)) - end - end - - if traceRNG2 then - if data.rng2 ~= gameRNG2 then - anyDesync = true - print(string.format("DESYNC in RNG2: data = time %d, rng1 %d, rng2 %d; game = time %d, %d, %d", data.time, data.rng1, data.rng2, gameTime, gameRNG1, gameRNG2)) - end - end - - local returnAddress = 0 - - if self.rngLogMethod == "trace" then - returnAddress = core.readInteger(registers.ESP) - local returnAddress1 = data.extra.ra1 - local returnAddress2 = data.extra.ra2 - - -- print(returnAddress, returnAddress1, returnAddress2) - - if traceF == 1 then - if returnAddress1 ~= nil then - if (returnAddress1 ~= returnAddress) then - anyDesync = true - print(string.format("DESYNC returnAddress1: data = time %d, ret %X; game = time %d, ret %X", data.time, returnAddress1, gameTime, returnAddress)) - end - else - anyDesync = true - print(string.format("DESYNC returnAddress, expected 2 but got 1: data = time %d, ret2 %X; game = time %d, ret1 %X", data.time, returnAddress2, gameTime, returnAddress)) - end - end - - - if traceF == 2 then - if returnAddress2 ~= nil then - if (returnAddress2 ~= returnAddress) then - anyDesync = true - print(string.format("DESYNC returnAddress2: data = time %d, ret %X; game = time %d, ret %X", data.time, returnAddress2, gameTime, returnAddress)) - end - else - anyDesync = true - print(string.format("DESYNC returnAddress, expected 1 but got 2: data = time %d, ret1 %X; game = time %d, ret2 %X", data.time, returnAddress1, gameTime, returnAddress)) - end - end - end - - if not anyDesync then - --print(string.format("SYNCED: data = time %d, rng1 %d, rng2 %d; game = time %d, %d, %d", data.time, data.rng1, data.rng2, gameTime, gameRNG1, gameRNG2)) - else - --print("") - --debug.debug() -- TODO decide - end - - end -end - -return Recorder \ No newline at end of file +--[[ +Replay implementation +--]] + +local utils = require("code/utils") + +local validation = require("code/validation") +local Recorder = {} + +-- Recorder class +function Recorder:new(params) + local o = setmetatable({}, {__index = self}) + o:setName(params.name or "test_recording1") + o.rngLogMethod = params.rngLogMethod or "trace" + o.RECORDER_STATES = {NONE = 0, RECORD = 1, PLAYBACK = 2} + o.mode = "none" + o.MAX_PACKET_SIZE = validation.MAX_PAYLOAD + o.commandDataAddress = core.allocate(o.MAX_PACKET_SIZE, true) + o.nextUpCommandTimeAddress = core.allocate(4, true) + o.commandRecorderState = core.allocate(4, true) + o.rngRecorderState = core.allocate(4, true) + o.infoRecorderState = core.allocate(4, true) + + return o +end + +function Recorder:setName(name) + self.name = name + self.commandsFileName = self.name .. "-commands.json" + self.rngFileName = self.name .. "-rng-sync.json" + self.infoFileName = self.name .. "-infself.json" +end + +function Recorder:reset() + self.mode = "none" + self.nextCommand, self.cachedRNG, self.info, self.mapSeed = nil, nil, nil, nil + self.lastCommandTime = nil + core.writeInteger(self.nextUpCommandTimeAddress, 0) + for _, address in ipairs({self.commandRecorderState, self.rngRecorderState, self.infoRecorderState}) do + core.writeInteger(address, self.RECORDER_STATES.NONE) + end + for _, key in ipairs({"commandsFile", "rngFile", "infoFile"}) do + local file = self[key] + self[key] = nil + if file then file:close() end + end +end + +-- Open the complete set before committing session state. A failed open must +-- leave the current player and the recorder unchanged. +function Recorder:openFiles(mode) + assert(self.mode == "none", "A replay session is already active") + local opened, created = {}, {} + local ok, message = pcall(function() + if mode == "w" then + for _, key in ipairs({"commands", "rng", "info"}) do + local path = self[key .. "FileName"] + local existing = io.open(path, "r") + if existing then existing:close(); error("Cannot overwrite recording: " .. path) end + end + end + for _, key in ipairs({"commands", "rng", "info"}) do + local path = self[key .. "FileName"] + local file, reason = io.open(path, mode) + assert(file, "Cannot open recording " .. path .. ": " .. tostring(reason)) + opened[key .. "File"] = file + if mode == "w" then created[#created + 1] = path end + end + if mode == "r" then + local line = opened.infoFile:read() + local info = validation.info(line and json:decode(line)) + opened.infoFile:seek("set", 0) + -- Read all commands before any native mutation, including late corruption. + while true do + local commandLine = opened.commandsFile:read() + if not commandLine then break end + validation.command(json:decode(commandLine)) + end + opened.commandsFile:seek("set", 0) + end + end) + if not ok then + for _, file in pairs(opened) do file:close() end + for _, path in ipairs(created) do os.remove(path) end + error(message) + end + self.nextCommand, self.cachedRNG, self.lastCommandTime = nil, nil, nil + self.firstDesync = nil + for key, file in pairs(opened) do self[key] = file end +end + +function Recorder:startRecording() + self:openFiles("w") + self.mode = "record" + core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.RECORD) + core.writeInteger(self.rngRecorderState, self.RECORDER_STATES.RECORD) + core.writeInteger(self.infoRecorderState, self.RECORDER_STATES.RECORD) +end + +function Recorder:stopRecording() self:reset() end + +function Recorder:startPlayback() + self:openFiles("r") + self.mode = "play" + core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.PLAYBACK) + core.writeInteger(self.rngRecorderState, self.RECORDER_STATES.PLAYBACK) + core.writeInteger(self.infoRecorderState, self.RECORDER_STATES.PLAYBACK) +end + +function Recorder:stopPlayback() self:reset() end + +function Recorder:discardFiles() + os.remove(self.commandsFileName) + os.remove(self.rngFileName) + os.remove(self.infoFileName) +end + +function Recorder:saveCommand(commandCategory, time, address, size, player) + local data = json:encode({ + commandCategory = commandCategory, + time = time, + data = utils.tableToHex(core.readBytes(address, size)), + size = size, + player = player, + }) + self.commandsFile:write(data .. "\n") + self.commandsFile:flush() +end + +function Recorder:loadCommand() + local data = self.commandsFile:read() -- reads a line + if data == nil then + return nil + end + return json:decode(data) -- note that data is returned, not an address to data +end + +function Recorder:saveRNG(time, index1, rng1, index2, rng2, extra) + local data = json:encode({ + time = time, + index1 = index1, + rng1 = rng1, + index2 = index2, + rng2 = rng2, + extra = extra, + }) + self.rngFile:write(data .. "\n") + self.rngFile:flush() +end + +function Recorder:loadRNG() + local data = self.rngFile:read() + if data == nil then + return nil + end + return json:decode(data) +end + +function Recorder:saveInfo(gameType, mapSeed, matchSeed, RNGvalue1, RNGvalue2, RNGindex1, RNGindex2) -- Saves RNG starting values to replicate game later + local data = json:encode({ + gameType = gameType, + mapSeed = mapSeed, + matchSeed = matchSeed, + RNGvalue1 = RNGvalue1, + RNGvalue2 = RNGvalue2, + RNGindex1 = RNGindex1, + RNGindex2 = RNGindex2, + }) + self.infoFile:write(data .. "\n") + self.infoFile:flush() +end + +function Recorder:loadInfo() + local data = self.infoFile:read() + if data == nil then + return nil + end + self.info = json:decode(data) -- store in this object I suppose. It is state + return self.info +end + +function Recorder:ScheduleCommandWrapper(commandCategory, player, time, address) + self._scheduleCommand(0x191d768, commandCategory, player, time, address) +end + +function Recorder:scheduleCommand(command) + validation.command(command) + local bytes = utils.hexToTable(command.data) + -- The native scheduler derives its own length. Never let a short record expose + -- stale bytes from the previous command's allocation. + for i = #bytes + 1, self.MAX_PACKET_SIZE do bytes[i] = 0 end + core.writeBytes(self.commandDataAddress, bytes) + self:ScheduleCommandWrapper(command.commandCategory, command.player, command.time, self.commandDataAddress) +end + +function Recorder:peekCommand() + + if self.nextCommand == nil then + self.nextCommand = self:loadCommand() + end + + if self.nextCommand == nil then + -- We have reached the end of file: EOF + core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.NONE) + end + + return self.nextCommand +end + +function Recorder:peekCommandTime() + local c = self:peekCommand() + if c == nil then + return nil + end + return c.time +end + +function Recorder:consumeSavedCommand() + local c = self.nextCommand + + if c == nil then + c = self:peekCommand() + end + + self.nextCommand = nil + + return c +end + +-- 0x004428b5 SHC +function Recorder:onStartSkirmish(registers) -- SINGLEPLAYER ONLY (TODO make this work for multiplayer too) + if core.readInteger(self.commandRecorderState) == self.RECORDER_STATES.RECORD then + local gameRNG1index = core.readInteger(0x01a3160c) + local gameRNG2index = core.readInteger(0x01a31608) + local gameRNG1 = core.readSmallInteger(0x01a279c0) + local gameRNG2 = core.readSmallInteger(0x01a279c2) + local mapSeed = self.mapSeed + local matchSeed = core.readInteger(0x01a279c4) + + self:saveInfo(0, mapSeed, matchSeed, gameRNG1, gameRNG2, gameRNG1index, gameRNG2index) + + print("Saved skirmish information:") + print(string.format("Gametype=%d, mapSeed=%d, matchSeed=%d, gameRNG1=%d, gameRNG2=%d, gameRNG1index=%d, gameRNG2index=%d", 0, mapSeed, matchSeed, gameRNG1, gameRNG2, gameRNG1index, gameRNG2index)) + + elseif core.readInteger(self.commandRecorderState) == self.RECORDER_STATES.PLAYBACK then + local skirmishInfo = self:loadInfo() + local populateRNG1040 = core.exposeCode(0x0046a760, 1, 1) + + -- Set SEC_CurrentPlayerSlotID to 0 (Disallows actions during playback) TODO move this + print("recorder in Playback state") + --core.writeInteger(0x01a275dc, 0) + + -- Load mapSeed and fill RNG table + core.writeInteger(0x01a279c4, skirmishInfo.mapSeed) + populateRNG1040(0x01a279c0); + + -- Load matchSeed (populateRNG1040() is called later in LaunchGame() after the map is setup) + core.writeInteger(0x01a279c4, skirmishInfo.matchSeed) + + -- Load starting RNG values + core.writeInteger(0x01a3160c, skirmishInfo.RNGindex1) + core.writeInteger(0x01a31608, skirmishInfo.RNGindex2) + core.writeSmallInteger(0x01a279c0, skirmishInfo.RNGvalue1) + core.writeSmallInteger(0x01a279c2, skirmishInfo.RNGvalue2) + + print("Loaded skirmish information:") + print(string.format("Gametype=%d, mapSeed=%d, matchSeed=%d, gameRNG1=%d, gameRNG2=%d, gameRNG1index=%d, gameRNG2index=%d", skirmishInfo.gameType, skirmishInfo.mapSeed, skirmishInfo.matchSeed, skirmishInfo.RNGvalue1, skirmishInfo.RNGvalue2, skirmishInfo.RNGindex1, skirmishInfo.RNGindex2)) + + end + return registers +end + +-- 0x00442877 SHC +function Recorder:onBeforeSetMatchSeed(registers) -- SINGLEPLAYER ONLY (TODO make this work for multiplayer too) + self.mapSeed = core.readInteger(0x01a279c4) + local setTimeBasedSeed = core.exposeCode(0x0046a740, 1, 1) + + -- original code + setTimeBasedSeed(0x01a279c0) + return registers +end + +-- 0x0042bf4c SHC +function Recorder:onCustomSkirmishGame(registers) -- SINGLEPLAYER ONLY (TODO test if playerIDs work fine in multiplayer) + -- Make singleplayer skirmishes use real playerID for commands, not -1 + local DAT_QueuedCommandPlayer = 0x191de0c + core.writeInteger(DAT_QueuedCommandPlayer, 01) -- In Singleplayer multiplayerID is always 01 + return registers +end + +function Recorder:scheduleNextCommand(registers) + print("Consuming command") + local c = self:consumeSavedCommand() + if c == nil then + print("... no command left") + --self.finishedPlayback = true -- already done by peekCommand higher up in the call hierarchy + core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.NONE) + return + end + + print(string.format("Matchtime now: %d", core.readInteger(0x01fe7da8))) + print(string.format("Scheduling the command: Command", c.commandCategory, c.time, self.commandDataAddress, c.size, c.player)) + self:scheduleCommand(c) + + print("Peeking at the next command") + local c = self:peekCommand() + if c == nil then + print("... no command left") + --self.finishedPlayback = true -- already done by peekCommand higher up in the call hierarchy + core.writeInteger(self.commandRecorderState, self.RECORDER_STATES.NONE) + return + end + + print(string.format("Setting next trigger time: %d", c.time)) + core.writeInteger(self.nextUpCommandTimeAddress, c.time) +end + +function Recorder:onReceiveAllTransmittedCommandsASM(scheduleNextCommandAddress) + -- Schedules saved commands if commandRecorderState is in PLAYBACK mode + return { + core.AssemblyLambda([[ + startOfFunction: + mov eax, [commandRecorderState] + cmp eax, 2 + jne endOfFunction + + checkStarted: + mov eax, dword [SEC_MatchTime] + cmp eax, 0 + jle endOfFunction + + checkTime: + add eax, 64 + mov edx, dword [nextUpCommandTimeAddress] + cmp eax, edx + jg takeCommand + jmp endOfFunction + + takeCommand: + call scheduleNextCommandAddress + jmp startOfFunction + + endOfFunction: + ]], { + commandRecorderState = self.commandRecorderState, + SEC_MatchTime = 0x01fe7da8, + nextUpCommandTimeAddress = self.nextUpCommandTimeAddress, + scheduleNextCommandAddress = scheduleNextCommandAddress, + }) +} +end + +function Recorder:onCommand(commandCategory, time, address, size, player) + if core.readInteger(self.commandRecorderState) == 1 and time > 0 then + print(string.format("Recording Command", commandCategory, time, address, size, player)) + self:saveCommand(commandCategory, time, address, size, player) + end +end + + -- on sent commands +function Recorder:onTransmitCommand(registers) + local commandCategory = core.readInteger(registers.ESP + 4) + local time = core.readInteger(registers.ESP + 8) + local address = core.readInteger(registers.ESP + 12) + local size = core.readInteger(registers.ESP + 16) + -- local idTo = core.readInteger(registers.ESP + 20) + local player = core.readInteger(0x0191de0c) + print(string.format("Transmitted Command", commandCategory, time, address, size, player)) + + self:onCommand(commandCategory, time, address, size, player) +end + +-- on received commands +function Recorder:onScheduleCommand(registers) + local commandCategory = core.readInteger(registers.ESP + 4 + 0x10) + local player = core.readInteger(registers.ESP + 8 + 0x10) + local time = core.readInteger(registers.ESP + 12 + 0x10) + local address = core.readInteger(registers.ESP + 16 + 0x10) + local size = core.readInteger(0x0194af98) + print(string.format("Received Command", commandCategory, time, address, size, player)) + + self:onCommand(commandCategory, time, address, size, player) +end + +function Recorder:fakeMultiplayerIdentities(registers) + if self.originalInMultiplayer then + registers.EDX = 1 -- multiplayer + else + registers.EDX = core.readInteger(registers.ECX + 0x618) -- original game mode + end + return registers +end + +function Recorder:syncCheck(registers, traceF) + local actual = { + time = core.readInteger(0x01fe7da8), + rng1 = core.readSmallInteger(0x01a279c0), + rng2 = core.readSmallInteger(0x01a279c2), + index1 = core.readInteger(0x01a3160c), + index2 = core.readInteger(0x01a31608), + extra = {}, + } + if traceF then actual.extra["ra" .. traceF] = core.readInteger(registers.ESP) end + if self.mode == "record" and core.readInteger(self.rngRecorderState) == 1 then + self:saveRNG(actual.time, actual.index1, actual.rng1, actual.index2, actual.rng2, actual.extra) + elseif self.mode == "play" and core.readInteger(self.rngRecorderState) == 2 then + local expected = self.cachedRNG or self:loadRNG() + self.cachedRNG = nil + if not expected then + core.writeInteger(self.rngRecorderState, 0) + return registers + end + local reason + for _, key in ipairs({"time", "rng1", "rng2", "index1", "index2"}) do + if actual[key] ~= expected[key] then reason = key; break end + end + if not reason and traceF and self.rngLogMethod == "trace" then + if not expected.extra or expected.extra["ra" .. traceF] ~= actual.extra["ra" .. traceF] then + reason = "RNG call site" + end + end + if reason and not self.firstDesync then + self.firstDesync = {reason = reason, expected = expected, actual = actual} + print("REPLAY DESYNC at tick " .. actual.time .. ": " .. reason) + core.writeInteger(self.rngRecorderState, 0) + core.writeInteger(self.commandRecorderState, 0) + end + end + return registers +end + +return Recorder diff --git a/code/ui.lua b/code/ui.lua index 2f8abda..a321160 100644 --- a/code/ui.lua +++ b/code/ui.lua @@ -1,104 +1,104 @@ -local menuItemSize = 0x50 - -local originalMenuItemArraySize = 0x1D10 -local originalMenuItemArrayAddress = 0x005e9848 - -local newSize = originalMenuItemArraySize + 2 * menuItemSize -- 2 additional MenuItems -local newMenuItemArrayAddress = core.allocate(newSize, true) - -local newItem1Address = newMenuItemArrayAddress + newSize - 3 * menuItemSize -local newItem2Address = newMenuItemArrayAddress + newSize - 2 * menuItemSize -local newTerminatorAddress = newMenuItemArrayAddress + newSize - 1 * menuItemSize - -local placeRecordButton = function(recorder) - -- init button - - local recordButtonCallback = function() - if recorder.mode == "none" then - recorder:startRecording() - core.writeInteger(newItem1Address + 32, 0x40000239) -- buttonGraphic - print("Starting new recording!") - elseif recorder.mode == "record" then - recorder:stopRecording() - recorder:discardFiles() - core.writeInteger(newItem1Address + 32, 0x4000023A) -- buttonGraphic - print("discarded") - end - end - - core.writeInteger(newItem1Address + 0, 0x00000003) -- type - core.writeInteger(newItem1Address + 4, 380) -- xPos - core.writeInteger(newItem1Address + 8, 560) -- yPos - core.writeInteger(newItem1Address + 12, 80) -- width - core.writeInteger(newItem1Address + 16, 80) -- height - core.writeInteger(newItem1Address + 20, utils.createLuaFunctionWrapper(recordButtonCallback)) -- - core.writeInteger(newItem1Address + 24, 2) -- callbackParam - core.writeInteger(newItem1Address + 28, 0x0042AE90) -- renderFunc - core.writeInteger(newItem1Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on - core.writeInteger(newItem1Address + 36, 0x00000003) -- unknown - core.writeInteger(newItem1Address + 40, 0x00000000) -- unknown - core.writeInteger(newItem1Address + 44, 0x00000000) -- unknown - core.writeInteger(newItem1Address + 48, 0x0000FFF0) -- unknown - - core.writeInteger(newItem1Address + 72, 0x00000045) -- unknown -end - -local placePlaybackButton = function(recorder) - -- init button - - local playbackButtonCallback = function() - print("playback button") - if recorder.mode == "none" then - recorder:startPlayback() - core.writeInteger(newItem2Address + 32, 0x40000239) -- buttonGraphic - print("Starting new playback!") - elseif recorder.mode == "play" then - recorder:stopPlayback() - core.writeInteger(newItem2Address + 32, 0x4000023A) -- buttonGraphic - print("discarded playback") - end - end - - core.writeInteger(newItem2Address + 0, 0x00000003) -- type - core.writeInteger(newItem2Address + 4, 290) -- xPos - core.writeInteger(newItem2Address + 8, 560) -- yPos - core.writeInteger(newItem2Address + 12, 80) -- width - core.writeInteger(newItem2Address + 16, 80) -- height - core.writeInteger(newItem2Address + 20, utils.createLuaFunctionWrapper(playbackButtonCallback)) -- - core.writeInteger(newItem2Address + 24, 2) -- callbackParam - core.writeInteger(newItem2Address + 28, 0x0042AE90) -- renderFunc - core.writeInteger(newItem2Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on - core.writeInteger(newItem2Address + 36, 0x00000003) -- unknown - core.writeInteger(newItem2Address + 40, 0x00000000) -- unknown - core.writeInteger(newItem2Address + 44, 0x00000000) -- unknown - core.writeInteger(newItem2Address + 48, 0x0000FFF0) -- unknown - - core.writeInteger(newItem2Address + 72, 0x00000045) -- unknown -end - -return { - createButtons = function(recorder) - -- copy original ui array to new array - core.copyMemory(newMenuItemArrayAddress, originalMenuItemArrayAddress, originalMenuItemArraySize) - - -- copy 0x66 terminator one element further back - core.copyMemory(newTerminatorAddress, - newItem1Address, - menuItemSize) - - placeRecordButton(recorder) - placePlaybackButton(recorder) - - -- overwrite reference - core.writeCode(0x0059AB30, { core.AssemblyLambda([[ - push newMenuItemArrayAddress - ]], - { newMenuItemArrayAddress = newMenuItemArrayAddress, }) }) - - end, - - resetButtons = function() - core.writeInteger(newItem1Address + 32, 0x4000023A) -- reset recording button graphic - core.writeInteger(newItem2Address + 32, 0x4000023A) -- reset playback button graphic - end, -} \ No newline at end of file +local menuItemSize = 0x50 + +local originalMenuItemArraySize = 0x1D10 +local originalMenuItemArrayAddress = 0x005e9848 + +local newSize = originalMenuItemArraySize + 2 * menuItemSize -- 2 additional MenuItems +local newMenuItemArrayAddress = core.allocate(newSize, true) + +local newItem1Address = newMenuItemArrayAddress + newSize - 3 * menuItemSize +local newItem2Address = newMenuItemArrayAddress + newSize - 2 * menuItemSize +local newTerminatorAddress = newMenuItemArrayAddress + newSize - 1 * menuItemSize + +local placeRecordButton = function(recorder) + -- init button + + local recordButtonCallback = function() + if recorder.mode == "none" then + recorder:startRecording() + core.writeInteger(newItem1Address + 32, 0x40000239) -- buttonGraphic + print("Starting new recording!") + elseif recorder.mode == "record" then + recorder:stopRecording() + recorder:discardFiles() + core.writeInteger(newItem1Address + 32, 0x4000023A) -- buttonGraphic + print("discarded") + end + end + + core.writeInteger(newItem1Address + 0, 0x00000003) -- type + core.writeInteger(newItem1Address + 4, 380) -- xPos + core.writeInteger(newItem1Address + 8, 560) -- yPos + core.writeInteger(newItem1Address + 12, 80) -- width + core.writeInteger(newItem1Address + 16, 80) -- height + core.writeInteger(newItem1Address + 20, utils.createLuaFunctionWrapper(recordButtonCallback)) -- + core.writeInteger(newItem1Address + 24, 2) -- callbackParam + core.writeInteger(newItem1Address + 28, 0x0042AE90) -- renderFunc + core.writeInteger(newItem1Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on + core.writeInteger(newItem1Address + 36, 0x00000003) -- unknown + core.writeInteger(newItem1Address + 40, 0x00000000) -- unknown + core.writeInteger(newItem1Address + 44, 0x00000000) -- unknown + core.writeInteger(newItem1Address + 48, 0x0000FFF0) -- unknown + + core.writeInteger(newItem1Address + 72, 0x00000045) -- unknown +end + +local placePlaybackButton = function(recorder) + -- init button + + local playbackButtonCallback = function() + print("playback button") + if recorder.mode == "none" then + recorder:startPlayback() + core.writeInteger(newItem2Address + 32, 0x40000239) -- buttonGraphic + print("Starting new playback!") + elseif recorder.mode == "play" then + recorder:stopPlayback() + core.writeInteger(newItem2Address + 32, 0x4000023A) -- buttonGraphic + print("discarded playback") + end + end + + core.writeInteger(newItem2Address + 0, 0x00000003) -- type + core.writeInteger(newItem2Address + 4, 290) -- xPos + core.writeInteger(newItem2Address + 8, 560) -- yPos + core.writeInteger(newItem2Address + 12, 80) -- width + core.writeInteger(newItem2Address + 16, 80) -- height + core.writeInteger(newItem2Address + 20, utils.createLuaFunctionWrapper(playbackButtonCallback)) -- + core.writeInteger(newItem2Address + 24, 2) -- callbackParam + core.writeInteger(newItem2Address + 28, 0x0042AE90) -- renderFunc + core.writeInteger(newItem2Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on + core.writeInteger(newItem2Address + 36, 0x00000003) -- unknown + core.writeInteger(newItem2Address + 40, 0x00000000) -- unknown + core.writeInteger(newItem2Address + 44, 0x00000000) -- unknown + core.writeInteger(newItem2Address + 48, 0x0000FFF0) -- unknown + + core.writeInteger(newItem2Address + 72, 0x00000045) -- unknown +end + +return { + createButtons = function(recorder) + -- copy original ui array to new array + core.copyMemory(newMenuItemArrayAddress, originalMenuItemArrayAddress, originalMenuItemArraySize) + + -- copy 0x66 terminator one element further back + core.copyMemory(newTerminatorAddress, + newItem1Address, + menuItemSize) + + placeRecordButton(recorder) + placePlaybackButton(recorder) + + -- overwrite reference + core.writeCode(0x0059AB30, { core.AssemblyLambda([[ + push newMenuItemArrayAddress + ]], + { newMenuItemArrayAddress = newMenuItemArrayAddress, }) }) + + end, + + resetButtons = function() + core.writeInteger(newItem1Address + 32, 0x4000023A) -- reset recording button graphic + core.writeInteger(newItem2Address + 32, 0x4000023A) -- reset playback button graphic + end, +} diff --git a/code/utils.lua b/code/utils.lua index d89ddbe..871ce18 100644 --- a/code/utils.lua +++ b/code/utils.lua @@ -1,29 +1,29 @@ - -return { - - tableToHex = function(t) - local r = "" - for k, v in ipairs(t) do - local h = string.format("%X", v) - if h:len() == 1 then - h = "0" .. h - end - r = r .. h - end - return r - end, - - hexToTable = function(h) - local r = {} - if h:len() % 2 ~= 0 then - error("invalid hex string, not of even size") - end - - for i=1,h:len()-1,2 do --TODO: -1? - table.insert(r, tonumber(h:sub(i, i + 1), 16)) - end - - return r - end, - -} \ No newline at end of file + +return { + + tableToHex = function(t) + local r = "" + for k, v in ipairs(t) do + local h = string.format("%X", v) + if h:len() == 1 then + h = "0" .. h + end + r = r .. h + end + return r + end, + + hexToTable = function(h) + local r = {} + if h:len() % 2 ~= 0 then + error("invalid hex string, not of even size") + end + + for i=1,h:len()-1,2 do --TODO: -1? + table.insert(r, tonumber(h:sub(i, i + 1), 16)) + end + + return r + end, + +} diff --git a/code/validation.lua b/code/validation.lua new file mode 100644 index 0000000..5abbcab --- /dev/null +++ b/code/validation.lua @@ -0,0 +1,36 @@ +-- Validate file-controlled values before crossing the native command boundary. +local M = {MAX_PAYLOAD = 1260} + +function M.integer(value, minimum, maximum, label) + assert(type(value) == "number" and value == math.floor(value) + and value >= minimum and value <= maximum, "Invalid replay " .. label) + return value +end + +function M.command(command) + assert(type(command) == "table", "Invalid replay command") + M.integer(command.commandCategory, 0, 122, "command category") + M.integer(command.time, 1, 2147483647, "command tick") + M.integer(command.player, 1, 8, "player slot") + M.integer(command.size, 0, M.MAX_PAYLOAD, "payload size") + assert(type(command.data) == "string" and #command.data == command.size * 2 + and not command.data:find("[^%x]"), "Invalid replay payload encoding") + return command +end + +function M.info(info) + assert(type(info) == "table", "Missing replay match information") + M.integer(info.gameType, 0, 0, "game type") + for _, key in ipairs({"mapSeed", "matchSeed"}) do + M.integer(info[key], -2147483648, 2147483647, key) + end + for _, key in ipairs({"RNGvalue1", "RNGvalue2"}) do + M.integer(info[key], -32768, 65535, key) + end + for _, key in ipairs({"RNGindex1", "RNGindex2"}) do + M.integer(info[key], 0, 19999, key) + end + return info +end + +return M diff --git a/definition.yml b/definition.yml index a283b57..9878a9c 100644 --- a/definition.yml +++ b/definition.yml @@ -1,7 +1,7 @@ -name: recorder -display-name: UCP-Recorder -version: 0.0.1 -type: module -author: Corax -meta: - version: 0.0.1 \ No newline at end of file +name: recorder +display-name: UCP-Recorder +version: 0.1.0 +type: module +author: Corax +meta: + version: 1.0.0 diff --git a/init.lua b/init.lua index fe381e6..d49d82c 100644 --- a/init.lua +++ b/init.lua @@ -1,156 +1,162 @@ -local fixes = require("code/fixes") -local ui = require("code/ui") -local recorder = require("code/recorder") - -return { - enable = function(self, config) - - --- Init --- - - fixes.apply() - local gameRecorder = recorder:new({name = "test_recording1", rngLogMethod = config.rngLogMethod}) - ui.createButtons(gameRecorder) - - --- Injections --- - - -- skirmish startup injects - core.detourCode(function(registers) - gameRecorder:onStartSkirmish(registers) - ui.resetButtons() - end, 0x004428b5, 6) - - core.writeCode(0x00442877, {0x90, 0x90, 0x90, 0x90, 0x90, -- make space for detourCode - 0x90, 0x90, 0x90, 0x90, 0x90}) - core.detourCode(function(registers) gameRecorder:onBeforeSetMatchSeed(registers) end, 0x00442877, 10) - - core.detourCode(function(registers) - gameRecorder.onCustomSkirmishGame(registers) - end, 0x0042bf4c, 6) - - -- after LaunchGame() - core.detourCode(function(registers) - - end, 0x004428c6, 10) - - -- skirmish end injects - - -- on switchToMenu - core.detourCode(function(registers) - local DAT_CurrentMenuID = core.readInteger(0x01fe7d1c) - local newMenuID = registers.EBP - - if newMenuID == 61 then - print("ingame menu change - exited match?") - print("Old menu: " .. DAT_CurrentMenuID) - print("New menu: " .. newMenuID) - print("onMenuChange") - if gameRecorder.mode == "record" then - print("stopping recording") - gameRecorder:stopRecording() - elseif gameRecorder.mode == "play" then - print("stopping playback") - gameRecorder:stopPlayback() - end - end - - end, 0x0046b358, 6) - - -- on loadGame - core.detourCode(function(registers) - print("onLoadGame") - if gameRecorder.mode == "record" then - print("stopping recording") - gameRecorder:stopRecording() - elseif gameRecorder.mode == "play" then - print("stopping playback") - gameRecorder:stopPlayback() - end - - end, 0x00495337, 6) - - -- on restartGame - core.detourCode(function(registers) - print("onRestartGame") - if gameRecorder.mode == "record" then - print("stopping recording") - gameRecorder:stopRecording() - elseif gameRecorder.mode == "play" then - print("stopping playback") - gameRecorder:stopPlayback() - end - - end, 0x00494ba5, 5) - - -- consuming recorded commands function - local scheduleNextCommandAddress = core.allocateCode({ - 0x90, 0x90, 0x90, 0x90, 0x90, - 0xC3, - }) - - core.detourCode(function(registers) gameRecorder:scheduleNextCommand(registers) end, scheduleNextCommandAddress, 5) - - gameRecorder._scheduleCommand = core.exposeCode(0x00480210, 5, 1) - - -- schedule saved commands when in PLAYBACK mode - -- gets called every tick - core.insertCode(0x00490690, 8, - gameRecorder:onReceiveAllTransmittedCommandsASM(scheduleNextCommandAddress), - nil, "after") - - -- process sent commands - core.detourCode(function(registers) gameRecorder:onTransmitCommand(registers) end, 0x00487c50, 6) - - -- process received commands - core.detourCode(function(registers) gameRecorder:onScheduleCommand(registers) end, 0x00480353, 6) - - --- fakes multiplayerIDs in singleplayer skirmish - -- nop and detour to prevent original code execution - core.writeCode(0x0047eaf0, { - 0x90, 0x90, 0x90, 0x90, 0x90, - 0x90, - }) - core.detourCode(function(registers) gameRecorder:fakeMultiplayerIdentities(registers) end, 0x0047eaf0, 6) - - -- Do not set DAT_QueuedCommandPlayer to -1 in Singleplayer skirmish - core.writeCode(0x004876a6, {0x90,0x90,0x90,0x90,0x90,0x90,}) - - --- Debugging and Logging --- - - local syncCheckAddress = core.allocateCode({ - 0x90, 0x90, 0x90, 0x90, 0x90, - 0xC3, - }) - - core.detourCode(function(registers) gameRecorder:syncCheck(registers) end, syncCheckAddress, 5) - - if config.rngLogMethod == "trace" then - local traceRNG1 = false -- TODO remove - local traceRNG2 = true -- remove - - local rngFunction1 = 0x0046a800 - local rngFunction2 = 0x0046a7d0 - - if traceRNG1 then core.detourCode(function(registers) gameRecorder:syncCheck(registers, 1) end, rngFunction1, 6) end - - if traceRNG2 then core.detourCode(function(registers) gameRecorder:syncCheck(registers, 2) end, rngFunction2, 6) end - - end - - -- optional - -- Makes every game's RNG use the same seed - if config.useFixedSeed then - - core.detourCode(function(registers) - - registers.EAX = seed - - return registers - end, 0x0046a74a, 6) - - end - - end, - - disable = function(self, config) - end, -} +local fixes = require("code/fixes") +local ui = require("code/ui") +local recorder = require("code/recorder") + +return { + enable = function(self, config) + + --- Init --- + + require("code/native").verify() + fixes.apply() + local gameRecorder = recorder:new({name = "test_recording1", rngLogMethod = config.rngLogMethod}) + ui.createButtons(gameRecorder) + + --- Injections --- + + -- skirmish startup injects + core.detourCode(function(registers) + gameRecorder:onStartSkirmish(registers) + ui.resetButtons() + return registers + end, 0x004428b5, 6) + + core.writeCode(0x00442877, {0x90, 0x90, 0x90, 0x90, 0x90, -- make space for detourCode + 0x90, 0x90, 0x90, 0x90, 0x90}) + core.detourCode(function(registers) return gameRecorder:onBeforeSetMatchSeed(registers) end, 0x00442877, 10) + + core.detourCode(function(registers) + return gameRecorder:onCustomSkirmishGame(registers) + end, 0x0042bf4c, 6) + + -- after LaunchGame() + core.detourCode(function(registers) + + return registers + end, 0x004428c6, 10) + + -- skirmish end injects + + -- on switchToMenu + core.detourCode(function(registers) + local DAT_CurrentMenuID = core.readInteger(0x01fe7d1c) + local newMenuID = registers.EBP + + if newMenuID == 61 then + print("ingame menu change - exited match?") + print("Old menu: " .. DAT_CurrentMenuID) + print("New menu: " .. newMenuID) + print("onMenuChange") + if gameRecorder.mode == "record" then + print("stopping recording") + gameRecorder:stopRecording() + elseif gameRecorder.mode == "play" then + print("stopping playback") + gameRecorder:stopPlayback() + end + end + + return registers + end, 0x0046b358, 6) + + -- on loadGame + core.detourCode(function(registers) + print("onLoadGame") + if gameRecorder.mode == "record" then + print("stopping recording") + gameRecorder:stopRecording() + elseif gameRecorder.mode == "play" then + print("stopping playback") + gameRecorder:stopPlayback() + end + + return registers + end, 0x00495337, 6) + + -- on restartGame + core.detourCode(function(registers) + print("onRestartGame") + if gameRecorder.mode == "record" then + print("stopping recording") + gameRecorder:stopRecording() + elseif gameRecorder.mode == "play" then + print("stopping playback") + gameRecorder:stopPlayback() + end + + return registers + end, 0x00494ba5, 5) + + -- consuming recorded commands function + local scheduleNextCommandAddress = core.allocateCode({ + 0x90, 0x90, 0x90, 0x90, 0x90, + 0xC3, + }) + + core.detourCode(function(registers) return gameRecorder:scheduleNextCommand(registers) end, scheduleNextCommandAddress, 5) + + gameRecorder._scheduleCommand = core.exposeCode(0x00480210, 5, 1) + + -- schedule saved commands when in PLAYBACK mode + -- gets called every tick + core.insertCode(0x00490690, 8, + gameRecorder:onReceiveAllTransmittedCommandsASM(scheduleNextCommandAddress), + nil, "after") + + -- process sent commands + core.detourCode(function(registers) return gameRecorder:onTransmitCommand(registers) end, 0x00487c50, 6) + + -- process received commands + core.detourCode(function(registers) return gameRecorder:onScheduleCommand(registers) end, 0x00480353, 6) + + --- fakes multiplayerIDs in singleplayer skirmish + -- nop and detour to prevent original code execution + core.writeCode(0x0047eaf0, { + 0x90, 0x90, 0x90, 0x90, 0x90, + 0x90, + }) + core.detourCode(function(registers) return gameRecorder:fakeMultiplayerIdentities(registers) end, 0x0047eaf0, 6) + + -- Do not set DAT_QueuedCommandPlayer to -1 in Singleplayer skirmish + core.writeCode(0x004876a6, {0x90,0x90,0x90,0x90,0x90,0x90,}) + + --- Debugging and Logging --- + + local syncCheckAddress = core.allocateCode({ + 0x90, 0x90, 0x90, 0x90, 0x90, + 0xC3, + }) + + core.detourCode(function(registers) return gameRecorder:syncCheck(registers) end, syncCheckAddress, 5) + + if config.rngLogMethod == "trace" then + local traceRNG1 = true + local traceRNG2 = true -- remove + + local rngFunction1 = 0x0046a800 + local rngFunction2 = 0x0046a7d0 + + if traceRNG1 then core.detourCode(function(registers) return gameRecorder:syncCheck(registers, 1) end, rngFunction1, 6) end + + if traceRNG2 then core.detourCode(function(registers) return gameRecorder:syncCheck(registers, 2) end, rngFunction2, 6) end + + end + + -- optional + -- Makes every game's RNG use the same seed + if config.useFixedSeed then + + core.detourCode(function(registers) + + registers.EAX = require("code/validation").integer(config.fixedSeed, -2147483648, 2147483647, "fixed seed") + + return registers + end, 0x0046a74a, 6) + + end + + end, + + disable = function(self, config) + end, +} diff --git a/options.yml b/options.yml index c3a55b5..77fee6c 100644 --- a/options.yml +++ b/options.yml @@ -1,27 +1,27 @@ -meta: - version: 0.0.1 -options: -- url: recorder.rngLogMethod - header: "Type of logging method" - text: "Type of logging method for recorded gameplay" - hasHeader: true - display: - contents: - type: string - value: "trace" -- url: recorder.useFixedSeed - header: "Use fixed seed" - text: "Using fixed seeds guarantees deterministic gameplay" - hasHeader: true - display: - contents: - type: boolean - value: true -- url: fixedSeed - header: "Fixed Seed" - text: "A seed can be entered in case fixed seeds are used" - hasHeader: true - display: - contents: - type: integer - value: 1 \ No newline at end of file +meta: + version: 1.0.0 +options: +- url: recorder.rngLogMethod + header: "Type of logging method" + text: "Type of logging method for recorded gameplay" + hasHeader: true + display: + contents: + type: string + value: "trace" +- url: recorder.useFixedSeed + header: "Use fixed seed" + text: "Using fixed seeds guarantees deterministic gameplay" + hasHeader: true + display: + contents: + type: boolean + value: true +- url: fixedSeed + header: "Fixed Seed" + text: "A seed can be entered in case fixed seeds are used" + hasHeader: true + display: + contents: + type: integer + value: 1 diff --git a/tests/test_recorder.py b/tests/test_recorder.py new file mode 100644 index 0000000..70bcb7e --- /dev/null +++ b/tests/test_recorder.py @@ -0,0 +1,148 @@ +"""Portable LuaJIT regressions; no game process or native patching required. + +Run: python -m pip install lupa && python -m unittest discover -s tests -v +""" +from pathlib import Path +import unittest +from lupa.luajit21 import LuaRuntime + +ROOT = Path(__file__).resolve().parents[1] + + +class RecorderTests(unittest.TestCase): + def setUp(self): + self.lua = LuaRuntime(unpack_returned_tuples=True) + self.lua.globals().source_root = ROOT.as_posix() + self.lua.execute(r''' +package.path = source_root .. '/?.lua;' .. package.path +memory, bytes, callbacks, files, handles, scheduled = {}, {}, {}, {}, {}, 0 +local nextAddress = 0x10000000 +core = { + allocate = function(size) local a=nextAddress; nextAddress=a+size+16; return a end, + readInteger = function(a) return memory[a] or 0 end, + writeInteger = function(a,v) memory[a]=v end, + readSmallInteger = function(a) return memory[a] or 0 end, + writeSmallInteger = function(a,v) memory[a]=v end, + readBytes = function(a,n) local t={}; for i=1,n do t[i]=bytes[a+i-1] or 0 end; return t end, + writeBytes = function(a,t) for i,v in ipairs(t) do bytes[a+i-1]=v end end, + writeCode = function() end, copyMemory = function() end, + AssemblyLambda = function(s,vars) return {assembly=s, variables=vars} end, + detourCode = function(f,a) callbacks[a]=f end, insertCode = function() end, + allocateCode = function() nextAddress=nextAddress+128; return nextAddress end, + exposeCode = function() return function() scheduled=scheduled+1 end end, +} +utils = {createLuaFunctionWrapper=function() return 0 end} +json = {encode=function(_,value) return value end, decode=function(_,value) return value end} +io.open = function(path,mode) + if path==failPath then return nil, 'injected failure' end + if mode=='r' and not files[path] then return nil, 'missing' end + if mode=='w' then files[path]={} end + local cursor=0 + local handle={closed=false} + function handle:read() cursor=cursor+1; return files[path][cursor] end + function handle:write(line) table.insert(files[path],line); return self end + function handle:flush() return true end + function handle:close() assert(not self.closed); self.closed=true; return true end + function handle:seek(_,offset) cursor=offset; return offset end + handles[#handles+1]=handle + return handle +end +os.remove=function(path) files[path]=nil; return true end +Recorder = require('code/recorder') +function fixture(name) + local r=Recorder:new({name=name,rngLogMethod='trace'}) + files[r.commandsFileName]={} + files[r.rngFileName]={} + files[r.infoFileName]={{gameType=0,mapSeed=1,matchSeed=1,RNGvalue1=1,RNGvalue2=1,RNGindex1=1,RNGindex2=1}} + return r +end +function command(tick) return {time=tick or 1,commandCategory=28,player=1,size=1,data='01'} end +function allClosed() for _,h in ipairs(handles) do assert(h.closed) end end +''') + + def check(self, script): + self.lua.execute(script) + + def test_reset_drops_prefetch_and_is_idempotent(self): + self.check(''' +local r=fixture('cache'); files[r.commandsFileName]={command(685)} +r:startPlayback(); assert(r:peekCommand().time==685); r.cachedRNG={time=999} +r:stopPlayback(); r:reset(); files[r.commandsFileName]={command(114)} +r:startPlayback(); assert(r:consumeSavedCommand().time==114 and not r.cachedRNG) +r:stopPlayback(); allClosed() +''') + + def test_missing_and_partial_files_do_not_commit_state(self): + self.check(''' +local r=Recorder:new({name='missing'}); memory[0x191de0c]=777 +assert(not pcall(function() r:startPlayback() end)) +assert(r.mode=='none' and memory[0x191de0c]==777); r:reset() +failPath=r.rngFileName +assert(not pcall(function() r:startRecording() end)) +assert(r.mode=='none' and not files[r.commandsFileName]); allClosed() +''') + + def test_preserves_local_player(self): + self.check(''' +local r=fixture('player'); memory[0x191de0c]=777; memory[0x01a275dc]=3 +r:startPlayback(); r:stopPlayback() +assert(memory[0x191de0c]==777 and memory[0x01a275dc]==3) +''') + + def test_instances_own_buffers(self): + self.check('local a=fixture("a"); local address=a.commandDataAddress; local b=fixture("b"); assert(a.commandDataAddress==address and address~=b.commandDataAddress)') + + def test_late_malformed_command_is_rejected_before_start(self): + self.check(''' +local r=fixture('late'); local bad=command(100); bad.data='XX' +files[r.commandsFileName]={command(1),bad} +assert(not pcall(function() r:startPlayback() end)); assert(r.mode=='none'); allClosed() +''') + + def test_invalid_packets_never_reach_native_boundary(self): + self.check(''' +local r=fixture('payload'); r._scheduleCommand=function() scheduled=scheduled+1 end +for _,change in ipairs({{size=1261,data=string.rep('AA',1261)}, {size=1,data='0102'}, + {data='GG'}, {commandCategory=999}, {player=0}, {time=0}, {time=1.5}}) do + local c=command(); for k,v in pairs(change) do c[k]=v end + assert(not pcall(function() r:scheduleCommand(c) end)) +end +assert(scheduled==0) +r:scheduleCommand(command()); assert(scheduled==1) +assert(bytes[r.commandDataAddress]==1 and bytes[r.commandDataAddress+1259]==0) +''') + + def test_rng_value_index_and_tick_mismatches_are_retained(self): + for key in ('rng1', 'rng2', 'index1', 'index2', 'time'): + with self.subTest(key=key): + self.lua.globals().different_key = key + self.check(''' +local r=fixture('rng'); r.mode='play'; memory[r.rngRecorderState]=2 +memory[0x2000]=0x420f76 +r.cachedRNG={time=0,rng1=0,rng2=0,index1=0,index2=0,extra={ra2=0x420f76}} +r.cachedRNG[different_key]=100 +r:syncCheck({ESP=0x2000},2) +assert(r.firstDesync and r.firstDesync.reason==different_key) +assert(memory[r.commandRecorderState]==0 and memory[r.rngRecorderState]==0) +''') + + def test_real_init_callbacks_return_register_changes(self): + self.check(''' +package.loaded['code/native']={verify=function() end} +local module=dofile(source_root..'/init.lua') +module:enable({rngLogMethod='trace',useFixedSeed=true,fixedSeed=123}) +assert(callbacks[0x46a74a]({EAX=456}).EAX==123) +memory[0x191d768+0x618]=99 +assert(callbacks[0x47eaf0]({ECX=0x191d768,EDX=685}).EDX==99) +''') + + def test_native_verification_fails_before_installing_hooks(self): + self.check(''' +local module=dofile(source_root..'/init.lua') +assert(not pcall(function() module:enable({}) end)) +assert(next(callbacks)==nil) +''') + + +if __name__ == '__main__': + unittest.main() From 13b75dc0b175658df7bbcbc8d3fe20277f489d41 Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:40:15 +0200 Subject: [PATCH 02/19] Port recorder native hooks to Extreme with verified address profiles --- CHANGELOG.md | 8 ++ code/fixes.lua | 19 ++-- code/native.lua | 193 ++++++++++++++++++++++++++++++------- code/recorder.lua | 63 ++++++------ code/ui.lua | 9 +- definition.yml | 2 +- docs/native-port.md | 43 +++++++++ init.lua | 39 ++++---- tests/check_executables.py | 52 ++++++++++ tests/test_recorder.py | 4 +- tools/build.py | 21 ++++ 11 files changed, 355 insertions(+), 98 deletions(-) create mode 100644 docs/native-port.md create mode 100644 tests/check_executables.py create mode 100644 tools/build.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e6e0745..c40387a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.2.0 + +- Port the existing native hooks and Skirmish controls to the checked Extreme executable. +- Resolve code and data addresses through separate Crusader/Extreme profiles, including Extreme's larger player-state structure. +- Add an optional original-executable verification tool and document the native layout differences. + +Validation at this stage: the regression suite passes, both original executables pass address checks, Crusader starts a Skirmish and records player commands, and Extreme reaches its main menu with the module enabled. Complete replay verification and the new session/browser workflow are still under development. Existing recordings must not be shared between variants. + ## 0.1.0 - Fix the player-identity detour losing its returned registers and apply the configured fixed seed. diff --git a/code/fixes.lua b/code/fixes.lua index 464cf8a..9b4cc4d 100644 --- a/code/fixes.lua +++ b/code/fixes.lua @@ -1,3 +1,4 @@ +local native = require("code/native") --[[ Fixes which might be needed for stable recordings Test this @@ -8,26 +9,26 @@ return { --- faulty randomness line? This one is called extra when loading from the load game menu -- Dust spawning normally returns with RET 0x2c (11 arguments). -- Replace the whole CALL and perform exactly its 44-byte cleanup. - core.writeCode(0x004fc4a3, {0x90, 0x90, 0x90, 0x90, 0x90}) - core.writeCode(0x004fc627, {0x83, 0xC4, 0x2C, 0x90, 0x90}) + core.writeCode(native.addr(0x004fc4a3), {0x90, 0x90, 0x90, 0x90, 0x90}) + core.writeCode(native.addr(0x004fc627), {0x83, 0xC4, 0x2C, 0x90, 0x90}) --- Mother RNG fix, nopping out. Could also switch with RNG1 to keep the baby cry noise - core.writeCode(0x005474a3, {0x90, 0x90, 0x90, 0x90, 0x90}) + core.writeCode(native.addr(0x005474a3), {0x90, 0x90, 0x90, 0x90, 0x90}) -- removing all function calls to RNG1 that come from the music thread. - core.writeCode(0x0047a8d5, {0x90, 0x90, 0x90, 0x90, 0x90}) - core.writeCode(0x0047a86b, {0x90, 0x90, 0x90, 0x90, 0x90}) + core.writeCode(native.addr(0x0047a8d5), {0x90, 0x90, 0x90, 0x90, 0x90}) + core.writeCode(native.addr(0x0047a86b), {0x90, 0x90, 0x90, 0x90, 0x90}) - core.writeCode(0x0047c348, {0x90, 0x90, 0x90, 0x90, 0x90}) + core.writeCode(native.addr(0x0047c348), {0x90, 0x90, 0x90, 0x90, 0x90}) --- Right click menu fixes (during pause): -- Do not process tick when game is paused and pull down terrain or flatten terrain is used - core.writeCode(0x0045ceff, {0x90, 0x90}) + core.writeCode(native.addr(0x0045ceff), {0x90, 0x90}) -- Do not increase match time when game is paused and flatten terrain is used - core.writeCode(0x0045ce34, {0xEB, 0x46}) + core.writeCode(native.addr(0x0045ce34), {0xEB, 0x46}) -- Removing mothers and children (TODO test without this) - core.writeCode(0x004582ed, {0xEB}) + core.writeCode(native.addr(0x004582ed), {0xEB}) end } diff --git a/code/native.lua b/code/native.lua index 7a24185..e735569 100644 --- a/code/native.lua +++ b/code/native.lua @@ -1,36 +1,163 @@ --- Original SHC 1.41 patch bytes. Verify the complete set before writing. -local sites = { - {0x004428B5, {0x89, 0x3D, 0xA4, 0x9C, 0xFE, 0x01}}, - {0x00442877, {0xB9, 0xC0, 0x79, 0xA2, 0x01, 0xE8, 0xBF, 0x7E, 0x02, 0x00}}, - {0x0042BF4C, {0x89, 0x1D, 0xA4, 0x9C, 0xFE, 0x01}}, - {0x004428C6, {0xC7, 0x05, 0x94, 0xA0, 0xFE, 0x01, 0x01, 0x00, 0x00, 0x00}}, - {0x0046B358, {0x89, 0x6E, 0x18, 0x89, 0x46, 0x04}}, - {0x00495337, {0x8B, 0x0D, 0x24, 0x66, 0x12, 0x01}}, - {0x00494BA5, {0xB9, 0xB0, 0xC9, 0xF2, 0x00}}, - {0x00490690, {0x83, 0xEC, 0x68, 0xA1, 0x20, 0x42, 0xB9, 0x00}}, - {0x00487C50, {0x81, 0xEC, 0xF4, 0x03, 0x00, 0x00}}, - {0x00480353, {0x8B, 0x8E, 0x24, 0xD8, 0x02, 0x00}}, - {0x0047EAF0, {0x8B, 0x91, 0x18, 0x06, 0x00, 0x00}}, - {0x004876A6, {0x89, 0xB7, 0xA4, 0x06, 0x00, 0x00}}, - {0x0046A800, {0x8B, 0x81, 0x4C, 0x9C, 0x00, 0x00}}, - {0x0046A7D0, {0x8B, 0x81, 0x48, 0x9C, 0x00, 0x00}}, - {0x0046A74A, {0x83, 0xC4, 0x04, 0x89, 0x46, 0x04}}, - {0x0059AB30, {0x68, 0x48, 0x98, 0x5E, 0x00}}, - {0x004FC4A3, {0xE8, 0x28, 0xE3, 0xF6, 0xFF}}, - {0x004FC627, {0xE8, 0xB4, 0x84, 0xF0, 0xFF}}, - {0x005474A3, {0xE8, 0x28, 0x33, 0xF2, 0xFF}}, - {0x0047A8D5, {0xE8, 0x26, 0xFF, 0xFE, 0xFF}}, - {0x0047A86B, {0xE8, 0x90, 0xFF, 0xFE, 0xFF}}, - {0x0047C348, {0xE8, 0xB3, 0xE4, 0xFE, 0xFF}}, - {0x0045CEFF, {0x75, 0x2D}}, - {0x0045CE34, {0x7D, 0x46}}, - {0x004582ED, {0x75}}, +-- Verified SHC 1.41 / SHC Extreme 1.41 addresses; see docs/native-port.md. +local profiles = { + {name = "SHC", sha256 = "3bb0a8c1e72331b3a30a5aa93ed94beca0081b476b04c1960e26d5b45387ac5a", addresses = { + [0x0042AE90] = 0x0042AE90, + [0x0042BF4C] = 0x0042BF4C, + [0x00442877] = 0x00442877, + [0x004428B5] = 0x004428B5, + [0x004428C6] = 0x004428C6, + [0x004582ED] = 0x004582ED, + [0x0045CE34] = 0x0045CE34, + [0x0045CEFF] = 0x0045CEFF, + [0x0046A740] = 0x0046A740, + [0x0046A74A] = 0x0046A74A, + [0x0046A760] = 0x0046A760, + [0x0046A7D0] = 0x0046A7D0, + [0x0046A800] = 0x0046A800, + [0x0046B358] = 0x0046B358, + [0x0047A86B] = 0x0047A86B, + [0x0047A8D5] = 0x0047A8D5, + [0x0047C348] = 0x0047C348, + [0x0047EAF0] = 0x0047EAF0, + [0x00480210] = 0x00480210, + [0x00480353] = 0x00480353, + [0x004876A6] = 0x004876A6, + [0x00487C50] = 0x00487C50, + [0x00490690] = 0x00490690, + [0x00494BA5] = 0x00494BA5, + [0x00495337] = 0x00495337, + [0x004FC4A3] = 0x004FC4A3, + [0x004FC627] = 0x004FC627, + [0x005474A3] = 0x005474A3, + [0x0059AB30] = 0x0059AB30, + [0x005E9848] = 0x005E9848, + [0x0191D768] = 0x0191D768, + [0x0191DE0C] = 0x0191DE0C, + [0x0194AF98] = 0x0194AF98, + [0x01A275DC] = 0x01A275DC, + [0x01A279C0] = 0x01A279C0, + [0x01A279C2] = 0x01A279C2, + [0x01A279C4] = 0x01A279C4, + [0x01A31608] = 0x01A31608, + [0x01A3160C] = 0x01A3160C, + [0x01FE7D1C] = 0x01FE7D1C, + [0x01FE7DA8] = 0x01FE7DA8, + }, sites = { + {0x004428B5, {137,61,164,156,254,1}}, + {0x00442877, {185,192,121,162,1,232,191,126,2,0}}, + {0x0042BF4C, {137,29,164,156,254,1}}, + {0x004428C6, {199,5,148,160,254,1,1,0,0,0}}, + {0x0046B358, {137,110,24,137,70,4}}, + {0x00495337, {139,13,36,102,18,1}}, + {0x00494BA5, {185,176,201,242,0}}, + {0x00490690, {131,236,104,161,32,66,185,0}}, + {0x00487C50, {129,236,244,3,0,0}}, + {0x00480353, {139,142,36,216,2,0}}, + {0x0047EAF0, {139,145,24,6,0,0}}, + {0x004876A6, {137,183,164,6,0,0}}, + {0x0046A800, {139,129,76,156,0,0}}, + {0x0046A7D0, {139,129,72,156,0,0}}, + {0x0046A74A, {131,196,4,137,70,4}}, + {0x0059AB30, {104,72,152,94,0}}, + {0x004FC4A3, {232,40,227,246,255}}, + {0x004FC627, {232,180,132,240,255}}, + {0x005474A3, {232,40,51,242,255}}, + {0x0047A8D5, {232,38,255,254,255}}, + {0x0047A86B, {232,144,255,254,255}}, + {0x0047C348, {232,179,228,254,255}}, + {0x0045CEFF, {117,45}}, + {0x0045CE34, {125,70}}, + {0x004582ED, {117}}, + }}, + {name = "Extreme", sha256 = "55648e6b05d67d37a5773fe699bbb17a2d6ad4de1bb9dbded9a21caef82bd7fb", addresses = { + [0x0042AE90] = 0x0042AEC0, + [0x0042BF4C] = 0x0042BF7C, + [0x00442877] = 0x00442A37, + [0x004428B5] = 0x00442A75, + [0x004428C6] = 0x00442A86, + [0x004582ED] = 0x0045851D, + [0x0045CE34] = 0x0045D044, + [0x0045CEFF] = 0x0045D10F, + [0x0046A740] = 0x0046A960, + [0x0046A74A] = 0x0046A96A, + [0x0046A760] = 0x0046A980, + [0x0046A7D0] = 0x0046A9F0, + [0x0046A800] = 0x0046AA20, + [0x0046B358] = 0x0046B578, + [0x0047A86B] = 0x0047AA3B, + [0x0047A8D5] = 0x0047AAA5, + [0x0047C348] = 0x0047C518, + [0x0047EAF0] = 0x0047ECC0, + [0x00480210] = 0x004803E0, + [0x00480353] = 0x00480523, + [0x004876A6] = 0x004877B6, + [0x00487C50] = 0x00487D60, + [0x00490690] = 0x004907F0, + [0x00494BA5] = 0x00494D05, + [0x00495337] = 0x00495497, + [0x004FC4A3] = 0x004FC823, + [0x004FC627] = 0x004FC9A7, + [0x005474A3] = 0x005478C3, + [0x0059AB30] = 0x0059AF60, + [0x005E9848] = 0x005E9708, + [0x0191D768] = 0x023547D8, + [0x0191DE0C] = 0x02354E7C, + [0x0194AF98] = 0x02382008, + [0x01A275DC] = 0x024BAADC, + [0x01A279C0] = 0x024BAEC0, + [0x01A279C2] = 0x024BAEC2, + [0x01A279C4] = 0x024BAEC4, + [0x01A31608] = 0x024C4B08, + [0x01A3160C] = 0x024C4B0C, + [0x01FE7D1C] = 0x02A7B21C, + [0x01FE7DA8] = 0x02A7B2A8, + }, sites = { + {0x00442A75, {137,61,164,209,167,2}}, + {0x00442A37, {185,192,174,75,2,232,31,127,2,0}}, + {0x0042BF7C, {137,29,164,209,167,2}}, + {0x00442A86, {199,5,148,213,167,2,1,0,0,0}}, + {0x0046B578, {137,110,24,137,70,4}}, + {0x00495497, {139,13,164,106,18,1}}, + {0x00494D05, {185,48,206,242,0}}, + {0x004907F0, {131,236,104,161,176,67,185,0}}, + {0x00487D60, {129,236,244,3,0,0}}, + {0x00480523, {139,142,36,216,2,0}}, + {0x0047ECC0, {139,145,24,6,0,0}}, + {0x004877B6, {137,183,164,6,0,0}}, + {0x0046AA20, {139,129,76,156,0,0}}, + {0x0046A9F0, {139,129,72,156,0,0}}, + {0x0046A96A, {131,196,4,137,70,4}}, + {0x0059AF60, {104,8,151,94,0}}, + {0x004FC823, {232,200,225,246,255}}, + {0x004FC9A7, {232,68,129,240,255}}, + {0x005478C3, {232,40,49,242,255}}, + {0x0047AAA5, {232,118,255,254,255}}, + {0x0047AA3B, {232,224,255,254,255}}, + {0x0047C518, {232,3,229,254,255}}, + {0x0045D10F, {117,45}}, + {0x0045D044, {125,70}}, + {0x0045851D, {117}}, + }}, } -return {verify = function() - for _, site in ipairs(sites) do - local actual = core.readBytes(site[1], #site[2]) - for i, expected in ipairs(site[2]) do - assert(actual[i] == expected, string.format("Recorder: unsupported executable or conflicting patch at %08X; no recorder hooks installed", site[1])) +local M = {} +function M.verify() + for _, profile in ipairs(profiles) do + local matches = true + for _, site in ipairs(profile.sites) do + local bytes = core.readBytes(site[1], #site[2]) + for i, expected in ipairs(site[2]) do + if bytes[i] ~= expected then matches = false; break end + end + if not matches then break end end + if matches then M.profile = profile; return profile end end -end} + error("Recorder: unsupported executable or conflicting native patch; no recorder hooks installed") +end +function M.addr(classic) + local profile = M.profile or M.verify() + local address = profile.addresses[classic] + assert(address, string.format("Unmapped recorder address %08X", classic)) + return address +end +return M diff --git a/code/recorder.lua b/code/recorder.lua index 4e8b14c..43420d8 100644 --- a/code/recorder.lua +++ b/code/recorder.lua @@ -1,3 +1,4 @@ +local native = require("code/native") --[[ Replay implementation --]] @@ -180,7 +181,7 @@ function Recorder:loadInfo() end function Recorder:ScheduleCommandWrapper(commandCategory, player, time, address) - self._scheduleCommand(0x191d768, commandCategory, player, time, address) + self._scheduleCommand(native.addr(0x191d768), commandCategory, player, time, address) end function Recorder:scheduleCommand(command) @@ -227,15 +228,15 @@ function Recorder:consumeSavedCommand() return c end --- 0x004428b5 SHC +-- native.addr(0x004428b5) SHC function Recorder:onStartSkirmish(registers) -- SINGLEPLAYER ONLY (TODO make this work for multiplayer too) if core.readInteger(self.commandRecorderState) == self.RECORDER_STATES.RECORD then - local gameRNG1index = core.readInteger(0x01a3160c) - local gameRNG2index = core.readInteger(0x01a31608) - local gameRNG1 = core.readSmallInteger(0x01a279c0) - local gameRNG2 = core.readSmallInteger(0x01a279c2) + local gameRNG1index = core.readInteger(native.addr(0x01a3160c)) + local gameRNG2index = core.readInteger(native.addr(0x01a31608)) + local gameRNG1 = core.readSmallInteger(native.addr(0x01a279c0)) + local gameRNG2 = core.readSmallInteger(native.addr(0x01a279c2)) local mapSeed = self.mapSeed - local matchSeed = core.readInteger(0x01a279c4) + local matchSeed = core.readInteger(native.addr(0x01a279c4)) self:saveInfo(0, mapSeed, matchSeed, gameRNG1, gameRNG2, gameRNG1index, gameRNG2index) @@ -244,24 +245,24 @@ function Recorder:onStartSkirmish(registers) -- SINGLEPLAYER ONLY (TODO make thi elseif core.readInteger(self.commandRecorderState) == self.RECORDER_STATES.PLAYBACK then local skirmishInfo = self:loadInfo() - local populateRNG1040 = core.exposeCode(0x0046a760, 1, 1) + local populateRNG1040 = core.exposeCode(native.addr(0x0046a760), 1, 1) -- Set SEC_CurrentPlayerSlotID to 0 (Disallows actions during playback) TODO move this print("recorder in Playback state") - --core.writeInteger(0x01a275dc, 0) + --core.writeInteger(native.addr(0x01a275dc), 0) -- Load mapSeed and fill RNG table - core.writeInteger(0x01a279c4, skirmishInfo.mapSeed) - populateRNG1040(0x01a279c0); + core.writeInteger(native.addr(0x01a279c4), skirmishInfo.mapSeed) + populateRNG1040(native.addr(0x01a279c0)); -- Load matchSeed (populateRNG1040() is called later in LaunchGame() after the map is setup) - core.writeInteger(0x01a279c4, skirmishInfo.matchSeed) + core.writeInteger(native.addr(0x01a279c4), skirmishInfo.matchSeed) -- Load starting RNG values - core.writeInteger(0x01a3160c, skirmishInfo.RNGindex1) - core.writeInteger(0x01a31608, skirmishInfo.RNGindex2) - core.writeSmallInteger(0x01a279c0, skirmishInfo.RNGvalue1) - core.writeSmallInteger(0x01a279c2, skirmishInfo.RNGvalue2) + core.writeInteger(native.addr(0x01a3160c), skirmishInfo.RNGindex1) + core.writeInteger(native.addr(0x01a31608), skirmishInfo.RNGindex2) + core.writeSmallInteger(native.addr(0x01a279c0), skirmishInfo.RNGvalue1) + core.writeSmallInteger(native.addr(0x01a279c2), skirmishInfo.RNGvalue2) print("Loaded skirmish information:") print(string.format("Gametype=%d, mapSeed=%d, matchSeed=%d, gameRNG1=%d, gameRNG2=%d, gameRNG1index=%d, gameRNG2index=%d", skirmishInfo.gameType, skirmishInfo.mapSeed, skirmishInfo.matchSeed, skirmishInfo.RNGvalue1, skirmishInfo.RNGvalue2, skirmishInfo.RNGindex1, skirmishInfo.RNGindex2)) @@ -270,20 +271,20 @@ function Recorder:onStartSkirmish(registers) -- SINGLEPLAYER ONLY (TODO make thi return registers end --- 0x00442877 SHC +-- native.addr(0x00442877) SHC function Recorder:onBeforeSetMatchSeed(registers) -- SINGLEPLAYER ONLY (TODO make this work for multiplayer too) - self.mapSeed = core.readInteger(0x01a279c4) - local setTimeBasedSeed = core.exposeCode(0x0046a740, 1, 1) + self.mapSeed = core.readInteger(native.addr(0x01a279c4)) + local setTimeBasedSeed = core.exposeCode(native.addr(0x0046a740), 1, 1) -- original code - setTimeBasedSeed(0x01a279c0) + setTimeBasedSeed(native.addr(0x01a279c0)) return registers end --- 0x0042bf4c SHC +-- native.addr(0x0042bf4c) SHC function Recorder:onCustomSkirmishGame(registers) -- SINGLEPLAYER ONLY (TODO test if playerIDs work fine in multiplayer) -- Make singleplayer skirmishes use real playerID for commands, not -1 - local DAT_QueuedCommandPlayer = 0x191de0c + local DAT_QueuedCommandPlayer = native.addr(0x191de0c) core.writeInteger(DAT_QueuedCommandPlayer, 01) -- In Singleplayer multiplayerID is always 01 return registers end @@ -298,7 +299,7 @@ function Recorder:scheduleNextCommand(registers) return end - print(string.format("Matchtime now: %d", core.readInteger(0x01fe7da8))) + print(string.format("Matchtime now: %d", core.readInteger(native.addr(0x01fe7da8)))) print(string.format("Scheduling the command: Command", c.commandCategory, c.time, self.commandDataAddress, c.size, c.player)) self:scheduleCommand(c) @@ -343,7 +344,7 @@ function Recorder:onReceiveAllTransmittedCommandsASM(scheduleNextCommandAddress) endOfFunction: ]], { commandRecorderState = self.commandRecorderState, - SEC_MatchTime = 0x01fe7da8, + SEC_MatchTime = native.addr(0x01fe7da8), nextUpCommandTimeAddress = self.nextUpCommandTimeAddress, scheduleNextCommandAddress = scheduleNextCommandAddress, }) @@ -364,7 +365,7 @@ function Recorder:onTransmitCommand(registers) local address = core.readInteger(registers.ESP + 12) local size = core.readInteger(registers.ESP + 16) -- local idTo = core.readInteger(registers.ESP + 20) - local player = core.readInteger(0x0191de0c) + local player = core.readInteger(native.addr(0x0191de0c)) print(string.format("Transmitted Command", commandCategory, time, address, size, player)) self:onCommand(commandCategory, time, address, size, player) @@ -376,7 +377,7 @@ function Recorder:onScheduleCommand(registers) local player = core.readInteger(registers.ESP + 8 + 0x10) local time = core.readInteger(registers.ESP + 12 + 0x10) local address = core.readInteger(registers.ESP + 16 + 0x10) - local size = core.readInteger(0x0194af98) + local size = core.readInteger(native.addr(0x0194af98)) print(string.format("Received Command", commandCategory, time, address, size, player)) self:onCommand(commandCategory, time, address, size, player) @@ -393,11 +394,11 @@ end function Recorder:syncCheck(registers, traceF) local actual = { - time = core.readInteger(0x01fe7da8), - rng1 = core.readSmallInteger(0x01a279c0), - rng2 = core.readSmallInteger(0x01a279c2), - index1 = core.readInteger(0x01a3160c), - index2 = core.readInteger(0x01a31608), + time = core.readInteger(native.addr(0x01fe7da8)), + rng1 = core.readSmallInteger(native.addr(0x01a279c0)), + rng2 = core.readSmallInteger(native.addr(0x01a279c2)), + index1 = core.readInteger(native.addr(0x01a3160c)), + index2 = core.readInteger(native.addr(0x01a31608)), extra = {}, } if traceF then actual.extra["ra" .. traceF] = core.readInteger(registers.ESP) end diff --git a/code/ui.lua b/code/ui.lua index a321160..2b1db27 100644 --- a/code/ui.lua +++ b/code/ui.lua @@ -1,7 +1,8 @@ +local native = require("code/native") local menuItemSize = 0x50 local originalMenuItemArraySize = 0x1D10 -local originalMenuItemArrayAddress = 0x005e9848 +local originalMenuItemArrayAddress = native.addr(0x005e9848) local newSize = originalMenuItemArraySize + 2 * menuItemSize -- 2 additional MenuItems local newMenuItemArrayAddress = core.allocate(newSize, true) @@ -33,7 +34,7 @@ local placeRecordButton = function(recorder) core.writeInteger(newItem1Address + 16, 80) -- height core.writeInteger(newItem1Address + 20, utils.createLuaFunctionWrapper(recordButtonCallback)) -- core.writeInteger(newItem1Address + 24, 2) -- callbackParam - core.writeInteger(newItem1Address + 28, 0x0042AE90) -- renderFunc + core.writeInteger(newItem1Address + 28, native.addr(0x0042AE90)) -- renderFunc core.writeInteger(newItem1Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on core.writeInteger(newItem1Address + 36, 0x00000003) -- unknown core.writeInteger(newItem1Address + 40, 0x00000000) -- unknown @@ -66,7 +67,7 @@ local placePlaybackButton = function(recorder) core.writeInteger(newItem2Address + 16, 80) -- height core.writeInteger(newItem2Address + 20, utils.createLuaFunctionWrapper(playbackButtonCallback)) -- core.writeInteger(newItem2Address + 24, 2) -- callbackParam - core.writeInteger(newItem2Address + 28, 0x0042AE90) -- renderFunc + core.writeInteger(newItem2Address + 28, native.addr(0x0042AE90)) -- renderFunc core.writeInteger(newItem2Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on core.writeInteger(newItem2Address + 36, 0x00000003) -- unknown core.writeInteger(newItem2Address + 40, 0x00000000) -- unknown @@ -90,7 +91,7 @@ return { placePlaybackButton(recorder) -- overwrite reference - core.writeCode(0x0059AB30, { core.AssemblyLambda([[ + core.writeCode(native.addr(0x0059AB30), { core.AssemblyLambda([[ push newMenuItemArrayAddress ]], { newMenuItemArrayAddress = newMenuItemArrayAddress, }) }) diff --git a/definition.yml b/definition.yml index 9878a9c..4dba96d 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.1.0 +version: 0.2.0 type: module author: Corax meta: diff --git a/docs/native-port.md b/docs/native-port.md new file mode 100644 index 0000000..d4f8b1c --- /dev/null +++ b/docs/native-port.md @@ -0,0 +1,43 @@ +# SHC / Extreme address verification + +The port uses separate, checked profiles for the original 1.41 executables. All +patch bytes are checked before the first patch is installed. The SHA-256 in each +profile documents the binary used for analysis; runtime selection checks the +patch sites, rather than assuming every file called `Stronghold Crusader.exe` +has the expected layout. + +| Item | Crusader | Extreme | +| --- | --- | --- | +| GameSynchronyState | `0191d768` | `023547d8` | +| Current local handle offset | `6a4` | `6a4` | +| Current player slot offset | `109e74` | `166304` | +| Command ring write index offset | `109ee0` | `166370` | +| Command ring offset / stride | `3c67c` / `1272` | `3c67c` / `1272` | +| RNG structure | `01a279c0` | `024baec0` | +| Simulation tick | `01fe7da8` | `02a7b2a8` | +| Native received-command scheduler | `00480210` | `004803e0` | +| Player identity translation | `0047eaf0` | `0047ecc0` | +| Skirmish menu items | `005e9848` | `005e9708` | + +Instruction signatures were matched between executables with relocation operands +masked, then the differing operands were inspected. In particular, Extreme grows +the synchrony structure: applying a single address delta to both its beginning +and its current-player fields would be incorrect. The menu constructor's object +reference and the next array boundary also confirm the relocated menu array. + +Ghidra's named SHC program and OpenSHC's reconstructed classes were used to check +field meanings and calling conventions. The dust-spawn replacement occupies the +entire five-byte CALL and performs its 44-byte argument cleanup in both builds. + +This port does not make recordings portable between game variants. Session files +must retain the variant and compatible simulation settings. Multiplayer is a +separate capability and is not implied by locating Extreme's command scheduler. + +For an optional check against your legally installed originals: + +``` +python tests/check_executables.py "path/to/Stronghold Crusader Extreme" +``` + +The check reads both files and exercises profile selection through LuaJIT; it +does not launch, patch or redistribute the executables. diff --git a/init.lua b/init.lua index d49d82c..a47b9e4 100644 --- a/init.lua +++ b/init.lua @@ -1,3 +1,4 @@ +local native = require("code/native") local fixes = require("code/fixes") local ui = require("code/ui") local recorder = require("code/recorder") @@ -19,27 +20,27 @@ return { gameRecorder:onStartSkirmish(registers) ui.resetButtons() return registers - end, 0x004428b5, 6) + end, native.addr(0x004428b5), 6) - core.writeCode(0x00442877, {0x90, 0x90, 0x90, 0x90, 0x90, -- make space for detourCode + core.writeCode(native.addr(0x00442877), {0x90, 0x90, 0x90, 0x90, 0x90, -- make space for detourCode 0x90, 0x90, 0x90, 0x90, 0x90}) - core.detourCode(function(registers) return gameRecorder:onBeforeSetMatchSeed(registers) end, 0x00442877, 10) + core.detourCode(function(registers) return gameRecorder:onBeforeSetMatchSeed(registers) end, native.addr(0x00442877), 10) core.detourCode(function(registers) return gameRecorder:onCustomSkirmishGame(registers) - end, 0x0042bf4c, 6) + end, native.addr(0x0042bf4c), 6) -- after LaunchGame() core.detourCode(function(registers) return registers - end, 0x004428c6, 10) + end, native.addr(0x004428c6), 10) -- skirmish end injects -- on switchToMenu core.detourCode(function(registers) - local DAT_CurrentMenuID = core.readInteger(0x01fe7d1c) + local DAT_CurrentMenuID = core.readInteger(native.addr(0x01fe7d1c)) local newMenuID = registers.EBP if newMenuID == 61 then @@ -57,7 +58,7 @@ return { end return registers - end, 0x0046b358, 6) + end, native.addr(0x0046b358), 6) -- on loadGame core.detourCode(function(registers) @@ -71,7 +72,7 @@ return { end return registers - end, 0x00495337, 6) + end, native.addr(0x00495337), 6) -- on restartGame core.detourCode(function(registers) @@ -85,7 +86,7 @@ return { end return registers - end, 0x00494ba5, 5) + end, native.addr(0x00494ba5), 5) -- consuming recorded commands function local scheduleNextCommandAddress = core.allocateCode({ @@ -95,30 +96,30 @@ return { core.detourCode(function(registers) return gameRecorder:scheduleNextCommand(registers) end, scheduleNextCommandAddress, 5) - gameRecorder._scheduleCommand = core.exposeCode(0x00480210, 5, 1) + gameRecorder._scheduleCommand = core.exposeCode(native.addr(0x00480210), 5, 1) -- schedule saved commands when in PLAYBACK mode -- gets called every tick - core.insertCode(0x00490690, 8, + core.insertCode(native.addr(0x00490690), 8, gameRecorder:onReceiveAllTransmittedCommandsASM(scheduleNextCommandAddress), nil, "after") -- process sent commands - core.detourCode(function(registers) return gameRecorder:onTransmitCommand(registers) end, 0x00487c50, 6) + core.detourCode(function(registers) return gameRecorder:onTransmitCommand(registers) end, native.addr(0x00487c50), 6) -- process received commands - core.detourCode(function(registers) return gameRecorder:onScheduleCommand(registers) end, 0x00480353, 6) + core.detourCode(function(registers) return gameRecorder:onScheduleCommand(registers) end, native.addr(0x00480353), 6) --- fakes multiplayerIDs in singleplayer skirmish -- nop and detour to prevent original code execution - core.writeCode(0x0047eaf0, { + core.writeCode(native.addr(0x0047eaf0), { 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, }) - core.detourCode(function(registers) return gameRecorder:fakeMultiplayerIdentities(registers) end, 0x0047eaf0, 6) + core.detourCode(function(registers) return gameRecorder:fakeMultiplayerIdentities(registers) end, native.addr(0x0047eaf0), 6) -- Do not set DAT_QueuedCommandPlayer to -1 in Singleplayer skirmish - core.writeCode(0x004876a6, {0x90,0x90,0x90,0x90,0x90,0x90,}) + core.writeCode(native.addr(0x004876a6), {0x90,0x90,0x90,0x90,0x90,0x90,}) --- Debugging and Logging --- @@ -133,8 +134,8 @@ return { local traceRNG1 = true local traceRNG2 = true -- remove - local rngFunction1 = 0x0046a800 - local rngFunction2 = 0x0046a7d0 + local rngFunction1 = native.addr(0x0046a800) + local rngFunction2 = native.addr(0x0046a7d0) if traceRNG1 then core.detourCode(function(registers) return gameRecorder:syncCheck(registers, 1) end, rngFunction1, 6) end @@ -151,7 +152,7 @@ return { registers.EAX = require("code/validation").integer(config.fixedSeed, -2147483648, 2147483647, "fixed seed") return registers - end, 0x0046a74a, 6) + end, native.addr(0x0046a74a), 6) end diff --git a/tests/check_executables.py b/tests/check_executables.py new file mode 100644 index 0000000..b7021ec --- /dev/null +++ b/tests/check_executables.py @@ -0,0 +1,52 @@ +"""Optional original-binary validation. Requires lupa, uses no native writes.""" +from pathlib import Path +import argparse +import struct +from lupa.luajit21 import LuaRuntime + + +def image_reader(path): + image = path.read_bytes() + pe = struct.unpack_from(' Date: Sat, 5 Sep 2026 13:28:08 +0200 Subject: [PATCH 03/19] Add isolated replay sessions with checked snapshots and completion (0.3.0) --- CHANGELOG.md | 12 ++ README.md | 51 +++---- code/engine-sites.lua | 37 ++++++ code/engine.lua | 154 ++++++++++++++++++++++ code/platform.lua | 41 ++++++ code/recorder.lua | 19 ++- code/session-recorder.lua | 185 ++++++++++++++++++++++++++ code/sessions.lua | 169 ++++++++++++++++++++++++ code/ui.lua | 10 +- code/validation.lua | 37 ++++++ definition.yml | 2 +- docs/replay-sessions.md | 39 ++++++ init.lua | 234 ++++++++++----------------------- tests/check_executables.py | 5 + tests/test_engine.py | 68 ++++++++++ tests/test_recorder.py | 15 ++- tests/test_session_files.py | 123 +++++++++++++++++ tests/test_sessions.py | 155 ++++++++++++++++++++++ tests/test_settings_capture.py | 36 +++++ 19 files changed, 1186 insertions(+), 206 deletions(-) create mode 100644 code/engine-sites.lua create mode 100644 code/engine.lua create mode 100644 code/platform.lua create mode 100644 code/session-recorder.lua create mode 100644 code/sessions.lua create mode 100644 docs/replay-sessions.md create mode 100644 tests/test_engine.py create mode 100644 tests/test_session_files.py create mode 100644 tests/test_sessions.py create mode 100644 tests/test_settings_capture.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c40387a..a855841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.3.0 + +- Store each recording in its own folder with a starting save, full RNG state, UCP configuration, resolved settings and extension versions. +- Begin recording after Skirmish initialization, keeping lobby commands and randomness out of the replay. +- Validate file hashes, command order, player slots and checkpoint timelines before loading a replay. +- Respect the native command queue's capacity, check inferred payload sizes, and block spectator commands during playback. +- Verify RNG checkpoints and the ending RNG state; stop with a diagnostic on divergence or pending commands. +- Publish completed recordings only after closing and validating their streams. Cancelled or failed captures remain distinguishable. +- Restore temporary save/load dialog state and filenames after errors; never overwrite an earlier recording. + +Validation: 34 portable regression tests and native-site checks against Crusader and Extreme. The new starting-save capture and playback path has **not yet passed an end-to-end in-game test**. This is a review/test build, single-player only. Save/load and network commands are unsupported during capture; encountering one marks the capture failed. Replay browsing and settings relaunch follow separately. + ## 0.2.0 - Port the existing native hooks and Skirmish controls to the checked Extreme executable. diff --git a/README.md b/README.md index bb9fe5b..bbdb4ad 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,32 @@ -# UCP Recorder +# UCP Recorder -**WORK IN PROGRESS** +Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -## Getting Started -Used as a module for UCP3. Skirmish lobbies provide UI for record-mode and playback-mode. -At this time, only works in **SHC - Singleplayer-Only** +Version 0.3.0 adds individual replay sessions, starting saves and settings snapshots. **The new save/load playback path still needs end-to-end in-game verification.** Multiplayer is not supported yet. -#### Step 1 -Install UCP3 Developer mode with the ucp-recorder module turned on. +## Testing this build -#### Step 2 -Create Skirmish lobby and choose map & NPCs. +Use a separate game installation with UCP3 developer mode and this module enabled. If your game needs Graphics API Replacer, keep it and its dependencies enabled in the test configuration. -#### Step 3 -Mark the record-checkbox. -![record-checkbox](/img/step3.png) +1. Open a single-player Skirmish lobby and configure the match. +2. Enable the right-hand recorder checkbox near the bottom. This arms a new recording; clicking it again cancels. +3. Start the match. The module captures the starting save and begins recording. +4. Leave through **Quit Mission** to finalize the recording. Do not terminate the process to finish a replay. +5. Return to a Skirmish lobby with the same active UCP settings. The left-hand playback checkbox loads the latest completed session for the current game variant. The saved starting state replaces manual map/AI setup. -#### Step 4 -Start the match and play. -Player actions will be recorded as shown in the terminal. +These remain temporary checkbox controls. A labelled replay browser follows in the next stage. Do not delete older recordings to make a new one: every session has a separate folder under `ucp/replays/`. -#### Step 5 -Exit the match using Escape + "Quit Mission" dialog. -The recording will be automatically saved inside the game folder. +On failure, inspect `ucp3.log` and the session's `last-error.txt` or `desync.json`. A failed or cancelled capture cannot be played. Saving/loading during a recording is currently unsupported and fails the capture explicitly. -#### Step 6 -To view the recording, make sure the Skirmish lobby map and NPCs are configured the same as before. -Lastly, mark the playback-checkbox and start the match. -![playback-checkbox](/img/step6.png) -Any potential errors or desyncs will be printed in the terminal. +See [session format and limitations](docs/replay-sessions.md), [native port notes](docs/native-port.md) and [changelog](CHANGELOG.md). -#### Warning -Currently you need to delete existing recording files in your Game folder whenever you want to make a new recording. +## Development -## Features +```sh +python -m pip install lupa==2.6 +python -m unittest discover -s tests -v +python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" +python tools/build.py +``` -- [x] SHC singleplayer skirmish recording -- [ ] SHC-E recordings -- [ ] Multiplayer recordings -- [ ] Loading recording files via UI \ No newline at end of file +The builder creates `dist/recorder-0.3.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/engine-sites.lua b/code/engine-sites.lua new file mode 100644 index 0000000..a87b4f9 --- /dev/null +++ b/code/engine-sites.lua @@ -0,0 +1,37 @@ +-- Checked native entry points used by session capture and playback. +return { + SHC = { + tick = {address=0x0045CE44, bytes={185,192,121,162,1}}, + save = {address=0x00474480, bytes={131,236,16,83,85,86}}, + fileName = {address=0x0046C300, bytes={139,129,196,11,0,0}}, + mapName = {address=0x0046C2E0, bytes={139,68,36,4,61,244,1,0,0}}, + load = {address=0x004950B0, bytes={129,236,244,3,0,0}}, + queue = {address=0x00489100, bytes={83,86,139,241,139,134,224,158,16,0}}, + copySize = {address=0x00480375, bytes={139,134,48,216,2,0}}, + switchMenu = {address=0x0046B350, bytes={0,0,139,68,36}}, + resources = 0x011BF130, + packager = 0x00F2B3D0, + sections = 0x00B92A58, + menuText = 0x011265A8, + gameCore = 0x01FE7D10, + paused = 0x01FEA054, + writeIndexOffset = 0x00109EE0, + }, + Extreme = { + tick = {address=0x0045D054, bytes={185,192,174,75,2}}, + save = {address=0x004746B0, bytes={131,236,16,83,85,86}}, + fileName = {address=0x0046C520, bytes={139,129,196,11,0,0}}, + mapName = {address=0x0046C500, bytes={139,68,36,4,61,244,1,0,0}}, + load = {address=0x00495210, bytes={129,236,244,3,0,0}}, + queue = {address=0x00489210, bytes={83,86,139,241,139,134,112,99,22,0}}, + copySize = {address=0x00480545, bytes={139,134,48,216,2,0}}, + switchMenu = {address=0x0046B570, bytes={0,0,139,68,36}}, + resources = 0x01293C20, + packager = 0x00F2B850, + sections = 0x00B92BE8, + menuText = 0x01126A28, + gameCore = 0x02A7B210, + paused = 0x02A7D554, + writeIndexOffset = 0x00166370, + }, +} diff --git a/code/engine.lua b/code/engine.lua new file mode 100644 index 0000000..5b84ecd --- /dev/null +++ b/code/engine.lua @@ -0,0 +1,154 @@ +local native = require('code/native') +local allSites = require('code/engine-sites') +local M = {} + +-- Always restore temporary native state, including when a Lua/native wrapper fails. +local function temporarily(run, restore) + local ok, reason=xpcall(run,debug.traceback) + restore() + assert(ok,reason) +end + +function M.verify() + local sites = assert(allSites[native.profile.name]) + for name, site in pairs(sites) do + if type(site)=='table' then + local actual=core.readBytes(site.address, #site.bytes) + for i, value in ipairs(site.bytes) do + assert(actual[i]==value, 'Recorder session hook conflicts at ' .. name) + end + end + end + return sites +end + +function M.new(sites) + local e={sites=sites, base=native.addr(0x191d768), rng=native.addr(0x1a279c0)} + e.schedule=core.exposeCode(native.addr(0x480210),5,1) + e.saveNative=core.exposeCode(sites.save.address,2,1) + e.loadNative=core.exposeCode(sites.load.address,1,0) + e.buffer=core.allocate(1260,true) + e.pathBuffer=core.allocate(512,true) + e.pendingSlots={} + return setmetatable(e,{__index=M}) +end + +function M:tick() return core.readInteger(native.addr(0x1fe7da8)) end +function M:player() return core.readInteger(native.addr(0x1a275dc)) end +function M:singlePlayer() + local mode=core.readInteger(self.base+0x618) + return mode==0 or mode==99 +end +function M:pause() core.writeInteger(self.sites.paused,1) end + +function M:rngState() + return {core.readSmallInteger(self.rng),core.readSmallInteger(self.rng+2), + core.readInteger(self.rng+0x9c48),core.readInteger(self.rng+0x9c4c)} +end + +function M:saveSnapshot(path) + assert(#path<500 and self:singlePlayer(), 'Snapshot requires a single-player game') + local resource=self.sites.resources + local oldType=core.readInteger(resource+0xbc4) + local filename=resource+0x7aee0+1001 + local oldName=core.readBytes(filename,1001) + local oldProgress=core.readInteger(self.sites.packager+0x20) + core.writeInteger(resource+0xbc4,1) + core.writeString(filename,path) + core.writeInteger(self.sites.packager+0x20,0) -- no progress callback/audio during capture + temporarily(function() self.saveNative(self.sites.packager,self.sites.sections) end,function() + core.writeBytes(filename,oldName) -- writeString would add a byte past this fixed array. + core.writeInteger(resource+0xbc4,oldType) + core.writeInteger(self.sites.packager+0x20,oldProgress) + end) + local f=assert(io.open(path,'rb'),'Native save did not produce a starting snapshot') + local size=f:seek('end'); local closed=f:close() + assert(size and size>1000 and closed,'Native starting snapshot is incomplete') +end + +function M:loadSnapshot(path) + assert(#path<500 and self:singlePlayer(), 'Snapshot requires a single-player game') + local state=self.sites.menuText + local old={} + for _,offset in ipairs({0x58,0x7c,0x80,0x884}) do old[offset]=core.readInteger(state+offset) end + self.overridePath=path + core.writeString(self.pathBuffer,path) + core.writeInteger(state+0x58,31) -- native Load action + core.writeInteger(state+0x7c,1) + core.writeInteger(state+0x80,0) + core.writeInteger(state+0x884,0) + self.loading=true + temporarily(function() self.loadNative(0) end,function() + self.loading=false + self.overridePath=nil + -- These four fields belong to the load dialog; preserve the game's menu transition. + for offset,value in pairs(old) do core.writeInteger(state+offset,value) end + end) + self.pendingSlots={} +end + +function M:canSchedule() + local index=core.readInteger(self.base+self.sites.writeIndexOffset) + if index<0 or index>=200 then return false end + local state=core.readByte(self.base+0x3c67c+index*1272+9) + return state==0 or state>=10 +end + +function M:scheduleCommand(command) + require('code/validation').command(command) + assert(self:canSchedule(),'Native command ring is full') + local bytes=require('code/utils').hexToTable(command.data) + for i=#bytes+1,1260 do bytes[i]=0 end + core.writeBytes(self.buffer,bytes) + local slot=core.readInteger(self.base+self.sites.writeIndexOffset) + self.expectedSize=command.size + self.copyError=nil + temporarily(function() + self.schedule(self.base,command.commandCategory,command.player,command.time,self.buffer) + end,function() self.expectedSize=nil end) + if self.copyError then + core.writeByte(self.base+0x3c67c+slot*1272+9,10) + error(self.copyError) + end + self.pendingSlots[slot]=true +end + +function M:commandsPending() + for slot in pairs(self.pendingSlots) do + local state=core.readByte(self.base+0x3c67c+slot*1272+9) + if state>0 and state<10 then return true end + self.pendingSlots[slot]=nil + end + return false +end + +function M:install(recorder) + local originalFileName, originalMapName, originalQueue + originalFileName=core.hookCode(function(this) + if self.overridePath and core.readInteger(this+0xbc4)==1 then return self.pathBuffer end + return originalFileName(this) + end,self.sites.fileName.address,1,1,#self.sites.fileName.bytes) + local dummy=core.allocate(16,true); core.writeString(dummy,'replay') + originalMapName=core.hookCode(function(this,index) + if self.overridePath then return dummy end + return originalMapName(this,index) + end,self.sites.mapName.address,2,1,#self.sites.mapName.bytes) + originalQueue=core.hookCode(function(this,category) + if recorder.mode=='play' and not self.loading then return 0 end + return originalQueue(this,category) + end,self.sites.queue.address,2,1,#self.sites.queue.bytes) + -- Replace exactly MOV EAX,[ESI+commandSize] before the timed payload copy. + -- A corrupt record cannot cause a native read beyond its declared payload. + core.writeBytes(self.sites.copySize.address,{0x90,0x90,0x90,0x90,0x90,0x90}) + core.detourCode(function(registers) + local size=core.readInteger(registers.ESI+0x2d830) + if self.expectedSize and registers.EDX==self.buffer and size~=self.expectedSize then + self.copyError='Native payload size differs from replay record' + size=0 + end + registers.EAX=size + return registers + end,self.sites.copySize.address,6) +end + +return M diff --git a/code/platform.lua b/code/platform.lua new file mode 100644 index 0000000..42e2a36 --- /dev/null +++ b/code/platform.lua @@ -0,0 +1,41 @@ +-- Small Win32 adapter; stdcall is wrapped explicitly because RPS exposeCode +-- supports cdecl/thiscall, not stdcall. No shell command construction is used. +local M = {} +local libraries = {} +local buffers = {} +local function buffer(key,value) + assert(type(value)=='string' and #value<4096 and not value:find('\0',1,true),'Invalid replay path') + buffers[key]=buffers[key] or core.allocate(4096,true) + core.writeString(buffers[key],value) + return buffers[key] +end +function M.stdcall(library, name, count) + if not libraries[library] then libraries[library] = assert(core.openLibraryHandle(library)) end + local address = assert(libraries[library]:getProcAddress(name)) + local code = {0x55, 0x8B, 0xEC} -- push ebp; mov ebp,esp + for i = count, 1, -1 do + code[#code+1] = 0xFF; code[#code+1] = 0x75; code[#code+1] = 4+i*4 + end + code[#code+1] = core.callTo(address) + code[#code+1] = 0x5D; code[#code+1] = 0xC3 + return core.exposeCode(core.allocateCode(code), count, 0) +end + +local mkdir, getError +function M.mkdir(path) + mkdir = mkdir or M.stdcall('kernel32.dll', 'CreateDirectoryA', 2) + getError = getError or M.stdcall('kernel32.dll','GetLastError',0) + if mkdir(buffer('directory',path),0)~=0 then return true end + local reason=getError() + assert(reason==183,'Cannot create replay directory (Windows error '..reason..')') + return false +end + +local move +function M.replace(source, destination) + move = move or M.stdcall('kernel32.dll', 'MoveFileExA', 3) + local a,b=buffer('source',source),buffer('destination',destination) + assert(move(a, b, 9) ~= 0, 'Cannot finish writing replay metadata') +end + +return M diff --git a/code/recorder.lua b/code/recorder.lua index 43420d8..b467386 100644 --- a/code/recorder.lua +++ b/code/recorder.lua @@ -40,11 +40,16 @@ function Recorder:reset() for _, address in ipairs({self.commandRecorderState, self.rngRecorderState, self.infoRecorderState}) do core.writeInteger(address, self.RECORDER_STATES.NONE) end + local closeError for _, key in ipairs({"commandsFile", "rngFile", "infoFile"}) do local file = self[key] self[key] = nil - if file then file:close() end + if file then + local ok, closed, reason = pcall(file.close, file) + if not ok or not closed then closeError = closeError or reason or closed or 'Cannot close replay file' end + end end + assert(not closeError, tostring(closeError)) end -- Open the complete set before committing session state. A failed open must @@ -124,8 +129,8 @@ function Recorder:saveCommand(commandCategory, time, address, size, player) size = size, player = player, }) - self.commandsFile:write(data .. "\n") - self.commandsFile:flush() + assert(self.commandsFile:write(data .. "\n")) + assert(self.commandsFile:flush()) end function Recorder:loadCommand() @@ -145,8 +150,8 @@ function Recorder:saveRNG(time, index1, rng1, index2, rng2, extra) rng2 = rng2, extra = extra, }) - self.rngFile:write(data .. "\n") - self.rngFile:flush() + assert(self.rngFile:write(data .. "\n")) + assert(self.rngFile:flush()) end function Recorder:loadRNG() @@ -167,8 +172,8 @@ function Recorder:saveInfo(gameType, mapSeed, matchSeed, RNGvalue1, RNGvalue2, R RNGindex1 = RNGindex1, RNGindex2 = RNGindex2, }) - self.infoFile:write(data .. "\n") - self.infoFile:flush() + assert(self.infoFile:write(data .. "\n")) + assert(self.infoFile:flush()) end function Recorder:loadInfo() diff --git a/code/session-recorder.lua b/code/session-recorder.lua new file mode 100644 index 0000000..5c884b6 --- /dev/null +++ b/code/session-recorder.lua @@ -0,0 +1,185 @@ +local Base = require('code/recorder') +local store = require('code/sessions') +local native = require('code/native') +local validation = require('code/validation') +local Session = setmetatable({}, {__index=Base}) + +function Session:new(engine) + local o=Base:new({name='unused',rngLogMethod='checkpoints'}) + o.engine=engine + o.halt=core.allocate(4,true) + o.status='idle' + return setmetatable(o,{__index=self}) +end + +function Session:guard(callback) + local ok, reason=xpcall(callback,debug.traceback) + if not ok then + if self.status=='error' then return false end -- retain the first failure + self.status='error'; self.error=tostring(reason) + core.writeInteger(self.halt,1) + if self.active then self.engine:pause() end + print('Replay stopped: ' .. self.error) + if self.manifest then + pcall(store.write,store.path(self.manifest.id)..'/last-error.txt',self.error) + if self.mode=='record' then + self.manifest.status='failed'; pcall(store.save,self.manifest) + end + end + for _,key in ipairs({'commandsFile','rngFile','infoFile'}) do + local file=self[key]; self[key]=nil + if file then pcall(file.close,file) end + end + end + return ok +end + +function Session:startRecording() + assert(self.mode=='none','A replay session is already active') + assert(self.engine:singlePlayer(),'Recording currently supports single-player Skirmish') + self.manifest=store.new(native.profile) + self:setName(store.path(self.manifest.id)..'/stream') + self:openFiles('w') + self.mode='record'; self.status='armed'; self.active=false + self.error=nil; self.observedTick=false + core.writeInteger(self.halt,0) +end + +function Session:activateRecording() + if self.mode~='record' or self.status~='armed' then return end + local path=store.path(self.manifest.id) + self.engine:saveSnapshot(path..'/start.sav') + store.write(path..'/rng.bin',core.readString(self.engine.rng,0x9c50)) + self.manifest.snapshotHash=sha.sha256(store.read(path..'/start.sav')) + self.manifest.rngHash=sha.sha256(store.read(path..'/rng.bin')) + self.manifest.player=self.engine:player() + self.manifest.startTick=self.engine:tick() + self.manifest.lastTick=self.manifest.startTick + local r=self.engine:rngState() + local seed=core.readInteger(self.engine.rng+4) + self:saveInfo(0,seed,seed,r[1],r[2],r[4],r[3]) + self.manifest.status='recording'; store.save(self.manifest) + self.active=true; self.status='recording' + print('Recording '..self.manifest.id) +end + +function Session:startPlayback(id) + assert(self.mode=='none','A replay session is already active') + assert(self.engine:singlePlayer(),'Replay playback is single-player only') + if not id then + for _, item in ipairs(store.list()) do + if item.status=='complete' and item.variant==native.profile.name then id=item.id; break end + end + end + assert(id,'No completed recording is available') + local manifest=store.load(id,native.profile) + assert(store.compatible(manifest),'Replay requires its recorded UCP settings') + store.preflight(manifest) + local path=store.path(id) + local snapshot=store.read(path..'/start.sav') + local rng=store.read(path..'/rng.bin') + assert(sha.sha256(snapshot)==manifest.snapshotHash,'Starting save is damaged') + assert(#rng==0x9c50 and sha.sha256(rng)==manifest.rngHash,'Starting RNG state is damaged') + self:setName(path..'/stream') + self:openFiles('r') + self.manifest=manifest + self.mode='play'; self.status='loading'; self.active=false + self.error=nil + core.writeInteger(self.halt,0) + self.engine:loadSnapshot(path..'/start.sav') + local bytes={}; for i=1,#rng do bytes[i]=rng:byte(i) end + core.writeBytes(self.engine.rng,bytes) + assert(self.engine:tick()==manifest.startTick,'Loaded save has a different starting tick') + assert(self.engine:player()==manifest.player,'Loaded save has a different player slot') + self.active=true; self.status='playing'; self.playedCommands=0 + self.nextCheckpoint=nil + print('Playing '..id) +end + +function Session:onCommand(category,time,address,size) + if self.status~='recording' or not self.active or time<=0 then return end + validation.integer(size,0,1260,'native command size') + validation.sessionCommand({commandCategory=category,time=time,player=self.engine:player(), + size=size,data=require('code/utils').tableToHex(core.readBytes(address,size))},self.manifest) + self:saveCommand(category,time,address,size,self.engine:player()) + self.manifest.commandCount=self.manifest.commandCount+1 +end + +function Session:feed() + if self.status~='playing' then return end + local now=self.engine:tick() + while self.engine:canSchedule() do + local c=self:peekCommand() + if not c or c.time>now+64 then return end + assert(c.time>=now,'Replay command missed its simulation tick') + validation.sessionCommand(c,self.manifest) + self.engine:scheduleCommand(c) + self:consumeSavedCommand() + self.playedCommands=self.playedCommands+1 + end +end + +function Session:onTick() + if not self.active then return end + local now=self.engine:tick() + if self.status=='recording' then + self.manifest.lastTick=now + self.manifest.finalRng=self.engine:rngState() + self.observedTick=true + if now%64==0 then + local line=json:encode({time=now,rng=self.engine:rngState()}) + assert(self.rngFile:write(line..'\n')); assert(self.rngFile:flush()) + end + elseif self.status=='playing' then + assert(now<=self.manifest.lastTick,'Replay passed its ending tick') + if now%64==0 then + local line=self.rngFile:read() + assert(line,'Replay verification data ended early') + local expected=json:decode(line) + local actual=self.engine:rngState() + assert(expected.time==now,'Replay checkpoint tick differs') + for i=1,4 do + if actual[i]~=expected.rng[i] then + self.firstDesync={time=now,expected=expected.rng,actual=actual} + store.write(store.path(self.manifest.id)..'/desync.json',json:encode(self.firstDesync)) + error('RNG divergence at tick '..now..' (field '..i..')') + end + end + end + if now>=self.manifest.lastTick then + assert(self.playedCommands==self.manifest.commandCount,'Replay ended before all commands were scheduled') + assert(not self.engine:commandsPending(),'Replay ended with commands still waiting to execute') + local actual=self.engine:rngState() + for i=1,4 do assert(actual[i]==self.manifest.finalRng[i],'Final RNG state differs at tick '..now) end + self.status='finished'; core.writeInteger(self.halt,1); self.engine:pause() + store.write(store.path(self.manifest.id)..'/last-playback.json',json:encode({ + status='finished',lastTick=now,commands=self.playedCommands,rngCheckpoints='matched'})) + end + end +end + +function Session:reset() + local manifest=self.mode=='record' and self.manifest + local complete=manifest and self.active and self.observedTick and self.status=='recording' + self.active=false + local closed,reason=pcall(Base.reset,self) + if manifest then + if complete and closed then + local ok,finishError=pcall(store.finish,manifest) + if not ok then + manifest.status='failed'; pcall(store.save,manifest) + error(finishError) + end + else + manifest.status=(self.status=='error' or not closed) and 'failed' or 'cancelled' + store.save(manifest) + end + end + assert(closed,reason) + self.status='idle'; self.manifest=nil; self.nextCheckpoint=nil + self.observedTick=false; self.error=nil + core.writeInteger(self.halt,0) +end + +function Session:discardFiles() end -- Cancelling never deletes a previous recording. +return Session diff --git a/code/sessions.lua b/code/sessions.lua new file mode 100644 index 0000000..5839449 --- /dev/null +++ b/code/sessions.lua @@ -0,0 +1,169 @@ +local platform = require('code/platform') +local validation = require('code/validation') +local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v1'} +local activeSettings + +local function read(path) + local f, err = io.open(path, 'rb') + assert(f, tostring(err)) + local data = assert(f:read('*a')) + assert(f:close()) + return data +end + +local function write(path, data) + local f, err = io.open(path, 'wb') + assert(f, tostring(err)) + local ok, reason = f:write(data) + local closed, closeReason = f:close() + assert(ok and closed, tostring(reason or closeReason)) +end + +function M.path(id) + assert(type(id) == 'string' and #id < 80 and id:match('^[%w_-]+$'), 'Invalid replay identifier') + return M.ROOT .. '/' .. id +end + +-- Stable encoding: Lua table iteration order is not a compatibility boundary. +local function canonical(value) + if type(value)~='table' then return json:encode(value) end + local keys={}; for key in pairs(value) do keys[#keys+1]=key end + table.sort(keys,function(a,b) return tostring(a)b.id end) + return result +end + +function M.load(id, profile) + local manifest=json:decode(read(M.path(id) .. '/manifest.json')) + assert(type(manifest)=='table' and manifest.format==M.FORMAT and manifest.id==id, 'Unsupported replay format') + assert(manifest.variant==profile.name and manifest.executable==profile.sha256, 'Replay requires ' .. tostring(manifest.variant)) + assert(manifest.simulationProfile==M.PROFILE, 'Replay uses a different simulation profile') + assert(manifest.status=='complete', 'Recording was not completed') + validation.manifest(manifest) + assert(sha.sha256(read(M.path(id)..'/ucp-config.yml'))==manifest.settingsHash, 'Recorded settings are damaged') + assert(sha.sha256(read(M.path(id)..'/environment.json'))==manifest.environmentHash,'Recorded environment is damaged') + return manifest +end + +local streams={commands='stream-commands.json',checkpoints='stream-rng-sync.json',info='stream-infself.json'} + +-- Seal only commands that fall within the last observed simulation boundary. +-- Inputs queued for a later tick when the player leaves are not part of this replay. +function M.finish(manifest) + local path=M.path(manifest.id) + local commandsPath=path..'/'..streams.commands + local input=assert(io.open(commandsPath,'rb')) + local output, openError=io.open(commandsPath..'.tmp','wb') + if not output then input:close(); error(openError) end + local count, previous=0,manifest.startTick + local ok,reason=xpcall(function() + for line in input:lines() do + local c=validation.sessionCommand(json:decode(line),manifest) + assert(c.time>=previous,'Replay commands are out of order') + previous=c.time + if c.time<=manifest.lastTick then + assert(output:write(line..'\n')); count=count+1 + end + end + end,debug.traceback) + local inClosed=input:close(); local outClosed=output:close() + assert(ok and inClosed and outClosed,reason or 'Cannot finish replay command stream') + platform.replace(commandsPath..'.tmp',commandsPath) + manifest.commandCount=count + for name,file in pairs(streams) do manifest[name..'Hash']=sha.sha256(read(path..'/'..file)) end + M.preflight(manifest) + manifest.status='complete' + M.save(manifest) +end + +-- Check every stream before loading the native save, including data near EOF. +function M.preflight(manifest) + validation.manifest(manifest) + local path=M.path(manifest.id) + local data={} + for name,file in pairs(streams) do + data[name]=read(path..'/'..file) + assert(sha.sha256(data[name])==manifest[name..'Hash'],'Replay '..name..' stream is damaged') + end + local count,previous=0,manifest.startTick + for line in data.commands:gmatch('[^\r\n]+') do + local c=validation.sessionCommand(json:decode(line),manifest) + assert(c.time>=previous and c.time<=manifest.lastTick,'Replay command tick is outside its ordered timeline') + count=count+1; previous=c.time + end + assert(count==manifest.commandCount,'Replay command count differs') + local tick=math.ceil(manifest.startTick/64)*64 + for line in data.checkpoints:gmatch('[^\r\n]+') do + local checkpoint=json:decode(line) + assert(type(checkpoint)=='table' and checkpoint.time==tick and tick<=manifest.lastTick,'Invalid replay checkpoint timeline') + validation.rng(checkpoint.rng) + tick=tick+64 + end + assert(tick>manifest.lastTick,'Replay verification data ended early') + validation.info(json:decode(data.info)) +end + +function M.compatible(manifest) + local current=M.settings() + return current.hash==manifest.settingsHash and current.environmentHash==manifest.environmentHash +end + +M.read, M.write = read, write +return M diff --git a/code/ui.lua b/code/ui.lua index 2b1db27..dc1b7b5 100644 --- a/code/ui.lua +++ b/code/ui.lua @@ -32,7 +32,10 @@ local placeRecordButton = function(recorder) core.writeInteger(newItem1Address + 8, 560) -- yPos core.writeInteger(newItem1Address + 12, 80) -- width core.writeInteger(newItem1Address + 16, 80) -- height - core.writeInteger(newItem1Address + 20, utils.createLuaFunctionWrapper(recordButtonCallback)) -- + core.writeInteger(newItem1Address + 20, utils.createLuaFunctionWrapper(function(registers) + recorder:guard(recordButtonCallback) + return registers + end)) core.writeInteger(newItem1Address + 24, 2) -- callbackParam core.writeInteger(newItem1Address + 28, native.addr(0x0042AE90)) -- renderFunc core.writeInteger(newItem1Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on @@ -65,7 +68,10 @@ local placePlaybackButton = function(recorder) core.writeInteger(newItem2Address + 8, 560) -- yPos core.writeInteger(newItem2Address + 12, 80) -- width core.writeInteger(newItem2Address + 16, 80) -- height - core.writeInteger(newItem2Address + 20, utils.createLuaFunctionWrapper(playbackButtonCallback)) -- + core.writeInteger(newItem2Address + 20, utils.createLuaFunctionWrapper(function(registers) + recorder:guard(playbackButtonCallback) + return registers + end)) core.writeInteger(newItem2Address + 24, 2) -- callbackParam core.writeInteger(newItem2Address + 28, native.addr(0x0042AE90)) -- renderFunc core.writeInteger(newItem2Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on diff --git a/code/validation.lua b/code/validation.lua index 5abbcab..ee01e6e 100644 --- a/code/validation.lua +++ b/code/validation.lua @@ -33,4 +33,41 @@ function M.info(info) return info end +-- Single-player simulation commands from OpenSHC's GameCommandType. Protocol, +-- lobby, save/load and unknown extension commands must never run from a replay. +local gameplay = {} +for _,id in ipairs({14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29, + 31,34,35,36,38,41,42,43,44,45,68,69,70,71,78,113,119}) do gameplay[id]=true end + +function M.sessionCommand(command, manifest) + M.command(command) + assert(gameplay[command.commandCategory], + 'Unsupported replay command category '..command.commandCategory..' (save/load and network commands are excluded)') + assert(command.commandCategory~=119 or manifest.variant=='Extreme','Tactical powers require Extreme') + assert(command.player==manifest.player,'Replay command uses a different player slot') + return command +end + +function M.rng(values) + assert(type(values)=='table' and #values==4,'Invalid replay RNG checkpoint') + M.integer(values[1],-32768,65535,'RNG value 1') + M.integer(values[2],-32768,65535,'RNG value 2') + M.integer(values[3],0,19999,'RNG index 2') + M.integer(values[4],0,19999,'RNG index 1') +end + +function M.manifest(value) + assert(type(value)=='table','Invalid replay manifest') + M.integer(value.player,1,8,'player slot') + M.integer(value.startTick,0,2147483647,'starting tick') + M.integer(value.lastTick,value.startTick,2147483647,'ending tick') + M.integer(value.commandCount,0,2147483647,'command count') + M.rng(value.finalRng) + for _,key in ipairs({'settingsHash','environmentHash','snapshotHash','rngHash','commandsHash','checkpointsHash','infoHash'}) do + assert(type(value[key])=='string' and #value[key]==64 and not value[key]:find('[^%x]'), + 'Invalid replay '..key) + end + return value +end + return M diff --git a/definition.yml b/definition.yml index 4dba96d..74a8e3b 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.2.0 +version: 0.3.0 type: module author: Corax meta: diff --git a/docs/replay-sessions.md b/docs/replay-sessions.md new file mode 100644 index 0000000..5132696 --- /dev/null +++ b/docs/replay-sessions.md @@ -0,0 +1,39 @@ +# Replay sessions (0.3.0) + +This stage replaces manual recreation of the lobby with a native starting save and full RNG snapshot. It is ready for code review and isolated testing, not yet confirmed for complete live playback. + +## Files + +Each session reserves `ucp/replays/YYYYMMDD-HHMMSS-NNNN/`. Its manifest records the game variant, expected executable identity, simulation patch profile, active UCP settings, resolved configuration and extension versions. The folder contains: + +- `start.sav`: native save captured after Skirmish initialization. +- `rng.bin`: the complete 40,016-byte RNG structure, including its table. +- `ucp-config.yml` and `environment.json`: startup settings and resolved environment. +- `stream-commands.json`, `stream-rng-sync.json`, `stream-infself.json`: commands, 64-tick RNG checkpoints and initial match information. +- `manifest.json`: status, timeline, command count, final RNG values/indices and SHA-256 hashes. + +The manifest moves through armed, recording and complete. Cancelled and failed sessions cannot play. Stream close failures and validation failures prevent completion. Metadata is replaced through a temporary file; this prevents a partial JSON document becoming the published manifest, but does not promise recovery from every power-loss scenario. + +## Simulation boundary + +Capture starts after the native Skirmish launch function initializes the game. Checkpoints observe the same point immediately before each simulation tick, independent of render-frame count. The ending state is the last observed boundary, not the later time at which an exit dialog runs. Future commands still queued when recording ends are excluded from the finalized stream. + +Playback preflights all files, loads the native starting save, restores the full RNG structure, and checks the loaded tick and local player. The native scheduler receives recorded commands with their original tick. It must have a free slot; its inferred payload length must match the recorded length. At completion, all recorded commands must have been scheduled and their slots consumed. RNG verification covers two values and two indices, **not a full world-state checksum**. + +The local single-player identity stays intact. New commands queued by a spectator are blocked while playback is active, finished or failed. This hook may also block commands from automation extensions; compatibility with those extensions requires separate analysis and testing. + +## Configuration compatibility + +Settings are captured when the module enables, not reread when the user begins a recording. Playback requires the captured configuration bytes and the resolved configuration/extension-version environment to match. A restart with the saved configuration is necessary when these differ. This stage does not relaunch automatically. + +Version equality does not prove that unpacked extensions have identical source files, or that map assets and external resources are unchanged. No extension download, installation or version switching is performed. Replays are local test artifacts; native save/payload handling has not been audited as a parser for hostile downloaded files. + +## Unsupported cases + +Multiplayer capture/playback, resynchronization packets, lobby commands, native save/load commands, unknown extension commands and cross-variant playback are excluded. Unsupported commands during recording fail the capture with an error instead of silently omitting actions. Game speed and pause behaviour, automated actions, native save/load RNG side effects, repeated playback, match-end transitions and Extreme matches still require live verification. + +## Native evidence + +`engine-sites.lua` contains checked entry bytes for both executables. Save/load uses FilePackager and the game's load-dialog handler, with a temporary filename override limited to save resources. Temporary resource type, filename, progress callback and list-selection fields are restored after failure. The scheduler copy guard replaces the six-byte load of `commandSize` before the payload copy. The simulation hook runs before the native RNG advance and tick increment; its halt branch skips that tick sequence. + +See `tests/check_executables.py` for read-only checks against original executable files and `tests/test_engine.py` for injected native-wrapper failures. Neither replaces a successful live capture/load/replay comparison. diff --git a/init.lua b/init.lua index a47b9e4..953528d 100644 --- a/init.lua +++ b/init.lua @@ -1,163 +1,71 @@ -local native = require("code/native") -local fixes = require("code/fixes") -local ui = require("code/ui") -local recorder = require("code/recorder") - -return { - enable = function(self, config) - - --- Init --- - - require("code/native").verify() - fixes.apply() - local gameRecorder = recorder:new({name = "test_recording1", rngLogMethod = config.rngLogMethod}) - ui.createButtons(gameRecorder) - - --- Injections --- - - -- skirmish startup injects - core.detourCode(function(registers) - gameRecorder:onStartSkirmish(registers) - ui.resetButtons() - return registers - end, native.addr(0x004428b5), 6) - - core.writeCode(native.addr(0x00442877), {0x90, 0x90, 0x90, 0x90, 0x90, -- make space for detourCode - 0x90, 0x90, 0x90, 0x90, 0x90}) - core.detourCode(function(registers) return gameRecorder:onBeforeSetMatchSeed(registers) end, native.addr(0x00442877), 10) - - core.detourCode(function(registers) - return gameRecorder:onCustomSkirmishGame(registers) - end, native.addr(0x0042bf4c), 6) - - -- after LaunchGame() - core.detourCode(function(registers) - - return registers - end, native.addr(0x004428c6), 10) - - -- skirmish end injects - - -- on switchToMenu - core.detourCode(function(registers) - local DAT_CurrentMenuID = core.readInteger(native.addr(0x01fe7d1c)) - local newMenuID = registers.EBP - - if newMenuID == 61 then - print("ingame menu change - exited match?") - print("Old menu: " .. DAT_CurrentMenuID) - print("New menu: " .. newMenuID) - print("onMenuChange") - if gameRecorder.mode == "record" then - print("stopping recording") - gameRecorder:stopRecording() - elseif gameRecorder.mode == "play" then - print("stopping playback") - gameRecorder:stopPlayback() - end - end - - return registers - end, native.addr(0x0046b358), 6) - - -- on loadGame - core.detourCode(function(registers) - print("onLoadGame") - if gameRecorder.mode == "record" then - print("stopping recording") - gameRecorder:stopRecording() - elseif gameRecorder.mode == "play" then - print("stopping playback") - gameRecorder:stopPlayback() - end - - return registers - end, native.addr(0x00495337), 6) - - -- on restartGame - core.detourCode(function(registers) - print("onRestartGame") - if gameRecorder.mode == "record" then - print("stopping recording") - gameRecorder:stopRecording() - elseif gameRecorder.mode == "play" then - print("stopping playback") - gameRecorder:stopPlayback() - end - - return registers - end, native.addr(0x00494ba5), 5) - - -- consuming recorded commands function - local scheduleNextCommandAddress = core.allocateCode({ - 0x90, 0x90, 0x90, 0x90, 0x90, - 0xC3, - }) - - core.detourCode(function(registers) return gameRecorder:scheduleNextCommand(registers) end, scheduleNextCommandAddress, 5) - - gameRecorder._scheduleCommand = core.exposeCode(native.addr(0x00480210), 5, 1) - - -- schedule saved commands when in PLAYBACK mode - -- gets called every tick - core.insertCode(native.addr(0x00490690), 8, - gameRecorder:onReceiveAllTransmittedCommandsASM(scheduleNextCommandAddress), - nil, "after") - - -- process sent commands - core.detourCode(function(registers) return gameRecorder:onTransmitCommand(registers) end, native.addr(0x00487c50), 6) - - -- process received commands - core.detourCode(function(registers) return gameRecorder:onScheduleCommand(registers) end, native.addr(0x00480353), 6) - - --- fakes multiplayerIDs in singleplayer skirmish - -- nop and detour to prevent original code execution - core.writeCode(native.addr(0x0047eaf0), { - 0x90, 0x90, 0x90, 0x90, 0x90, - 0x90, - }) - core.detourCode(function(registers) return gameRecorder:fakeMultiplayerIdentities(registers) end, native.addr(0x0047eaf0), 6) - - -- Do not set DAT_QueuedCommandPlayer to -1 in Singleplayer skirmish - core.writeCode(native.addr(0x004876a6), {0x90,0x90,0x90,0x90,0x90,0x90,}) - - --- Debugging and Logging --- - - local syncCheckAddress = core.allocateCode({ - 0x90, 0x90, 0x90, 0x90, 0x90, - 0xC3, - }) - - core.detourCode(function(registers) return gameRecorder:syncCheck(registers) end, syncCheckAddress, 5) - - if config.rngLogMethod == "trace" then - local traceRNG1 = true - local traceRNG2 = true -- remove - - local rngFunction1 = native.addr(0x0046a800) - local rngFunction2 = native.addr(0x0046a7d0) - - if traceRNG1 then core.detourCode(function(registers) return gameRecorder:syncCheck(registers, 1) end, rngFunction1, 6) end - - if traceRNG2 then core.detourCode(function(registers) return gameRecorder:syncCheck(registers, 2) end, rngFunction2, 6) end - - end - - -- optional - -- Makes every game's RNG use the same seed - if config.useFixedSeed then - - core.detourCode(function(registers) - - registers.EAX = require("code/validation").integer(config.fixedSeed, -2147483648, 2147483647, "fixed seed") - - return registers - end, native.addr(0x0046a74a), 6) - - end - - end, - - disable = function(self, config) - end, -} +local native=require('code/native') +local Engine=require('code/engine') +local Session=require('code/session-recorder') +local module={} + +function module:enable(config) + native.verify() + local sites=Engine.verify() + local seed + if config.useFixedSeed then + seed=require('code/validation').integer(config.fixedSeed,-2147483648,2147483647,'fixed seed') + end + require('code/sessions').captureSettings() + require('code/fixes').apply() + local engine=Engine.new(sites) + local recorder=Session:new(engine) + self.recorder=recorder + engine:install(recorder) + local ui=require('code/ui') + ui.createButtons(recorder) + + local function observe(address,size,callback) + core.detourCode(function(registers) + recorder:guard(function() callback(registers) end) + return registers + end,address,size) + end + observe(native.addr(0x4428c6),10,function() + recorder:activateRecording() + ui.resetButtons() + end) + observe(native.addr(0x487c50),6,function(registers) + if recorder.active and recorder.mode=='record' then recorder:onTransmitCommand(registers) end + end) + observe(native.addr(0x490690),8,function() recorder:feed() end) + observe(native.addr(0x46b358),6,function(registers) + if registers.EBP==61 and not engine.loading then recorder:reset(); ui.resetButtons() end + end) + observe(native.addr(0x495337),6,function() + if not engine.loading then recorder:reset() end + end) + observe(native.addr(0x494ba5),5,function() recorder:reset() end) + + local tickCallback=core.allocateCode({0x90,0x90,0x90,0x90,0x90,0xC3}) + observe(tickCallback,5,function() recorder:onTick() end) + core.insertCode(sites.tick.address,5,{ + core.AssemblyLambda([[ + pushfd + pushad + call callback + popad + cmp dword [halt], 0 + je continueTick + popfd + jmp skipTick + continueTick: + popfd + ]],{callback=tickCallback,halt=recorder.halt,skipTick=sites.tick.address+0x25}) + },nil,'after') + if seed then + core.detourCode(function(registers) + registers.EAX=seed + return registers + end,native.addr(0x46a74a),6) + end +end + +function module:disable() + if self.recorder then self.recorder:reset() end +end +return module diff --git a/tests/check_executables.py b/tests/check_executables.py index b7021ec..2fef2c3 100644 --- a/tests/check_executables.py +++ b/tests/check_executables.py @@ -43,6 +43,11 @@ def check(folder): menu = native.addr(0x59ab30) assert reader(menu, 1) == b'\x68' assert struct.unpack(' Date: Sat, 5 Sep 2026 13:50:28 +0200 Subject: [PATCH 04/19] Pause failed native loads without halting rejected replay preflight --- code/engine.lua | 2 +- code/session-recorder.lua | 7 ++++--- tests/test_sessions.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/code/engine.lua b/code/engine.lua index 5b84ecd..144d43e 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -139,7 +139,7 @@ function M:install(recorder) end,self.sites.queue.address,2,1,#self.sites.queue.bytes) -- Replace exactly MOV EAX,[ESI+commandSize] before the timed payload copy. -- A corrupt record cannot cause a native read beyond its declared payload. - core.writeBytes(self.sites.copySize.address,{0x90,0x90,0x90,0x90,0x90,0x90}) + core.writeCode(self.sites.copySize.address,{0x90,0x90,0x90,0x90,0x90,0x90}) core.detourCode(function(registers) local size=core.readInteger(registers.ESI+0x2d830) if self.expectedSize and registers.EDX==self.buffer and size~=self.expectedSize then diff --git a/code/session-recorder.lua b/code/session-recorder.lua index 5c884b6..fb73c58 100644 --- a/code/session-recorder.lua +++ b/code/session-recorder.lua @@ -15,10 +15,11 @@ end function Session:guard(callback) local ok, reason=xpcall(callback,debug.traceback) if not ok then - if self.status=='error' then return false end -- retain the first failure + if self.status=='error' and (self.active or self.mode~='none') then return false end -- retain the first session failure self.status='error'; self.error=tostring(reason) - core.writeInteger(self.halt,1) - if self.active then self.engine:pause() end + local stopSimulation=self.active or self.mode=='play' + core.writeInteger(self.halt,stopSimulation and 1 or 0) + if stopSimulation then self.engine:pause() end print('Replay stopped: ' .. self.error) if self.manifest then pcall(store.write,store.path(self.manifest.id)..'/last-error.txt',self.error) diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 9faef91..1f37710 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -152,4 +152,18 @@ def test_save_and_network_commands_are_not_replayed(self): local c=command(); c.commandCategory=119 assert(not pcall(validation.sessionCommand,c,{player=1,variant='SHC'})) assert(pcall(validation.sessionCommand,c,{player=1,variant='Extreme'})) +''') + + def test_rejected_preflight_does_not_halt_a_later_normal_match(self): + self.check(''' +local r=session() +assert(not r:guard(function() error('damaged recording before native load') end)) +assert(r.mode=='none' and memory[r.halt]==0 and not paused) +''') + + def test_failed_native_load_halts_even_before_playback_becomes_active(self): + self.check(''' +local r=session(); r.mode='play'; r.status='loading'; r.active=false +assert(not r:guard(function() error('load failed after native changes') end)) +assert(memory[r.halt]==1 and paused) ''') From 0779e00277dc0c6b570d07a95fb247f0bc55f32a Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:51:56 +0200 Subject: [PATCH 05/19] Add native replay picker and recorded-settings restart handoff (0.4.0) --- .github/workflows/tests.yml | 7 +- CHANGELOG.md | 13 ++- README.md | 15 ++-- code/browser.lua | 65 ++++++++++++++ code/native-ui.lua | 95 ++++++++++++++++++++ code/platform.lua | 45 ++++++++-- code/restart-script.lua | 33 +++++++ code/restart.lua | 30 +++++++ code/ui-sites.lua | 30 +++++++ code/ui.lua | 169 +++++++++++++---------------------- definition.yml | 2 +- docs/replay-browser.md | 26 ++++++ init.lua | 3 +- tests/check_executables.py | 11 +-- tests/test_browser.py | 68 ++++++++++++++ tests/test_native_ui.py | 64 +++++++++++++ tests/test_recorder.py | 1 + tests/test_restart.py | 52 +++++++++++ tests/test_restart_helper.py | 98 ++++++++++++++++++++ tests/test_win32_abi.py | 61 +++++++++++++ 20 files changed, 757 insertions(+), 131 deletions(-) create mode 100644 code/browser.lua create mode 100644 code/native-ui.lua create mode 100644 code/restart-script.lua create mode 100644 code/restart.lua create mode 100644 code/ui-sites.lua create mode 100644 docs/replay-browser.md create mode 100644 tests/test_browser.py create mode 100644 tests/test_native_ui.py create mode 100644 tests/test_restart.py create mode 100644 tests/test_restart_helper.py create mode 100644 tests/test_win32_abi.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e96e43..0d3decb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,11 +2,14 @@ name: Recorder tests on: [push, pull_request] jobs: lua: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.12' - - run: python -m pip install lupa==2.6 + - run: python -m pip install lupa==2.6 unicorn==2.1.4 - run: python -m unittest discover -s tests -v diff --git a/CHANGELOG.md b/CHANGELOG.md index a855841..818beaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.4.0 + +- Replace temporary Skirmish checkboxes with labelled **Record next match** and **Replays** controls. +- Add a paged replay picker using the game's native red frame, text and input handling. Show session status, tick duration and settings compatibility before playback. +- Add **Queue settings restart**: a hidden helper waits for the current game to exit, verifies the executable and saved configuration, and restarts with that configuration. The normal UCP configuration stays unchanged. +- Preselect the requested replay when opening the browser after a queued restart. +- Check native UI entry points separately for Crusader and Extreme; avoid overwriting existing modal IDs or writing past menu arrays. +- Keep rejected preflight checks from freezing a later normal match, while pausing after a failed native playback load. + +Validation: 53 tests, including Windows helper tests with process waiting/launching replaced by fakes and emulation of the 32-bit Windows call wrappers; original-binary checks for both games. Native rendering, input handling and the real restart remain **pending live testing**, together with the full replay test from 0.3.0. No game was launched by these automated tests. + ## 0.3.0 - Store each recording in its own folder with a starting save, full RNG state, UCP configuration, resolved settings and extension versions. @@ -10,7 +21,7 @@ - Publish completed recordings only after closing and validating their streams. Cancelled or failed captures remain distinguishable. - Restore temporary save/load dialog state and filenames after errors; never overwrite an earlier recording. -Validation: 34 portable regression tests and native-site checks against Crusader and Extreme. The new starting-save capture and playback path has **not yet passed an end-to-end in-game test**. This is a review/test build, single-player only. Save/load and network commands are unsupported during capture; encountering one marks the capture failed. Replay browsing and settings relaunch follow separately. +Validation: 36 portable regression tests and native-site checks against Crusader and Extreme. The new starting-save capture and playback path has **not yet passed an end-to-end in-game test**. This is a review/test build, single-player only. Save/load and network commands are unsupported during capture; encountering one marks the capture failed. Replay browsing and settings relaunch follow separately. ## 0.2.0 diff --git a/README.md b/README.md index bbdb4ad..b94aa3c 100644 --- a/README.md +++ b/README.md @@ -2,31 +2,32 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.3.0 adds individual replay sessions, starting saves and settings snapshots. **The new save/load playback path still needs end-to-end in-game verification.** Multiplayer is not supported yet. +Version 0.4.0 adds a native replay browser and settings restart to individual sessions, starting saves and settings snapshots. **The new save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer is not supported yet. ## Testing this build Use a separate game installation with UCP3 developer mode and this module enabled. If your game needs Graphics API Replacer, keep it and its dependencies enabled in the test configuration. 1. Open a single-player Skirmish lobby and configure the match. -2. Enable the right-hand recorder checkbox near the bottom. This arms a new recording; clicking it again cancels. +2. Click **Record next match** near the bottom. This arms a new recording; **Cancel recording** cancels it. 3. Start the match. The module captures the starting save and begins recording. 4. Leave through **Quit Mission** to finalize the recording. Do not terminate the process to finish a replay. -5. Return to a Skirmish lobby with the same active UCP settings. The left-hand playback checkbox loads the latest completed session for the current game variant. The saved starting state replaces manual map/AI setup. +5. Return to a Skirmish lobby, open **Replays**, select a completed session and click **Play**. The saved starting state replaces manual map/AI setup. +6. If settings differ, click **Queue settings restart**, close the dialog, and exit the game normally. The helper reopens the same game executable with the recorded UCP configuration. Open the Skirmish replay browser again; the requested recording is selected. -These remain temporary checkbox controls. A labelled replay browser follows in the next stage. Do not delete older recordings to make a new one: every session has a separate folder under `ucp/replays/`. +Do not delete older recordings to make a new one: every session has a separate folder under `ucp/replays/`. A settings restart leaves your normal `ucp-config.yml` intact. It requires the recorded extension versions to be installed; it does not download dependencies. If the helper fails, inspect `ucp/replays/restart-error.txt`. On failure, inspect `ucp3.log` and the session's `last-error.txt` or `desync.json`. A failed or cancelled capture cannot be played. Saving/loading during a recording is currently unsupported and fails the capture explicitly. -See [session format and limitations](docs/replay-sessions.md), [native port notes](docs/native-port.md) and [changelog](CHANGELOG.md). +See [session format and limitations](docs/replay-sessions.md), [browser and restart testing](docs/replay-browser.md), [native port notes](docs/native-port.md) and [changelog](CHANGELOG.md). ## Development ```sh -python -m pip install lupa==2.6 +python -m pip install lupa==2.6 unicorn==2.1.4 python -m unittest discover -s tests -v python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.3.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.4.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/browser.lua b/code/browser.lua new file mode 100644 index 0000000..bbf35b8 --- /dev/null +++ b/code/browser.lua @@ -0,0 +1,65 @@ +local store=require('code/sessions') +local native=require('code/native') +local Browser={PAGE_SIZE=6} + +function Browser:new(recorder) + return setmetatable({recorder=recorder,items={},index=1,message='Choose a recording.'},{__index=self}) +end + +function Browser:refresh(preferred) + local previous=self.items[self.index] + preferred=preferred or (previous and previous.id) or os.getenv('UCP_RECORDER_REPLAY') + self.items={} + for _,item in ipairs(store.list()) do + if item.variant==native.profile.name then self.items[#self.items+1]=item end + end + self.index=1 + for i,item in ipairs(self.items) do if item.id==preferred then self.index=i; break end end + self:select(self.index) +end + +function Browser:select(index) + self.selected=nil + if #self.items==0 then self.index=1; self.message='No recordings yet. Record a Skirmish first.'; return end + self.index=math.max(1,math.min(#self.items,index)) + local item=self.items[self.index] + local ok,manifest=pcall(store.load,item.id,native.profile) + self.selected=ok and manifest or nil + if not ok then self.message=tostring(manifest):match('^[^\n]+'):gsub('^.-:%d+: ','') + elseif store.compatible(manifest) then self.message='Ready to play with your current settings.' + else self.message='Recorded settings differ. Queue a restart, then exit the game.' end +end + +function Browser:page(delta) + self:select(self.index+delta*self.PAGE_SIZE) +end + +function Browser:firstRow() + return math.floor((self.index-1)/self.PAGE_SIZE)*self.PAGE_SIZE+1 +end + +function Browser:row(index) + local item=self.items[index] + if not item then return '' end + local ticks=type(item.lastTick)=='number' and type(item.startTick)=='number' and item.lastTick-item.startTick or 0 + if ticks~=ticks or ticks<0 or ticks>2147483647 then ticks=0 end + ticks=math.floor(ticks) + local state=type(item.status)=='string' and item.status:sub(1,16) or 'unavailable' + return string.format('%s | %d ticks | %s',item.id,ticks,state) +end + +function Browser:play() + assert(self.selected,'Choose a completed recording') + assert(store.compatible(self.selected),'Restart with the recorded settings before playing') + assert(self.recorder.mode=='none','Finish or cancel the active recording first') + return self.recorder:guard(function() self.recorder:startPlayback(self.selected.id) end) +end + +function Browser:restart() + assert(self.selected,'Choose a completed recording') + assert(self.recorder.mode=='none','Finish or cancel the active recording first') + require('code/restart').queue(self.selected.id) + self.message='Restart queued. Exit the game to reopen with recorded settings.' +end + +return Browser diff --git a/code/native-ui.lua b/code/native-ui.lua new file mode 100644 index 0000000..e8d3d07 --- /dev/null +++ b/code/native-ui.lua @@ -0,0 +1,95 @@ +local native=require('code/native') +local profiles=require('code/ui-sites') +local M={ITEM_SIZE=0x50} + +function M.verify() + local sites=assert(profiles[native.profile.name]) + for name,site in pairs(sites) do + local actual=core.readBytes(site.address,#site.bytes) + for i,value in ipairs(site.bytes) do + assert(actual[i]==value,'Recorder UI conflicts at '..name) + end + end + return sites +end + +function M.new(sites,onError) + local o=setmetatable({sites=sites,onError=onError,textBuffer=core.allocate(1024,true)},{__index=M}) + o.textNative=core.exposeCode(sites.text.address,9,1) + o.borderNative=core.exposeCode(sites.border.address,6,1) + o.menuConstructor=core.exposeCode(sites.menuConstructor.address,2,1) + o.modalConstructor=core.exposeCode(sites.modalConstructor.address,10,1) + o.activateNative=core.exposeCode(sites.activateModal.address,3,1) + return o +end + +function M:callback(callback) + return utils.createLuaFunctionWrapper(function(registers) + local ok,reason=xpcall(function() callback(registers) end,debug.traceback) + if not ok then self.onError(reason) end + return registers + end) +end + +function M:text(label,x,y,alignment) + label=tostring(label):gsub('[\r\n%z]',' '):sub(1,150) + core.writeString(self.textBuffer,label) + local color=core.readSmallInteger(self.sites.gold.value)%65536 + self.textNative(self.sites.textManager.value,self.textBuffer,x,y,alignment or 0,color,0,0,0) +end + +function M:button(address,x,y,width,height,label,action,selected) + for offset=0,self.ITEM_SIZE-4,4 do core.writeInteger(address+offset,0) end + core.writeInteger(address,3) + core.writeInteger(address+4,x); core.writeInteger(address+8,y) + core.writeInteger(address+12,width); core.writeInteger(address+16,height) + core.writeInteger(address+20,self:callback(action)) + core.writeInteger(address+28,self:callback(function() + local state=self.sites.buttonState.value + local drawX,drawY=core.readInteger(state),core.readInteger(state+4) + local text=type(label)=='function' and label() or label + if text=='' then return end + local color=core.readSmallInteger(self.sites.gold.value)%65536 + local hover=core.readInteger(state+16)~=0 + if hover or (selected and selected()) then + self.borderNative(self.sites.pencil.value,drawX,drawY,drawX+width-1,drawY+height-1,color) + end + self:text(text,drawX+8,drawY+math.floor((height-12)/2)) + end)) + core.writeInteger(address+36,1) -- SIMPLE_RENDER, native button coordinates + core.writeSmallInteger(address+48,0xfff0) -- no user-control lookup +end + +function M:modal(items,count,width,height,render) + assert(count==#items,'Replay dialog item count differs') + local menu=core.allocate(0x44,true) + local array=core.allocate((count+1)*self.ITEM_SIZE,true) + for i,item in ipairs(items) do + local address=array+(i-1)*self.ITEM_SIZE + self:button(address,item.x,item.y,item.width,item.height,item.label,item.action,item.selected) + core.writeInteger(address+0x4c,menu) + end + core.writeInteger(array+count*self.ITEM_SIZE,0x66) + self.menuConstructor(menu,array) + local used={} + local pointer=core.readInteger(self.sites.modalStack.value) + local seen={}; local entries=0 + while pointer~=0 and pointer~=-1 and pointer~=0xffffffff do + entries=entries+1; assert(entries<1024,'Native modal list is too long') + assert(not seen[pointer],'Cyclic native modal list'); seen[pointer]=true + used[core.readInteger(pointer)]=true + pointer=core.readInteger(pointer+0x24) + end + local id=300; while used[id] do id=id+1; assert(id<1300,'No replay dialog slot') end + local dialog=core.allocate(40,true) + local callback=self:callback(function(registers) + render(core.readInteger(registers.ESP+4),core.readInteger(registers.ESP+8)) + end) + -- Native red double frame; centered coordinates and the game's own backdrop. + self.modalConstructor(dialog,id,-1,-1,width,height,512,0,callback,menu) + return id +end + +function M:show(id) self.activateNative(self.sites.modalComposition.value,id,0) end +function M:close() self:show(-1) end +return M diff --git a/code/platform.lua b/code/platform.lua index 42e2a36..29f255c 100644 --- a/code/platform.lua +++ b/code/platform.lua @@ -21,13 +21,17 @@ function M.stdcall(library, name, count) return core.exposeCode(core.allocateCode(code), count, 0) end -local mkdir, getError +local mkdir, getAttributes function M.mkdir(path) mkdir = mkdir or M.stdcall('kernel32.dll', 'CreateDirectoryA', 2) - getError = getError or M.stdcall('kernel32.dll','GetLastError',0) - if mkdir(buffer('directory',path),0)~=0 then return true end - local reason=getError() - assert(reason==183,'Cannot create replay directory (Windows error '..reason..')') + getAttributes = getAttributes or M.stdcall('kernel32.dll','GetFileAttributesA',1) + local address=buffer('directory',path) + if mkdir(address,0)~=0 then return true end + -- Do not carry GetLastError across Lua/RPS calls: intervening runtime work may + -- change it. An existing directory is the only non-error false result. + local attributes=getAttributes(address) + assert(attributes>=0 and attributes~=0xffffffff and math.floor(attributes/16)%2==1, + 'Cannot create replay directory: '..path) return false end @@ -38,4 +42,35 @@ function M.replace(source, destination) assert(move(a, b, 9) ~= 0, 'Cannot finish writing replay metadata') end +function M.identity() + local getModule=M.stdcall('kernel32.dll','GetModuleFileNameW',3) + local getPID=M.stdcall('kernel32.dll','GetCurrentProcessId',0) + local toUTF8=M.stdcall('kernel32.dll','WideCharToMultiByte',8) + local wide=core.allocate(8192,true) + local count=getModule(0,wide,4096) + assert(count>0 and count<4096,'Cannot identify the running game') + local path=buffer('executable','') + local bytes=toUTF8(65001,0,wide,count,path,4096,0,0) + assert(bytes>0,'Cannot encode the game executable path') + return {executable=core.readString(path,bytes),processId=getPID()} +end + +function M.spawnHidden(executable,commandLine) + local create=M.stdcall('kernel32.dll','CreateProcessW',10) + local toWide=M.stdcall('kernel32.dll','MultiByteToWideChar',6) + local close=M.stdcall('kernel32.dll','CloseHandle',1) + local function wide(key,text) + local utf8=buffer(key,text) + local result=core.allocate(8192,true) + assert(toWide(65001,8,utf8,-1,result,4096)>0,'Cannot encode restart command') + return result + end + local startup,process=core.allocate(68,true),core.allocate(16,true) + core.writeInteger(startup,68) + local ok=create(wide('helper-executable',executable),wide('helper-command',commandLine), + 0,0,0,0x08000000,0,0,startup,process) + assert(ok~=0,'Cannot start the replay restart helper') + close(core.readInteger(process)); close(core.readInteger(process+4)) +end + return M diff --git a/code/restart-script.lua b/code/restart-script.lua new file mode 100644 index 0000000..9d4178f --- /dev/null +++ b/code/restart-script.lua @@ -0,0 +1,33 @@ +-- A fixed helper program: replay metadata is read as JSON, never evaluated as code. +-- It waits for the current game to exit so two games do not compete for the renderer. +return [=[ +$ErrorActionPreference = 'Stop' +$requestPath = Join-Path (Get-Location).Path 'ucp/replays/restart-request.json' +try { + $request = Get-Content -LiteralPath $requestPath -Raw -Encoding UTF8 | ConvertFrom-Json + $gameRoot = [IO.Path]::GetFullPath((Get-Location).Path) + if ($request.id -notmatch '^[A-Za-z0-9_-]{1,79}$') { throw 'Invalid replay identifier' } + $sessionPath = Join-Path $gameRoot ('ucp/replays/' + $request.id) + $settingsPath = Join-Path $sessionPath 'ucp-config.yml' + $manifestPath = Join-Path $sessionPath 'manifest.json' + $manifest = Get-Content -LiteralPath $manifestPath -Raw -Encoding UTF8 | ConvertFrom-Json + if ($manifest.id -ne $request.id -or $manifest.status -ne 'complete') { throw 'Replay is incomplete' } + $executable = [IO.Path]::GetFullPath($request.executable) + if ([IO.Path]::GetDirectoryName($executable) -ne $gameRoot) { throw 'Game directory differs' } + if ((Get-FileHash -LiteralPath $executable -Algorithm SHA256).Hash -ne $manifest.executable) { throw 'Game executable differs' } + $gameProcess = Get-Process -Id $request.processId -ErrorAction SilentlyContinue + if ($null -ne $gameProcess) { + if ($gameProcess.Path -ne $executable) { throw 'Process identity differs' } + $gameProcess.WaitForExit() + } + if ((Get-FileHash -LiteralPath $executable -Algorithm SHA256).Hash -ne $manifest.executable) { throw 'Game executable changed while waiting' } + if ((Get-FileHash -LiteralPath $settingsPath -Algorithm SHA256).Hash -ne $manifest.settingsHash) { throw 'Recorded settings are damaged' } + $env:UCP_RECORDER_REPLAY = $request.id + # A Windows file path cannot contain a quote. This one ends in .yml, not a backslash. + $arguments = '--ucp-config-file="' + $settingsPath + '"' + Start-Process -FilePath $executable -WorkingDirectory $gameRoot -ArgumentList $arguments +} catch { + $_ | Out-String | Set-Content -LiteralPath (Join-Path (Get-Location).Path 'ucp/replays/restart-error.txt') + exit 1 +} +]=] diff --git a/code/restart.lua b/code/restart.lua new file mode 100644 index 0000000..2f00c2f --- /dev/null +++ b/code/restart.lua @@ -0,0 +1,30 @@ +local platform=require('code/platform') +local store=require('code/sessions') +local native=require('code/native') +local M={queued=false} + +-- Windows CRT argument quoting; no shell interprets this command line. +function M.quote(value) + assert(type(value)=='string' and not value:find('[%z\r\n]'),'Invalid restart argument') + value=value:gsub('(\\*)"',function(slashes) return slashes..slashes..'\\"' end) + value=value:gsub('(\\+)$',function(slashes) return slashes..slashes end) + return '"'..value..'"' +end + +function M.queue(id) + assert(not M.queued,'A restart is already waiting for the game to close') + local manifest=store.load(id,native.profile) + store.preflight(manifest) + local identity=platform.identity() + store.write(store.ROOT..'/restart-helper.ps1',require('code/restart-script')) + store.write(store.ROOT..'/restart-request.json',json:encode({ + id=manifest.id,executable=identity.executable,processId=identity.processId})) + local powershell=assert(os.getenv('SystemRoot'),'Windows system directory is unavailable').. + '\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' + local args=M.quote(powershell)..' -NoProfile -NonInteractive -WindowStyle Hidden -ExecutionPolicy Bypass -File '.. + M.quote(store.ROOT..'/restart-helper.ps1') + platform.spawnHidden(powershell,args) + M.queued=true +end + +return M diff --git a/code/ui-sites.lua b/code/ui-sites.lua new file mode 100644 index 0000000..76c13d8 --- /dev/null +++ b/code/ui-sites.lua @@ -0,0 +1,30 @@ +-- Native UI entry points/pointer operands checked against both original executables. +-- Signatures and ABI: UCP ui module ui/game.lua and ui/headers/latest/ui.h. +return { + SHC = { + menuConstructor = {address=0x004F4100, bytes={81,83,139,217}}, + modalConstructor = {address=0x004A9E00, bytes={139,84,36,8,139,193,139,76,36,4,137,8}}, + activateModal = {address=0x004A9ED0, bytes={83,85,51,237,57,108,36,16}}, + text = {address=0x00474250, bytes={131,124,36,28,0,83,86,139,241,117,6,199,6,0,0,0,0,139,92,36,12,133,219,116,124}}, + border = {address=0x004711B0, bytes={86,139,241,232,152,122,255,255,139,68,36,24,139,76,36,20,139,84,36,16,80,139,68,36,16,81,139,76,36,16,82,80,81,139,206,232,168,122,255,255,133,192,116,54}}, + buttonState = {address=0x00428615, bytes={139,61,168,49,237,0,141,68,56,101}, value=0x00ED31A8}, + textManager = {address=0x004337DC, bytes={137,21,120,117,21,2,126,5}, value=0x02157578}, + pencil = {address=0x004090B9, bytes={185,32,215,145,1,232,205,125,6,0,57,126,216}, value=0x0191D720}, + gold = {address=0x004679AA, bytes={102,163,216,51,223,0,232,155,254,255,255,106,88}, value=0x00DF33D8}, + modalComposition = {address=0x0042544C, bytes={185,144,124,254,1,232,122,74,8,0,94,91,233,147,114,9,0}, value=0x01FE7C90}, + modalStack = {address=0x004A9E40, bytes={139,21,164,66,223,0,137,80,36}, value=0x00DF42A4}, + }, + Extreme = { + menuConstructor = {address=0x004F4490, bytes={81,83,139,217}}, + modalConstructor = {address=0x004A9F70, bytes={139,84,36,8,139,193,139,76,36,4,137,8}}, + activateModal = {address=0x004AA040, bytes={83,85,51,237,57,108,36,16}}, + text = {address=0x00474480, bytes={131,124,36,28,0,83,86,139,241,117,6,199,6,0,0,0,0,139,92,36,12,133,219,116,124}}, + border = {address=0x004713D0, bytes={86,139,241,232,168,122,255,255,139,68,36,24,139,76,36,20,139,84,36,16,80,139,68,36,16,81,139,76,36,16,82,80,81,139,206,232,184,122,255,255,133,192,116,54}}, + buttonState = {address=0x00428645, bytes={139,61,40,54,237,0,141,68,56,101}, value=0x00ED3628}, + textManager = {address=0x00433A1C, bytes={137,21,120,170,190,2,126,5}, value=0x02BEAA78}, + pencil = {address=0x004090C9, bytes={185,144,71,53,2,232,221,127,6,0,57,126,216}, value=0x02354790}, + gold = {address=0x00467BDA, bytes={102,163,120,53,223,0,232,155,254,255,255,106,88}, value=0x00DF3578}, + modalComposition = {address=0x004254FC, bytes={185,144,177,167,2,232,58,75,8,0,94,91,233,83,115,9,0}, value=0x02A7B190}, + modalStack = {address=0x004A9FB0, bytes={139,21,60,67,223,0,137,80,36}, value=0x00DF433C}, + }, +} diff --git a/code/ui.lua b/code/ui.lua index dc1b7b5..14d2b7b 100644 --- a/code/ui.lua +++ b/code/ui.lua @@ -1,111 +1,62 @@ -local native = require("code/native") -local menuItemSize = 0x50 - -local originalMenuItemArraySize = 0x1D10 -local originalMenuItemArrayAddress = native.addr(0x005e9848) - -local newSize = originalMenuItemArraySize + 2 * menuItemSize -- 2 additional MenuItems -local newMenuItemArrayAddress = core.allocate(newSize, true) - -local newItem1Address = newMenuItemArrayAddress + newSize - 3 * menuItemSize -local newItem2Address = newMenuItemArrayAddress + newSize - 2 * menuItemSize -local newTerminatorAddress = newMenuItemArrayAddress + newSize - 1 * menuItemSize - -local placeRecordButton = function(recorder) - -- init button - - local recordButtonCallback = function() - if recorder.mode == "none" then - recorder:startRecording() - core.writeInteger(newItem1Address + 32, 0x40000239) -- buttonGraphic - print("Starting new recording!") - elseif recorder.mode == "record" then - recorder:stopRecording() - recorder:discardFiles() - core.writeInteger(newItem1Address + 32, 0x4000023A) -- buttonGraphic - print("discarded") - end - end - - core.writeInteger(newItem1Address + 0, 0x00000003) -- type - core.writeInteger(newItem1Address + 4, 380) -- xPos - core.writeInteger(newItem1Address + 8, 560) -- yPos - core.writeInteger(newItem1Address + 12, 80) -- width - core.writeInteger(newItem1Address + 16, 80) -- height - core.writeInteger(newItem1Address + 20, utils.createLuaFunctionWrapper(function(registers) - recorder:guard(recordButtonCallback) - return registers - end)) - core.writeInteger(newItem1Address + 24, 2) -- callbackParam - core.writeInteger(newItem1Address + 28, native.addr(0x0042AE90)) -- renderFunc - core.writeInteger(newItem1Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on - core.writeInteger(newItem1Address + 36, 0x00000003) -- unknown - core.writeInteger(newItem1Address + 40, 0x00000000) -- unknown - core.writeInteger(newItem1Address + 44, 0x00000000) -- unknown - core.writeInteger(newItem1Address + 48, 0x0000FFF0) -- unknown - - core.writeInteger(newItem1Address + 72, 0x00000045) -- unknown +local native=require('code/native') +local NativeUI=require('code/native-ui') +local Browser=require('code/browser') +local M={} + +function M.createButtons(recorder,sites) + local browser=Browser:new(recorder) + local ui=NativeUI.new(sites,function(reason) + browser.message=tostring(reason):match('^[^\n]+'):gsub('^.-:%d+: ','') + print('Replay menu: '..tostring(reason)) + end) + M.browser=browser + local items={} + local function button(x,y,width,label,action,selected) + items[#items+1]={x=x,y=y,width=width,height=30,label=label,action=action,selected=selected} + end + for row=0,Browser.PAGE_SIZE-1 do + local offset=row + button(24,76+row*36,632,function() return browser:row(browser:firstRow()+offset) end, + function() browser:select(browser:firstRow()+offset) end, + function() return browser.index==browser:firstRow()+offset end) + end + button(24,376,86,'Previous',function() browser:page(-1) end) + button(114,376,62,'Next',function() browser:page(1) end) + button(180,376,80,'Refresh',function() browser:refresh() end) + button(274,376,66,'Close',function() ui:close() end) + button(364,376,90,'Play',function() + if browser:play() then ui:close() + elseif recorder.error then browser.message=recorder.error:match('^[^\n]+'):gsub('^.-:%d+: ','') end + end) + button(466,376,190,'Queue settings restart',function() browser:restart() end) + local dialog=ui:modal(items,#items,680,440,function(x,y) + ui:text('Recorded Skirmishes',x+24,y+22) + ui:text(native.profile.name..' | '..#browser.items..' recordings',x+24,y+46) + local item=browser.items[browser.index] + ui:text(item and ('Selected: '..item.id) or 'Record your next match from the Skirmish lobby.',x+24,y+310) + ui:text(browser.message,x+24,y+336) + end) + + -- Extend only the original Skirmish menu's item array, retaining its sentinel. + local originalSize,itemSize=0x1d10,NativeUI.ITEM_SIZE + local array=core.allocate(originalSize+2*itemSize,true) + core.copyMemory(array,native.addr(0x5e9848),originalSize) + local record=array+originalSize-itemSize + local browse=record+itemSize + core.copyMemory(browse+itemSize,record,itemSize) + ui:button(record,250,550,145,30,function() + return recorder.status=='armed' and 'Cancel recording' or 'Record next match' + end,function() + if recorder.mode=='none' then recorder:guard(function() recorder:startRecording() end) + elseif recorder.mode=='record' then recorder:guard(function() recorder:stopRecording() end) end + end,function() return recorder.status=='armed' end) + ui:button(browse,410,550,145,30,'Replays',function() + browser:refresh(); ui:show(dialog) + end) + core.writeCode(native.addr(0x59ab30),{ + core.AssemblyLambda('push array',{array=array}) + }) end -local placePlaybackButton = function(recorder) - -- init button - - local playbackButtonCallback = function() - print("playback button") - if recorder.mode == "none" then - recorder:startPlayback() - core.writeInteger(newItem2Address + 32, 0x40000239) -- buttonGraphic - print("Starting new playback!") - elseif recorder.mode == "play" then - recorder:stopPlayback() - core.writeInteger(newItem2Address + 32, 0x4000023A) -- buttonGraphic - print("discarded playback") - end - end - - core.writeInteger(newItem2Address + 0, 0x00000003) -- type - core.writeInteger(newItem2Address + 4, 290) -- xPos - core.writeInteger(newItem2Address + 8, 560) -- yPos - core.writeInteger(newItem2Address + 12, 80) -- width - core.writeInteger(newItem2Address + 16, 80) -- height - core.writeInteger(newItem2Address + 20, utils.createLuaFunctionWrapper(function(registers) - recorder:guard(playbackButtonCallback) - return registers - end)) - core.writeInteger(newItem2Address + 24, 2) -- callbackParam - core.writeInteger(newItem2Address + 28, native.addr(0x0042AE90)) -- renderFunc - core.writeInteger(newItem2Address + 32, 0x4000023A) -- unkown | buttonGraphic 23A off 239 on - core.writeInteger(newItem2Address + 36, 0x00000003) -- unknown - core.writeInteger(newItem2Address + 40, 0x00000000) -- unknown - core.writeInteger(newItem2Address + 44, 0x00000000) -- unknown - core.writeInteger(newItem2Address + 48, 0x0000FFF0) -- unknown - - core.writeInteger(newItem2Address + 72, 0x00000045) -- unknown -end - -return { - createButtons = function(recorder) - -- copy original ui array to new array - core.copyMemory(newMenuItemArrayAddress, originalMenuItemArrayAddress, originalMenuItemArraySize) - - -- copy 0x66 terminator one element further back - core.copyMemory(newTerminatorAddress, - newItem1Address, - menuItemSize) - - placeRecordButton(recorder) - placePlaybackButton(recorder) - - -- overwrite reference - core.writeCode(native.addr(0x0059AB30), { core.AssemblyLambda([[ - push newMenuItemArrayAddress - ]], - { newMenuItemArrayAddress = newMenuItemArrayAddress, }) }) - - end, - - resetButtons = function() - core.writeInteger(newItem1Address + 32, 0x4000023A) -- reset recording button graphic - core.writeInteger(newItem2Address + 32, 0x4000023A) -- reset playback button graphic - end, -} +function M.resetButtons() end -- Labels derive from session state at render time. +return M diff --git a/definition.yml b/definition.yml index 74a8e3b..ea743d7 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.3.0 +version: 0.4.0 type: module author: Corax meta: diff --git a/docs/replay-browser.md b/docs/replay-browser.md new file mode 100644 index 0000000..e449ecd --- /dev/null +++ b/docs/replay-browser.md @@ -0,0 +1,26 @@ +# Replay browser and settings restart (0.4.0) + +The browser is a native modal dialog opened from the single-player Skirmish lobby. It uses MenuItem, Menu and MenuModal layouts documented by the UCP UI module and OpenSHC, and the original game's red frame, font, colours and button input. No new graphics library, CFFI module or external bitmap is required. + +`ui-sites.lua` records the exact entry bytes and pointer operands used for Crusader and Extreme. Verification runs before this module installs any hooks. The dialog finds an unused modal ID rather than taking a known menu's ID. Arrays allocate an extra sentinel entry; each button points to its owning Menu. + +`browser.lua` handles selection, paging and compatibility independently of rendering. The list includes failed/cancelled sessions so the user can distinguish them from completed captures; these cannot be played. Refresh preserves selection, and an empty list clears stale selection. Native rendering and input still require a live visual test at multiple resolutions in both games. + +## Restart handoff + +**Queue settings restart** validates the selected session and starts a hidden Windows PowerShell helper. It does not terminate the game. The user exits normally; the helper waits for that process and then starts the same executable with `--ucp-config-file=".../ucp-config.yml"`. The recorded file is passed directly to UCP rather than copied over the user's normal configuration. + +The helper verifies the executable hash, process identity and settings hash. A changed file prevents launch and produces `ucp/replays/restart-error.txt`. It passes `UCP_RECORDER_REPLAY` only through the child environment, allowing the browser to select the requested recording after restart. The user enters Skirmish and clicks Play; playback is not injected into an uncertain startup stage. + +The helper runs with the current user's rights. JSON data is never evaluated as PowerShell, game arguments are quoted, process handles are closed, and Win32 process/path calls use Unicode. No dependency installation or download occurs. The recorded module versions must be available already; the next game launch still performs normal UCP loading and compatibility checks. + +Tests execute the actual helper with `Get-Process` and `Start-Process` replaced by fakes, while keeping its real JSON, path and hash checks. They start no game. Real process handoff and graphics initialization after restart remain pending integration tests. + +## Live test checklist + +1. Verify labels, modal placement, row text, selection, empty/error lists and all buttons at minimum and normal display resolutions in both variants. +2. Record a short match, change speed, pause/resume, issue several commands, quit the mission and replay it through completion. +3. Replay a second time and choose an older recording; verify that no command/RNG cache survives. +4. Change UCP settings, queue a restart, exit normally and verify the matching recording loads with the stored settings. Check that the default configuration bytes are unchanged. +5. Repeat with Graphics API Replacer enabled and inspect errors without assuming a process launch proves renderer success. +6. Exercise in-game quit/restart/load, save/autosave rejection and attempted spectator actions; inspect completion status and the first divergence report. diff --git a/init.lua b/init.lua index 953528d..daceab6 100644 --- a/init.lua +++ b/init.lua @@ -6,6 +6,7 @@ local module={} function module:enable(config) native.verify() local sites=Engine.verify() + local uiSites=require('code/native-ui').verify() local seed if config.useFixedSeed then seed=require('code/validation').integer(config.fixedSeed,-2147483648,2147483647,'fixed seed') @@ -17,7 +18,7 @@ function module:enable(config) self.recorder=recorder engine:install(recorder) local ui=require('code/ui') - ui.createButtons(recorder) + ui.createButtons(recorder,uiSites) local function observe(address,size,callback) core.detourCode(function(registers) diff --git a/tests/check_executables.py b/tests/check_executables.py index 2fef2c3..c8fac3c 100644 --- a/tests/check_executables.py +++ b/tests/check_executables.py @@ -43,11 +43,12 @@ def check(folder): menu = native.addr(0x59ab30) assert reader(menu, 1) == b'\x68' assert struct.unpack('=range[1] and address+size<=range[2] then return end end + error('native UI allocation overrun') +end +local writeInteger,writeSmall=core.writeInteger,core.writeSmallInteger +core.writeInteger=function(a,v) checked(a,4); writeInteger(a,v) end +core.writeSmallInteger=function(a,v) checked(a,2); writeSmall(a,v) end +core.writeString=function() end +ui=NativeUI.new(sites,function(reason) error(reason) end) +''') + + def test_modal_items_fit_allocation_and_keep_sentinel(self): + self.check(''' +local menuAddress,arrayAddress +ui.menuConstructor=function(menu,array) menuAddress=menu; arrayAddress=array end +ui.modalConstructor=function(dialog,id,x,y,w,h,style,color,render,menu) + assert(id==300 and x==-1 and y==-1 and style==512 and menu==menuAddress) +end +local items={} +for i=1,12 do items[i]={x=20,y=i*30,width=200,height=30,label='item',action=function() end} end +ui:modal(items,#items,680,440,function() end) +assert(memory[arrayAddress+12*80]==0x66) +for i=0,11 do assert(memory[arrayAddress+i*80+0x4c]==menuAddress) end +''') + + def test_modal_id_collision_is_avoided(self): + self.check(''' +memory[sites.modalStack.value]=0x1234; memory[0x1234]=300; memory[0x1234+0x24]=0 +ui.modalConstructor=function(_,id) assert(id==301) end +assert(ui:modal({},0,680,440,function() end)==301) +''') + + def test_cyclic_modal_list_fails_instead_of_hanging(self): + self.check(''' +memory[sites.modalStack.value]=0x1234; memory[0x1234+0x24]=0x1234 +assert(not pcall(function() ui:modal({},0,680,440,function() end) end)) +''') + + def test_menu_text_sanitizes_controls_and_bounds_native_buffer(self): + self.check(r''' +local text +core.writeString=function(_,value) text=value end +ui:text('a\nb\0c'..string.rep('x',200),20,30) +assert(#text==150 and text:sub(1,5)=='a b c') +''') diff --git a/tests/test_recorder.py b/tests/test_recorder.py index 0776479..c58aace 100644 --- a/tests/test_recorder.py +++ b/tests/test_recorder.py @@ -136,6 +136,7 @@ def test_real_init_callbacks_return_register_changes(self): for _,site in pairs(sites) do if type(site)=='table' then core.writeBytes(site.address,site.bytes) end end +for _,site in pairs(require('code/ui-sites').SHC) do core.writeBytes(site.address,site.bytes) end core.hookCode=function() return function() return 0 end end core.writeString=function() end core.callTo=function() return {} end diff --git a/tests/test_restart.py b/tests/test_restart.py new file mode 100644 index 0000000..e0ac727 --- /dev/null +++ b/tests/test_restart.py @@ -0,0 +1,52 @@ +import subprocess +import unittest + +import test_recorder as fixture + + +class RestartTests(unittest.TestCase): + check = fixture.RecorderTests.check + + def setUp(self): + fixture.RecorderTests.setUp(self) + self.check(''' +json.encode=function(_,value) request=value; return 'request JSON' end +written={}; spawned=0; preflightOK=true +package.loaded['code/sessions']={ROOT='ucp/replays', + load=function(id) return {id=id} end, + preflight=function() assert(preflightOK,'damaged recording') end, + write=function(path,data) written[path]=data end, +} +package.loaded['code/platform']={ + identity=function() return {executable='C:/game/Crusader.exe',processId=123} end, + spawnHidden=function(exe,args) spawned=spawned+1; executable=exe; arguments=args end, +} +os.getenv=function(key) if key=='SystemRoot' then return 'C:\\\\Windows' end end +restart=require('code/restart') +''') + + def test_restart_only_queues_hidden_helper_and_preserves_settings(self): + self.check(''' +restart.queue('recording1') +assert(spawned==1 and request.id=='recording1' and request.processId==123) +assert(arguments:find('-WindowStyle Hidden',1,true)) +assert(written['ucp/replays/restart-helper.ps1'] and written['ucp/replays/restart-request.json']) +assert(not written['ucp-config.yml']) +assert(not pcall(restart.queue,'recording2') and spawned==1) +''') + + def test_corrupt_session_never_starts_helper(self): + self.check(''' +preflightOK=false +assert(not pcall(restart.queue,'recording1')); assert(spawned==0 and next(written)==nil) +''') + + def test_windows_quoting_matches_standard_library(self): + quote = self.lua.eval('restart.quote') + for value in ['C:\\Program Files\\game.exe', 'space and trailing slash\\', 'embedded " quote', 'two\\\\" quotes']: + with self.subTest(value=value): + self.assertEqual(quote(value), subprocess.list2cmdline([value])) + for value in ['bad\nline', 'bad\0path']: + with self.subTest(value=value): + ok, _ = self.lua.eval('function(v) return pcall(restart.quote,v) end')(value) + self.assertFalse(ok) diff --git a/tests/test_restart_helper.py b/tests/test_restart_helper.py new file mode 100644 index 0000000..50f3a65 --- /dev/null +++ b/tests/test_restart_helper.py @@ -0,0 +1,98 @@ +"""Run the actual PowerShell helper with process waiting/launching replaced by fakes. + +No game or other application is started. The helper still performs real path, +JSON and file-hash checks inside a temporary directory. +""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +from lupa.luajit21 import LuaRuntime + +ROOT = Path(__file__).resolve().parents[1] + + +@unittest.skipUnless(os.name == 'nt', 'Windows PowerShell helper') +class RestartHelperTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='replay helper ') + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.folder = self.root/'ucp/replays' + self.session = self.folder/'test-recording' + self.session.mkdir(parents=True) + self.exe = self.root/'Crusader test.exe' + self.exe.write_bytes(b'test executable; never launched') + self.settings = self.session/'ucp-config.yml' + self.settings.write_bytes(b'active: true\n') + self.manifest = {'id': 'test-recording', 'status': 'complete', + 'executable': hashlib.sha256(self.exe.read_bytes()).hexdigest(), + 'settingsHash': hashlib.sha256(self.settings.read_bytes()).hexdigest()} + (self.session/'manifest.json').write_text(json.dumps(self.manifest)) + (self.folder/'restart-request.json').write_text(json.dumps({ + 'id': 'test-recording', 'executable': str(self.exe), 'processId': 123})) + script = LuaRuntime().execute((ROOT/'code/restart-script.lua').read_text()) + (self.root/'helper.ps1').write_text(script, encoding='utf-8-sig') + (self.root/'runner.ps1').write_text(''' +$ErrorActionPreference = 'Stop' +function Get-Process { + param($Id,$ErrorAction) + $process = [pscustomobject]@{ Path = (Join-Path (Get-Location).Path 'Crusader test.exe') } + $process | Add-Member -MemberType ScriptMethod -Name WaitForExit -Value { + Set-Content -LiteralPath 'waited.txt' -Value 'waited' + if (Test-Path -LiteralPath 'swap-executable.txt') { + Set-Content -LiteralPath 'Crusader test.exe' -Value 'updated executable' + } + } + return $process +} +function Start-Process { + param($FilePath,$WorkingDirectory,$ArgumentList) + if (-not (Test-Path -LiteralPath 'waited.txt')) { throw 'Launch occurred before waiting' } + @{executable=$FilePath; cwd=$WorkingDirectory; arguments=$ArgumentList; replay=$env:UCP_RECORDER_REPLAY} | + ConvertTo-Json | Set-Content -LiteralPath 'launch.json' -Encoding UTF8 +} +& './helper.ps1' +''', encoding='utf-8-sig') + + def run_helper(self): + powershell = Path(os.environ['SystemRoot'])/'System32/WindowsPowerShell/v1.0/powershell.exe' + return subprocess.run([str(powershell), '-NoProfile', '-NonInteractive', + '-ExecutionPolicy', 'Bypass', '-File', str(self.root/'runner.ps1')], + cwd=self.root, capture_output=True, text=True, timeout=20, + creationflags=subprocess.CREATE_NO_WINDOW) + + def test_waits_then_requests_recorded_settings_without_overwriting_default(self): + normal = self.root/'ucp-config.yml' + normal.write_text('normal settings') + result = self.run_helper() + self.assertEqual(result.returncode, 0, result.stderr) + launch = json.loads((self.root/'launch.json').read_text(encoding='utf-8-sig')) + self.assertEqual(launch['executable'], str(self.exe)) + self.assertEqual(launch['arguments'], f'--ucp-config-file="{self.settings}"') + self.assertEqual(launch['replay'], 'test-recording') + self.assertEqual(normal.read_text(), 'normal settings') + + def test_settings_modified_while_waiting_do_not_launch(self): + self.settings.write_text('modified after recording') + result = self.run_helper() + self.assertFalse((self.root/'launch.json').exists()) + self.assertTrue((self.folder/'restart-error.txt').exists(), result.stderr) + + def test_wrong_executable_is_rejected_before_waiting(self): + self.exe.write_text('a different executable') + result = self.run_helper() + self.assertFalse((self.root/'launch.json').exists()) + self.assertFalse((self.root/'waited.txt').exists()) + self.assertTrue((self.folder/'restart-error.txt').exists(), result.stderr) + + def test_executable_changed_while_waiting_is_checked_again(self): + (self.root/'swap-executable.txt').touch() + result = self.run_helper() + self.assertTrue((self.root/'waited.txt').exists(), result.stderr) + self.assertFalse((self.root/'launch.json').exists()) + self.assertTrue((self.folder/'restart-error.txt').exists(), result.stderr) diff --git a/tests/test_win32_abi.py b/tests/test_win32_abi.py new file mode 100644 index 0000000..3a0666f --- /dev/null +++ b/tests/test_win32_abi.py @@ -0,0 +1,61 @@ +"""Emulate the generated 32-bit stdcall bridge; never call Windows or game code.""" +from pathlib import Path +import struct +import unittest + +from lupa.luajit21 import LuaRuntime + +try: + from unicorn import Uc, UC_ARCH_X86, UC_MODE_32, UC_HOOK_CODE + from unicorn.x86_const import UC_X86_REG_ESP, UC_X86_REG_EBP, UC_X86_REG_EAX +except ImportError: + Uc = None + +ROOT = Path(__file__).resolve().parents[1] + + +@unittest.skipIf(Uc is None, 'Install unicorn==2.1.4 for 32-bit ABI emulation') +class Win32ABITests(unittest.TestCase): + def test_generated_bridges_preserve_argument_order_and_caller_stack(self): + for count in (0, 1, 2, 3, 6, 8, 10): + with self.subTest(arguments=count): + lua = LuaRuntime() + lua.execute(''' +core={ + openLibraryHandle=function() return {getProcAddress=function() return 0x102000 end} end, + callTo=function(address) return {target=address} end, + allocateCode=function(code) captured=code; return 0x100000 end, + exposeCode=function(address,count,convention) return {address,count,convention} end, +} +''') + platform = lua.execute((ROOT/'code/platform.lua').read_text()) + bridge = platform.stdcall('fake.dll', 'fake', count) + self.assertEqual(list(bridge.values()), [0x100000, count, 0]) + code = bytearray() + for instruction in lua.globals().captured.values(): + if isinstance(instruction, int): + code.append(instruction) + else: + code += b'\xe8'+struct.pack(' Date: Sat, 5 Sep 2026 13:54:22 +0200 Subject: [PATCH 06/19] Surface restart-helper diagnostics in Windows CI --- tests/test_restart_helper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_restart_helper.py b/tests/test_restart_helper.py index 50f3a65..316156b 100644 --- a/tests/test_restart_helper.py +++ b/tests/test_restart_helper.py @@ -57,14 +57,19 @@ def setUp(self): ConvertTo-Json | Set-Content -LiteralPath 'launch.json' -Encoding UTF8 } & './helper.ps1' +exit $LASTEXITCODE ''', encoding='utf-8-sig') def run_helper(self): powershell = Path(os.environ['SystemRoot'])/'System32/WindowsPowerShell/v1.0/powershell.exe' - return subprocess.run([str(powershell), '-NoProfile', '-NonInteractive', + result = subprocess.run([str(powershell), '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', str(self.root/'runner.ps1')], cwd=self.root, capture_output=True, text=True, timeout=20, creationflags=subprocess.CREATE_NO_WINDOW) + error = self.folder/'restart-error.txt' + if error.exists(): + result.stderr += error.read_text(encoding='utf-8', errors='replace') + return result def test_waits_then_requests_recorded_settings_without_overwriting_default(self): normal = self.root/'ucp-config.yml' From e5cafeca6df7a34f3f872bf8c64a78fcf3f81eca Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:55:57 +0200 Subject: [PATCH 07/19] Hash replay files without depending on PowerShell module discovery --- code/restart-script.lua | 20 +++++++++++++++++--- tests/test_restart_helper.py | 1 + 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/code/restart-script.lua b/code/restart-script.lua index 9d4178f..c155432 100644 --- a/code/restart-script.lua +++ b/code/restart-script.lua @@ -2,6 +2,20 @@ -- It waits for the current game to exit so two games do not compete for the renderer. return [=[ $ErrorActionPreference = 'Stop' +function Get-ReplayHash { + param([string]$Path) + # Get-FileHash is a script module and may be absent from an inherited + # PowerShell 7 module path when this Windows PowerShell 5 helper starts. + $algorithm = [Security.Cryptography.SHA256]::Create() + $stream = $null + try { + $stream = [IO.File]::OpenRead($Path) + return [BitConverter]::ToString($algorithm.ComputeHash($stream)).Replace('-', '') + } finally { + if ($null -ne $stream) { $stream.Dispose() } + $algorithm.Dispose() + } +} $requestPath = Join-Path (Get-Location).Path 'ucp/replays/restart-request.json' try { $request = Get-Content -LiteralPath $requestPath -Raw -Encoding UTF8 | ConvertFrom-Json @@ -14,14 +28,14 @@ try { if ($manifest.id -ne $request.id -or $manifest.status -ne 'complete') { throw 'Replay is incomplete' } $executable = [IO.Path]::GetFullPath($request.executable) if ([IO.Path]::GetDirectoryName($executable) -ne $gameRoot) { throw 'Game directory differs' } - if ((Get-FileHash -LiteralPath $executable -Algorithm SHA256).Hash -ne $manifest.executable) { throw 'Game executable differs' } + if ((Get-ReplayHash $executable) -ne $manifest.executable) { throw 'Game executable differs' } $gameProcess = Get-Process -Id $request.processId -ErrorAction SilentlyContinue if ($null -ne $gameProcess) { if ($gameProcess.Path -ne $executable) { throw 'Process identity differs' } $gameProcess.WaitForExit() } - if ((Get-FileHash -LiteralPath $executable -Algorithm SHA256).Hash -ne $manifest.executable) { throw 'Game executable changed while waiting' } - if ((Get-FileHash -LiteralPath $settingsPath -Algorithm SHA256).Hash -ne $manifest.settingsHash) { throw 'Recorded settings are damaged' } + if ((Get-ReplayHash $executable) -ne $manifest.executable) { throw 'Game executable changed while waiting' } + if ((Get-ReplayHash $settingsPath) -ne $manifest.settingsHash) { throw 'Recorded settings are damaged' } $env:UCP_RECORDER_REPLAY = $request.id # A Windows file path cannot contain a quote. This one ends in .yml, not a backslash. $arguments = '--ucp-config-file="' + $settingsPath + '"' diff --git a/tests/test_restart_helper.py b/tests/test_restart_helper.py index 316156b..079a53d 100644 --- a/tests/test_restart_helper.py +++ b/tests/test_restart_helper.py @@ -39,6 +39,7 @@ def setUp(self): (self.root/'helper.ps1').write_text(script, encoding='utf-8-sig') (self.root/'runner.ps1').write_text(''' $ErrorActionPreference = 'Stop' +function Get-FileHash { throw 'Optional PowerShell script module is unavailable' } function Get-Process { param($Id,$ErrorAction) $process = [pscustomobject]@{ Path = (Join-Path (Get-Location).Path 'Crusader test.exe') } From 3ccc3df64a88c4846118b0f1e239603beca2da60 Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:26:13 +0200 Subject: [PATCH 08/19] Scope recorder simulation changes and controls to single-player (0.5.0) --- CHANGELOG.md | 11 +++++ README.md | 4 +- code/engine.lua | 11 ++++- code/fixes.lua | 61 ++++++++++++----------- code/native-ui.lua | 27 +++++++++++ code/scoped-code.lua | 59 +++++++++++++++++++++++ code/scoped-sites.lua | 27 +++++++++++ code/session-recorder.lua | 18 ++++++- code/sessions.lua | 2 +- code/ui-sites.lua | 2 + code/ui.lua | 3 ++ definition.yml | 2 +- docs/multiplayer-isolation.md | 23 +++++++++ init.lua | 30 +++--------- tests/check_executables.py | 2 +- tests/test_engine.py | 21 ++++++++ tests/test_native_ui.py | 18 +++++++ tests/test_recorder.py | 4 +- tests/test_scoped_code.py | 91 +++++++++++++++++++++++++++++++++++ tests/test_sessions.py | 27 +++++++++++ 20 files changed, 381 insertions(+), 62 deletions(-) create mode 100644 code/scoped-code.lua create mode 100644 code/scoped-sites.lua create mode 100644 docs/multiplayer-isolation.md create mode 100644 tests/test_scoped_code.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 818beaf..cc4131b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.5.0 + +- Gate randomness, dust, pause and fixed-seed changes by both an active recorder scope and the native single-player mode. Multiplayer and idle paths execute the original instruction sequences. +- Gate the simulation observer and halt path as well, so a stale replay cannot freeze multiplayer or suppress its commands. +- Abort an active capture on a multiplayer mode transition without changing the multiplayer pause flag. +- Hide recorder controls from both rendering and input in multiplayer using the native menu's skipped-item behaviour, preserving other extensions' items. +- Add x86 emulation comparisons for original versus gated paths in Crusader and Extreme, including condition flags, stack cleanup and stale halt state. +- Use simulation profile `recorder-sp-v2`; earlier experimental captures require their original module version. + +This closes the known unconditional simulation-patch gap. It does not add multiplayer recording or certify multiplayer compatibility. Live multiplayer comparison and full single-player replay/menu tests remain pending. + ## 0.4.0 - Replace temporary Skirmish checkboxes with labelled **Record next match** and **Replays** controls. diff --git a/README.md b/README.md index b94aa3c..58d6fc0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.4.0 adds a native replay browser and settings restart to individual sessions, starting saves and settings snapshots. **The new save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer is not supported yet. +Version 0.5.0 confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. ## Testing this build @@ -30,4 +30,4 @@ python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.4.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.5.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/engine.lua b/code/engine.lua index 144d43e..5ec57a3 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -29,6 +29,7 @@ function M.new(sites) e.loadNative=core.exposeCode(sites.load.address,1,0) e.buffer=core.allocate(1260,true) e.pathBuffer=core.allocate(512,true) + e.scope=core.allocate(4,true) e.pendingSlots={} return setmetatable(e,{__index=M}) end @@ -39,7 +40,13 @@ function M:singlePlayer() local mode=core.readInteger(self.base+0x618) return mode==0 or mode==99 end -function M:pause() core.writeInteger(self.sites.paused,1) end +function M:setScope(active) + assert(not active or self:singlePlayer(),'Replay simulation scope requires single-player') + core.writeInteger(self.scope,active and 1 or 0) +end +function M:pause() + if self:singlePlayer() then core.writeInteger(self.sites.paused,1) end +end function M:rngState() return {core.readSmallInteger(self.rng),core.readSmallInteger(self.rng+2), @@ -134,7 +141,7 @@ function M:install(recorder) return originalMapName(this,index) end,self.sites.mapName.address,2,1,#self.sites.mapName.bytes) originalQueue=core.hookCode(function(this,category) - if recorder.mode=='play' and not self.loading then return 0 end + if recorder.mode=='play' and self:singlePlayer() and not self.loading then return 0 end return originalQueue(this,category) end,self.sites.queue.address,2,1,#self.sites.queue.bytes) -- Replace exactly MOV EAX,[ESI+commandSize] before the timed payload copy. diff --git a/code/fixes.lua b/code/fixes.lua index 9b4cc4d..95b3a92 100644 --- a/code/fixes.lua +++ b/code/fixes.lua @@ -1,34 +1,33 @@ -local native = require("code/native") ---[[ - Fixes which might be needed for stable recordings - Test this -]]-- +local native=require('code/native') +local profiles=require('code/scoped-sites') +local emitter=require('code/scoped-code') +local M={} -return { - apply = function() - --- faulty randomness line? This one is called extra when loading from the load game menu - -- Dust spawning normally returns with RET 0x2c (11 arguments). - -- Replace the whole CALL and perform exactly its 44-byte cleanup. - core.writeCode(native.addr(0x004fc4a3), {0x90, 0x90, 0x90, 0x90, 0x90}) - core.writeCode(native.addr(0x004fc627), {0x83, 0xC4, 0x2C, 0x90, 0x90}) +function M.verify() + local sites=assert(profiles[native.profile.name]) + for _,site in ipairs(sites) do + local actual=core.readBytes(site.address,#site.bytes) + for i,byte in ipairs(site.bytes) do + assert(actual[i]==byte,'Recorder simulation hook conflicts at '..site.name) + end + end + return sites +end - --- Mother RNG fix, nopping out. Could also switch with RNG1 to keep the baby cry noise - core.writeCode(native.addr(0x005474a3), {0x90, 0x90, 0x90, 0x90, 0x90}) +function M.install(sites,enabled,mode,seed) + for _,site in ipairs(sites) do + if site.kind~='seed' or seed~=nil then + local size=#emitter.build(site,enabled,mode,seed,0) + local target=core.allocateCode(size) + core.writeCode(target,emitter.build(site,enabled,mode,seed,target)) + core.writeCode(site.address,emitter.jump(site.address,target,#site.bytes)) + end + end +end - -- removing all function calls to RNG1 that come from the music thread. - core.writeCode(native.addr(0x0047a8d5), {0x90, 0x90, 0x90, 0x90, 0x90}) - core.writeCode(native.addr(0x0047a86b), {0x90, 0x90, 0x90, 0x90, 0x90}) - - core.writeCode(native.addr(0x0047c348), {0x90, 0x90, 0x90, 0x90, 0x90}) - - --- Right click menu fixes (during pause): - -- Do not process tick when game is paused and pull down terrain or flatten terrain is used - core.writeCode(native.addr(0x0045ceff), {0x90, 0x90}) - - -- Do not increase match time when game is paused and flatten terrain is used - core.writeCode(native.addr(0x0045ce34), {0xEB, 0x46}) - - -- Removing mothers and children (TODO test without this) - core.writeCode(native.addr(0x004582ed), {0xEB}) - end -} +function M.installTick(site,enabled,mode,halt,callback) + local tick={address=site.address,bytes=site.bytes,kind='raw',patch='tick', + halt=halt,callback=callback,skipTick=site.address+0x25} + M.install({tick},enabled,mode) +end +return M diff --git a/code/native-ui.lua b/code/native-ui.lua index e8d3d07..741165d 100644 --- a/code/native-ui.lua +++ b/code/native-ui.lua @@ -92,4 +92,31 @@ end function M:show(id) self.activateNative(self.sites.modalComposition.value,id,0) end function M:close() self:show(-1) end + +function M:trackVisibility(referenceItems,predicate) + local callbacks={} + for _,item in ipairs(referenceItems) do callbacks[core.readInteger(item+20)]=true end + local original + original=core.hookCode(function(this,action) + if this==core.readInteger(referenceItems[1]+0x4c) then + local ok,reason=pcall(function() + local item=core.readInteger(this) + local visible=predicate() + -- Find our callbacks in the current array, allowing other modules to + -- reallocate or append items. A negative type skips only this item. + for _=1,4096 do + local kind=core.readInteger(item) + if kind==0x66 then return end + if callbacks[core.readInteger(item+20)] then + core.writeInteger(item,visible and 3 or -2147483645) + end + item=item+self.ITEM_SIZE + end + error('Replay menu array has no terminator') + end) + if not ok then self.onError(reason) end + end + return original(this,action) + end,self.sites.handleMenu.address,2,1,#self.sites.handleMenu.bytes) +end return M diff --git a/code/scoped-code.lua b/code/scoped-code.lua new file mode 100644 index 0000000..161d863 --- /dev/null +++ b/code/scoped-code.lua @@ -0,0 +1,59 @@ +-- Small x86 emitter for gates whose original instructions contain relative +-- branches/calls. core.insertCode copies bytes without relocating them. +local M={} +local function dword(out,value) + value=value%4294967296 + for _=1,4 do out[#out+1]=value%256; value=math.floor(value/256) end +end + +function M.jump(from,to,size) + local bytes={0xe9}; dword(bytes,to-from-5) + for i=6,size do bytes[i]=0x90 end + return bytes +end + +function M.build(site,enabled,mode,seed,origin) + local out,labels,refs={},{},{} + local function emit(...) for _,value in ipairs({...}) do out[#out+1]=value end end + local function rel(op,target) + for _,byte in ipairs(op) do emit(byte) end + if type(target)=='string' then refs[#refs+1]={offset=#out,target=target}; dword(out,0) + else dword(out,target-origin-#out-4) end + end + local function compare(address,value) emit(0x83,0x3d); dword(out,address); emit(value) end + emit(0x9c) -- pushfd: the conditional branches below must preserve incoming flags + compare(enabled,1); rel({0x0f,0x85},'original') + compare(mode,0); rel({0x0f,0x84},'patched') + compare(mode,99); rel({0x0f,0x85},'original') + labels.patched=#out; emit(0x9d) + if site.patch=='cleanup' then emit(0x8d,0x64,0x24,0x2c) -- lea esp,[esp+44], no flag changes + elseif site.patch=='taken' then rel({0xe9},site.target) + elseif site.patch=='fallthrough' then for i=3,#site.bytes do emit(site.bytes[i]) end + elseif site.patch=='seed' then + emit(0xb8); dword(out,seed) + for _,byte in ipairs(site.bytes) do emit(byte) end + elseif site.patch=='tick' then + emit(0x9c,0x60) -- preserve flags and registers across the Lua observer + rel({0xe8},site.callback) + emit(0x61) + compare(site.halt,0); rel({0x0f,0x85},'halted') + emit(0x9d) + for _,byte in ipairs(site.bytes) do emit(byte) end + rel({0xe9},site.address+#site.bytes) + labels.halted=#out; emit(0x9d); rel({0xe9},site.skipTick) + end + rel({0xe9},site.address+#site.bytes) + labels.original=#out; emit(0x9d) + if site.kind=='call' then rel({0xe8},site.target) + elseif site.kind=='branch' then + rel({0x0f,site.condition},site.target) + for i=3,#site.bytes do emit(site.bytes[i]) end + else for _,byte in ipairs(site.bytes) do emit(byte) end end + rel({0xe9},site.address+#site.bytes) + for _,ref in ipairs(refs) do + local encoded={}; dword(encoded,assert(labels[ref.target])-ref.offset-4) + for i,byte in ipairs(encoded) do out[ref.offset+i]=byte end + end + return out +end +return M diff --git a/code/scoped-sites.lua b/code/scoped-sites.lua new file mode 100644 index 0000000..c4ed60d --- /dev/null +++ b/code/scoped-sites.lua @@ -0,0 +1,27 @@ +-- Complete instructions for scope-gated simulation changes. +return { + SHC = { + {name="dustRNG",address=0x004FC4A3,kind="call",patch="skip",bytes={232,40,227,246,255}, target=0x0046A7D0}, + {name="dustEntity",address=0x004FC627,kind="call",patch="cleanup",bytes={232,180,132,240,255}, target=0x00404AE0}, + {name="motherSound",address=0x005474A3,kind="call",patch="skip",bytes={232,40,51,242,255}, target=0x0046A7D0}, + {name="music1",address=0x0047A8D5,kind="call",patch="skip",bytes={232,38,255,254,255}, target=0x0046A800}, + {name="music2",address=0x0047A86B,kind="call",patch="skip",bytes={232,144,255,254,255}, target=0x0046A800}, + {name="music3",address=0x0047C348,kind="call",patch="skip",bytes={232,179,228,254,255}, target=0x0046A800}, + {name="pause",address=0x0045CEFF,kind="branch",patch="fallthrough",bytes={117,45,161,84,160,254,1}, target=0x0045CF2E, condition=133}, + {name="pausedCamera",address=0x0045CE34,kind="branch",patch="taken",bytes={125,70,185,16,125,254,1}, target=0x0045CE7C, condition=141}, + {name="mothers",address=0x004582ED,kind="branch",patch="taken",bytes={117,43,131,60,16,0}, target=0x0045831A, condition=133}, + {name="seed",address=0x0046A74A,kind="seed",patch="seed",bytes={131,196,4,137,70,4}}, + }, + Extreme = { + {name="dustRNG",address=0x004FC823,kind="call",patch="skip",bytes={232,200,225,246,255}, target=0x0046A9F0}, + {name="dustEntity",address=0x004FC9A7,kind="call",patch="cleanup",bytes={232,68,129,240,255}, target=0x00404AF0}, + {name="motherSound",address=0x005478C3,kind="call",patch="skip",bytes={232,40,49,242,255}, target=0x0046A9F0}, + {name="music1",address=0x0047AAA5,kind="call",patch="skip",bytes={232,118,255,254,255}, target=0x0046AA20}, + {name="music2",address=0x0047AA3B,kind="call",patch="skip",bytes={232,224,255,254,255}, target=0x0046AA20}, + {name="music3",address=0x0047C518,kind="call",patch="skip",bytes={232,3,229,254,255}, target=0x0046AA20}, + {name="pause",address=0x0045D10F,kind="branch",patch="fallthrough",bytes={117,45,161,84,213,167,2}, target=0x0045D13E, condition=133}, + {name="pausedCamera",address=0x0045D044,kind="branch",patch="taken",bytes={125,70,185,16,178,167,2}, target=0x0045D08C, condition=141}, + {name="mothers",address=0x0045851D,kind="branch",patch="taken",bytes={117,43,131,60,16,0}, target=0x0045854A, condition=133}, + {name="seed",address=0x0046A96A,kind="seed",patch="seed",bytes={131,196,4,137,70,4}}, + }, +} diff --git a/code/session-recorder.lua b/code/session-recorder.lua index fb73c58..c60c902 100644 --- a/code/session-recorder.lua +++ b/code/session-recorder.lua @@ -17,7 +17,7 @@ function Session:guard(callback) if not ok then if self.status=='error' and (self.active or self.mode~='none') then return false end -- retain the first session failure self.status='error'; self.error=tostring(reason) - local stopSimulation=self.active or self.mode=='play' + local stopSimulation=self.engine:singlePlayer() and (self.active or self.mode=='play') core.writeInteger(self.halt,stopSimulation and 1 or 0) if stopSimulation then self.engine:pause() end print('Replay stopped: ' .. self.error) @@ -43,10 +43,12 @@ function Session:startRecording() self:openFiles('w') self.mode='record'; self.status='armed'; self.active=false self.error=nil; self.observedTick=false + self.engine:setScope(true) core.writeInteger(self.halt,0) end function Session:activateRecording() + self:reconcileMode() if self.mode~='record' or self.status~='armed' then return end local path=store.path(self.manifest.id) self.engine:saveSnapshot(path..'/start.sav') @@ -86,6 +88,7 @@ function Session:startPlayback(id) self.manifest=manifest self.mode='play'; self.status='loading'; self.active=false self.error=nil + self.engine:setScope(true) core.writeInteger(self.halt,0) self.engine:loadSnapshot(path..'/start.sav') local bytes={}; for i=1,#rng do bytes[i]=rng:byte(i) end @@ -98,6 +101,7 @@ function Session:startPlayback(id) end function Session:onCommand(category,time,address,size) + self:reconcileMode() if self.status~='recording' or not self.active or time<=0 then return end validation.integer(size,0,1260,'native command size') validation.sessionCommand({commandCategory=category,time=time,player=self.engine:player(), @@ -107,6 +111,7 @@ function Session:onCommand(category,time,address,size) end function Session:feed() + self:reconcileMode() if self.status~='playing' then return end local now=self.engine:tick() while self.engine:canSchedule() do @@ -121,6 +126,7 @@ function Session:feed() end function Session:onTick() + self:reconcileMode() if not self.active then return end local now=self.engine:tick() if self.status=='recording' then @@ -160,6 +166,8 @@ function Session:onTick() end function Session:reset() + self.engine:setScope(false) + core.writeInteger(self.halt,0) local manifest=self.mode=='record' and self.manifest local complete=manifest and self.active and self.observedTick and self.status=='recording' self.active=false @@ -182,5 +190,13 @@ function Session:reset() core.writeInteger(self.halt,0) end +function Session:reconcileMode() + if self.mode~='none' and not self.engine:singlePlayer() then + self.status='error' + self.error='Replay session ended when entering multiplayer' + self:reset() + end +end + function Session:discardFiles() end -- Cancelling never deletes a previous recording. return Session diff --git a/code/sessions.lua b/code/sessions.lua index 5839449..8cce08a 100644 --- a/code/sessions.lua +++ b/code/sessions.lua @@ -1,6 +1,6 @@ local platform = require('code/platform') local validation = require('code/validation') -local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v1'} +local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v2'} local activeSettings local function read(path) diff --git a/code/ui-sites.lua b/code/ui-sites.lua index 76c13d8..7a7fd19 100644 --- a/code/ui-sites.lua +++ b/code/ui-sites.lua @@ -2,6 +2,7 @@ -- Signatures and ABI: UCP ui module ui/game.lua and ui/headers/latest/ui.h. return { SHC = { + handleMenu = {address=0x004F6280, bytes={83,139,217,86,139,51,199,67,20,0,0,0,0}}, menuConstructor = {address=0x004F4100, bytes={81,83,139,217}}, modalConstructor = {address=0x004A9E00, bytes={139,84,36,8,139,193,139,76,36,4,137,8}}, activateModal = {address=0x004A9ED0, bytes={83,85,51,237,57,108,36,16}}, @@ -15,6 +16,7 @@ return { modalStack = {address=0x004A9E40, bytes={139,21,164,66,223,0,137,80,36}, value=0x00DF42A4}, }, Extreme = { + handleMenu = {address=0x004F6610, bytes={83,139,217,86,139,51,199,67,20,0,0,0,0}}, menuConstructor = {address=0x004F4490, bytes={81,83,139,217}}, modalConstructor = {address=0x004A9F70, bytes={139,84,36,8,139,193,139,76,36,4,137,8}}, activateModal = {address=0x004AA040, bytes={83,85,51,237,57,108,36,16}}, diff --git a/code/ui.lua b/code/ui.lua index 14d2b7b..8be5d1e 100644 --- a/code/ui.lua +++ b/code/ui.lua @@ -47,15 +47,18 @@ function M.createButtons(recorder,sites) ui:button(record,250,550,145,30,function() return recorder.status=='armed' and 'Cancel recording' or 'Record next match' end,function() + if not recorder.engine:singlePlayer() then return end if recorder.mode=='none' then recorder:guard(function() recorder:startRecording() end) elseif recorder.mode=='record' then recorder:guard(function() recorder:stopRecording() end) end end,function() return recorder.status=='armed' end) ui:button(browse,410,550,145,30,'Replays',function() + if not recorder.engine:singlePlayer() then return end browser:refresh(); ui:show(dialog) end) core.writeCode(native.addr(0x59ab30),{ core.AssemblyLambda('push array',{array=array}) }) + ui:trackVisibility({record,browse},function() return recorder.engine:singlePlayer() end) end function M.resetButtons() end -- Labels derive from session state at render time. diff --git a/definition.yml b/definition.yml index ea743d7..5332e01 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.4.0 +version: 0.5.0 type: module author: Corax meta: diff --git a/docs/multiplayer-isolation.md b/docs/multiplayer-isolation.md new file mode 100644 index 0000000..149ec16 --- /dev/null +++ b/docs/multiplayer-isolation.md @@ -0,0 +1,23 @@ +# Multiplayer isolation (0.5.0) + +The module previously installed unconditional RNG and gameplay patches, even with recording disabled. Its single-player start guard did not protect normal multiplayer from those changes. + +Each simulation patch now checks a recorder-owned scope flag and the native mode (`0` or `99` for single-player). All other modes execute the original instructions. A stale scope flag therefore cannot apply the changes in multiplayer. The fixed seed and tick observer use the same gates; the multiplayer tick path ignores a stale halt flag and does not call Lua recording code. + +Arming recording enables the scope; playback enables it only after preflight. Reset clears it before file finalization. A transition into multiplayer aborts the capture and closes files without writing the multiplayer pause flag. The command-queue guard independently checks single-player mode before blocking playback input. + +## Native equivalence checks + +`scoped-sites.lua` records complete original instruction sequences for both variants, extending the old one/two-byte branch checks to full instructions. `scoped-code.lua` emits and relocates calls/conditional branches explicitly: UCP's `core.insertCode` copies original bytes without fixing their relative destinations. + +The emulator executes the original and generated sequences and compares registers, flags, stack, destinations and observable callee effects. Cases cover idle single-player, multiplayer with both scope values, conditional flag combinations, stack-cleaning dust calls, fixed-seed writes and multiplayer with a stale halt flag. This demonstrates equivalence of the guarded sequences; it is not an emulation of a whole multiplayer match or proof that all other modules are compatible. + +## Menu integration + +Ghidra's `Menu::handleMenuItems` skips negative item types in its input and rendering passes. The recorder applies that state only to its own controls when multiplayer is active, then restores normal type `3` in single-player. It identifies controls by their callbacks in the current owning menu array, preserving appended items and accommodating array reallocation. It does not insert a terminator that would hide another extension's controls. + +The native guard is installed before menu traversal. Action callbacks also check game mode. Modal rendering, placement and navigation still need visual tests. + +## Next gaps + +Multiplayer recording needs all executed commands with actor identities, a defined initial-state boundary, and a strategy for multiplayer-only simulation behaviour during playback. Automation and extension state must be captured explicitly; version names alone are insufficient. Save/autosave handling, world-state verification and real capture/load/playback comparisons remain separate work. diff --git a/init.lua b/init.lua index daceab6..af90b0d 100644 --- a/init.lua +++ b/init.lua @@ -7,13 +7,15 @@ function module:enable(config) native.verify() local sites=Engine.verify() local uiSites=require('code/native-ui').verify() + local fixes=require('code/fixes') + local fixSites=fixes.verify() local seed if config.useFixedSeed then seed=require('code/validation').integer(config.fixedSeed,-2147483648,2147483647,'fixed seed') end require('code/sessions').captureSettings() - require('code/fixes').apply() local engine=Engine.new(sites) + fixes.install(fixSites,engine.scope,engine.base+0x618,seed) local recorder=Session:new(engine) self.recorder=recorder engine:install(recorder) @@ -22,7 +24,10 @@ function module:enable(config) local function observe(address,size,callback) core.detourCode(function(registers) - recorder:guard(function() callback(registers) end) + recorder:guard(function() + recorder:reconcileMode() + callback(registers) + end) return registers end,address,size) end @@ -44,26 +49,7 @@ function module:enable(config) local tickCallback=core.allocateCode({0x90,0x90,0x90,0x90,0x90,0xC3}) observe(tickCallback,5,function() recorder:onTick() end) - core.insertCode(sites.tick.address,5,{ - core.AssemblyLambda([[ - pushfd - pushad - call callback - popad - cmp dword [halt], 0 - je continueTick - popfd - jmp skipTick - continueTick: - popfd - ]],{callback=tickCallback,halt=recorder.halt,skipTick=sites.tick.address+0x25}) - },nil,'after') - if seed then - core.detourCode(function(registers) - registers.EAX=seed - return registers - end,native.addr(0x46a74a),6) - end + fixes.installTick(sites.tick,engine.scope,engine.base+0x618,recorder.halt,tickCallback) end function module:disable() diff --git a/tests/check_executables.py b/tests/check_executables.py index c8fac3c..f5573db 100644 --- a/tests/check_executables.py +++ b/tests/check_executables.py @@ -43,7 +43,7 @@ def check(folder): menu = native.addr(0x59ab30) assert reader(menu, 1) == b'\x68' assert struct.unpack(' Date: Sat, 5 Sep 2026 17:23:05 +0200 Subject: [PATCH 09/19] Capture and verify native replay command execution (0.6.0) --- CHANGELOG.md | 10 +++++ README.md | 4 +- code/command-journal.lua | 31 ++++++++++++++ code/engine-sites.lua | 6 +++ code/engine.lua | 77 +++++++++++++++++++++++++++++++---- code/session-recorder.lua | 15 +++---- code/sessions.lua | 2 +- definition.yml | 2 +- docs/command-execution.md | 11 +++++ docs/replay-sessions.md | 2 +- init.lua | 3 -- tests/test_command_journal.py | 34 ++++++++++++++++ tests/test_engine.py | 67 +++++++++++++++++++++++++++++- tests/test_sessions.py | 8 ++-- 14 files changed, 245 insertions(+), 27 deletions(-) create mode 100644 code/command-journal.lua create mode 100644 docs/command-execution.md create mode 100644 tests/test_command_journal.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cc4131b..cd2e00a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.6.0 + +- Capture timed commands when their native handlers return, including inputs delivered directly to the received-command scheduler. Record their actual execution tick and resolved actor. +- Verify playback command ownership, order, actor, tick, category and payload before dispatch, and count execution only after the handler returns. Suppress a mismatching dispatch and subsequent commands in the stopped batch. +- Keep pending replay commands owned until execution is observed, even if native code changes their slot state. +- Preserve the protocol dispatch hook and original multiplayer dispatch behaviour. +- Use simulation profile `recorder-sp-v3`; older experimental captures require their original version. + +Validation: 68 automated tests and original executable checks for Crusader and Extreme. Full live replay and custom extension protocol support remain pending. + ## 0.5.0 - Gate randomness, dust, pause and fixed-seed changes by both an active recorder scope and the native single-player mode. Multiplayer and idle paths execute the original instruction sequences. diff --git a/README.md b/README.md index 58d6fc0..a1f53fc 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.5.0 confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. +Version 0.6.0 verifies native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. ## Testing this build @@ -30,4 +30,4 @@ python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.5.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.6.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/command-journal.lua b/code/command-journal.lua new file mode 100644 index 0000000..b2ae2b3 --- /dev/null +++ b/code/command-journal.lua @@ -0,0 +1,31 @@ +-- Track native ring ownership separately from the ring's reusable state byte. +local validation=require('code/validation') +local M={} +function M.new() return setmetatable({slots={},nextSequence=1,executed=0},{__index=M}) end + +function M:queue(slot,command) + validation.command(command) + assert(not self.slots[slot],'Replay command slot reused before execution') + local copy={}; for k,v in pairs(command) do copy[k]=v end + self.slots[slot]={command=copy,sequence=self.nextSequence} + self.nextSequence=self.nextSequence+1 +end + +function M:before(slot,actual) + local entry=assert(self.slots[slot],'Unexpected native command during playback') + assert(entry.sequence==self.executed+1,'Native replay command execution order differs') + for _,key in ipairs({'commandCategory','time','player','size'}) do + assert(actual[key]==entry.command[key],'Native replay command '..key..' differs') + end + assert(actual.data:upper()==entry.command.data:upper(),'Native replay command payload differs') + return entry +end + +function M:after(slot,entry) + assert(entry and self.slots[slot]==entry,'Native replay command ownership changed during execution') + self.slots[slot]=nil + self.executed=self.executed+1 +end + +function M:pending() return next(self.slots)~=nil end +return M diff --git a/code/engine-sites.lua b/code/engine-sites.lua index a87b4f9..5ec9d9d 100644 --- a/code/engine-sites.lua +++ b/code/engine-sites.lua @@ -8,6 +8,9 @@ return { load = {address=0x004950B0, bytes={129,236,244,3,0,0}}, queue = {address=0x00489100, bytes={83,86,139,241,139,134,224,158,16,0}}, copySize = {address=0x00480375, bytes={139,134,48,216,2,0}}, + execute = {address=0x0048937F, bytes={15,190,148,49,132,198,3,0}}, + executed = {address=0x00489390, bytes={139,142,36,216,2,0}}, + actorOffset = 0x00109E70, switchMenu = {address=0x0046B350, bytes={0,0,139,68,36}}, resources = 0x011BF130, packager = 0x00F2B3D0, @@ -25,6 +28,9 @@ return { load = {address=0x00495210, bytes={129,236,244,3,0,0}}, queue = {address=0x00489210, bytes={83,86,139,241,139,134,112,99,22,0}}, copySize = {address=0x00480545, bytes={139,134,48,216,2,0}}, + execute = {address=0x0048948F, bytes={15,190,148,49,132,198,3,0}}, + executed = {address=0x004894A0, bytes={139,142,36,216,2,0}}, + actorOffset = 0x00166300, switchMenu = {address=0x0046B570, bytes={0,0,139,68,36}}, resources = 0x01293C20, packager = 0x00F2B850, diff --git a/code/engine.lua b/code/engine.lua index 5ec57a3..88888a5 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -30,10 +30,16 @@ function M.new(sites) e.buffer=core.allocate(1260,true) e.pathBuffer=core.allocate(512,true) e.scope=core.allocate(4,true) - e.pendingSlots={} + M.resetCommands(e) return setmetatable(e,{__index=M}) end +function M:resetCommands() + self.journal=require('code/command-journal').new() + self.received={} + self.executing=nil +end + function M:tick() return core.readInteger(native.addr(0x1fe7da8)) end function M:player() return core.readInteger(native.addr(0x1a275dc)) end function M:singlePlayer() @@ -91,12 +97,13 @@ function M:loadSnapshot(path) -- These four fields belong to the load dialog; preserve the game's menu transition. for offset,value in pairs(old) do core.writeInteger(state+offset,value) end end) - self.pendingSlots={} + self:resetCommands() end function M:canSchedule() local index=core.readInteger(self.base+self.sites.writeIndexOffset) if index<0 or index>=200 then return false end + if self.journal.slots[index] then return false end local state=core.readByte(self.base+0x3c67c+index*1272+9) return state==0 or state>=10 end @@ -117,16 +124,39 @@ function M:scheduleCommand(command) core.writeByte(self.base+0x3c67c+slot*1272+9,10) error(self.copyError) end - self.pendingSlots[slot]=true + self.journal:queue(slot,command) end function M:commandsPending() - for slot in pairs(self.pendingSlots) do - local state=core.readByte(self.base+0x3c67c+slot*1272+9) - if state>0 and state<10 then return true end - self.pendingSlots[slot]=nil + return self.journal:pending() +end + +function M:beforeCommand(recorder) + assert(not self.executing,'Nested native command execution is unsupported') + local slot=require('code/validation').integer(core.readInteger(self.base+0x2d824),0,199,'native ring slot') + local address=self.base+0x3c67c+slot*1272 + local source=recorder.mode=='play' and self.journal.slots[slot] or self.received[slot] + assert(source,'Native command has no captured payload/ownership') + local command=source.command + local actual={commandCategory=core.readByte(address+8),time=self:tick(),player=core.readInteger(self.base+self.sites.actorOffset), + size=command.size,data=require('code/utils').tableToHex(core.readBytes(address+10,command.size))} + require('code/validation').sessionCommand(actual,recorder.manifest) + assert(core.readInteger(address)==command.time,'Native command scheduling tick changed') + assert(actual.commandCategory==command.commandCategory and actual.data:upper()==command.data:upper(), + 'Native command changed between receipt and execution') + if recorder.mode=='play' then self.journal:before(slot,actual) end + self.executing={slot=slot,source=source,command=actual} +end + +function M:afterCommand(recorder) + local current=self.executing + self.executing=nil + if not current then return end + if recorder.mode=='play' then self.journal:after(current.slot,current.source) + else + self.received[current.slot]=nil + recorder:onExecutedCommand(current.command) end - return false end function M:install(recorder) @@ -154,8 +184,39 @@ function M:install(recorder) size=0 end registers.EAX=size + if recorder.active and recorder.status=='recording' and self:singlePlayer() then + recorder:guard(function() + require('code/validation').integer(size,0,1260,'native payload size') + local slot=core.readInteger(self.base+0x2d824) + require('code/validation').integer(slot,0,199,'native ring slot') + local address=self.base+0x3c67c+slot*1272 + self.received[slot]={command={commandCategory=core.readByte(address+8),time=core.readInteger(address), + size=size,data=require('code/utils').tableToHex(core.readBytes(registers.EDX,size))}} + end) + end return registers end,self.sites.copySize.address,6) + -- This instruction precedes protocol's dispatch hook. Leave its hook intact. + -- On failure use native category 0 (the no-op), preventing the bad dispatch. + core.writeCode(self.sites.execute.address,{0x90,0x90,0x90,0x90,0x90,0x90,0x90,0x90}) + core.detourCode(function(registers) + local category=core.readByte(registers.ESI+registers.ECX+0x3c684) + registers.EDX=category<128 and category or category-256 -- original MOVSX + if self:singlePlayer() and not self.loading and recorder.active then + local ok=recorder:guard(function() + assert(recorder.status=='playing' or recorder.status=='recording','Replay session is stopped') + self:beforeCommand(recorder) + end) + if not ok then registers.EDX=0 end + end + return registers + end,self.sites.execute.address,8) + core.detourCode(function(registers) + if self:singlePlayer() and not self.loading and recorder.active then + recorder:guard(function() self:afterCommand(recorder) end) + end + return registers + end,self.sites.executed.address,6) end return M diff --git a/code/session-recorder.lua b/code/session-recorder.lua index c60c902..1a59562 100644 --- a/code/session-recorder.lua +++ b/code/session-recorder.lua @@ -43,6 +43,7 @@ function Session:startRecording() self:openFiles('w') self.mode='record'; self.status='armed'; self.active=false self.error=nil; self.observedTick=false + self.engine:resetCommands() self.engine:setScope(true) core.writeInteger(self.halt,0) end @@ -100,13 +101,11 @@ function Session:startPlayback(id) print('Playing '..id) end -function Session:onCommand(category,time,address,size) - self:reconcileMode() - if self.status~='recording' or not self.active or time<=0 then return end - validation.integer(size,0,1260,'native command size') - validation.sessionCommand({commandCategory=category,time=time,player=self.engine:player(), - size=size,data=require('code/utils').tableToHex(core.readBytes(address,size))},self.manifest) - self:saveCommand(category,time,address,size,self.engine:player()) +function Session:onExecutedCommand(command) + if self.status~='recording' or not self.active then return end + validation.sessionCommand(command,self.manifest) + assert(self.commandsFile:write(json:encode(command)..'\n')) + assert(self.commandsFile:flush()) self.manifest.commandCount=self.manifest.commandCount+1 end @@ -156,6 +155,7 @@ function Session:onTick() if now>=self.manifest.lastTick then assert(self.playedCommands==self.manifest.commandCount,'Replay ended before all commands were scheduled') assert(not self.engine:commandsPending(),'Replay ended with commands still waiting to execute') + assert(self.engine.journal.executed==self.manifest.commandCount,'Replay native execution count differs') local actual=self.engine:rngState() for i=1,4 do assert(actual[i]==self.manifest.finalRng[i],'Final RNG state differs at tick '..now) end self.status='finished'; core.writeInteger(self.halt,1); self.engine:pause() @@ -167,6 +167,7 @@ end function Session:reset() self.engine:setScope(false) + self.engine:resetCommands() core.writeInteger(self.halt,0) local manifest=self.mode=='record' and self.manifest local complete=manifest and self.active and self.observedTick and self.status=='recording' diff --git a/code/sessions.lua b/code/sessions.lua index 8cce08a..1c896b7 100644 --- a/code/sessions.lua +++ b/code/sessions.lua @@ -1,6 +1,6 @@ local platform = require('code/platform') local validation = require('code/validation') -local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v2'} +local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v3'} local activeSettings local function read(path) diff --git a/definition.yml b/definition.yml index 5332e01..2b32ace 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.5.0 +version: 0.6.0 type: module author: Corax meta: diff --git a/docs/command-execution.md b/docs/command-execution.md new file mode 100644 index 0000000..5d8dbb6 --- /dev/null +++ b/docs/command-execution.md @@ -0,0 +1,11 @@ +# Native command execution (0.6.0) + +The old transmitter hook observed local submissions, rather than the order of commands that actually affected the match. Capture now observes timed received payloads and records a command only after its native handler returns. This includes direct calls to the received-command scheduler. Untimed commands and custom extension protocols remain outside the supported format. + +Ghidra/OpenSHC `GameSynchronyState::processWaitingCommands` resolves the current actor, selects the category, calls its handler, then marks the slot processed. The category load is at Crusader 0x48937f / Extreme 0x48948f; the post-call instruction is at 0x489390 / 0x4894a0. The eight-byte category load is replaced explicitly, including its signed-byte conversion. The six-byte post-call load is relocated by the detour. These sites contain no relative branches or calls. Exact original bytes are checked before installation. + +Payload lengths are captured at the scheduler copy boundary (0x480375 / 0x480545). Capture checks that the ring still contains that payload and scheduling tick, then records the actual execution tick and resolved actor. Playback associates each occupied slot with an immutable expected command and sequence number. A changed actor, payload, category, execution tick, scheduling tick, missing owner or changed order fails before dispatch. Native category zero suppresses that dispatch and the remainder of a stopped batch. The post-call observer retires ownership; a processed state byte alone never proves success. + +The category hook sits before protocol's seven-byte dispatch hook and does not overwrite it. Multiplayer, idle sessions and native snapshot loading follow the original path. This establishes command execution evidence, not multiplayer replay support: single-player identity, native snapshots, immediate commands, save/autosave, extension state and world-state verification still need their own handling. + +Tests inject wrong actors, modified payloads, delayed execution, changed scheduling ticks, unsupported categories, reordered commands, stale ownership and slot reuse. Both executable profiles are checked against original binaries. No live game is launched by these tests. diff --git a/docs/replay-sessions.md b/docs/replay-sessions.md index 5132696..e3f0bfc 100644 --- a/docs/replay-sessions.md +++ b/docs/replay-sessions.md @@ -18,7 +18,7 @@ The manifest moves through armed, recording and complete. Cancelled and failed s Capture starts after the native Skirmish launch function initializes the game. Checkpoints observe the same point immediately before each simulation tick, independent of render-frame count. The ending state is the last observed boundary, not the later time at which an exit dialog runs. Future commands still queued when recording ends are excluded from the finalized stream. -Playback preflights all files, loads the native starting save, restores the full RNG structure, and checks the loaded tick and local player. The native scheduler receives recorded commands with their original tick. It must have a free slot; its inferred payload length must match the recorded length. At completion, all recorded commands must have been scheduled and their slots consumed. RNG verification covers two values and two indices, **not a full world-state checksum**. +Playback preflights all files, loads the native starting save, restores the full RNG structure, and checks the loaded tick and local player. The native scheduler receives recorded commands with their recorded execution tick. It must have a free slot; its inferred payload length must match the recorded length. At completion, all recorded commands must have been scheduled and observed returning from their native handlers. Ownership, dispatch order, actor, tick, category and payload are checked before execution. RNG verification covers two values and two indices, **not a full world-state checksum**. The local single-player identity stays intact. New commands queued by a spectator are blocked while playback is active, finished or failed. This hook may also block commands from automation extensions; compatibility with those extensions requires separate analysis and testing. diff --git a/init.lua b/init.lua index af90b0d..a3d2ec5 100644 --- a/init.lua +++ b/init.lua @@ -35,9 +35,6 @@ function module:enable(config) recorder:activateRecording() ui.resetButtons() end) - observe(native.addr(0x487c50),6,function(registers) - if recorder.active and recorder.mode=='record' then recorder:onTransmitCommand(registers) end - end) observe(native.addr(0x490690),8,function() recorder:feed() end) observe(native.addr(0x46b358),6,function(registers) if registers.EBP==61 and not engine.loading then recorder:reset(); ui.resetButtons() end diff --git a/tests/test_command_journal.py b/tests/test_command_journal.py new file mode 100644 index 0000000..c8157e8 --- /dev/null +++ b/tests/test_command_journal.py @@ -0,0 +1,34 @@ +import unittest +import test_recorder as fixture + + +class CommandJournalTests(unittest.TestCase): + check = fixture.RecorderTests.check + setUp = fixture.RecorderTests.setUp + + def test_order_actor_payload_and_tick_are_verified(self): + self.check(''' +local Journal=require('code/command-journal') +for _,key in ipairs({'player','time','commandCategory','size','data'}) do + local j=Journal.new(); local c=command(); j:queue(0,c) + if key=='data' then c[key]='FF' else c[key]=c[key]+1 end + assert(not pcall(function() j:before(0,c) end)) + assert(j:pending() and j.executed==0) +end +local j=Journal.new(); j:queue(199,command()); j:queue(0,command()) +assert(not pcall(function() j:before(0,command()) end)) +assert(not pcall(function() j:before(1,command()) end)) +local first=j:before(199,command()); j:after(199,first) +local second=j:before(0,command()); j:after(0,second) +assert(not j:pending() and j.executed==2) +''') + + def test_ownership_survives_caller_mutation_and_rejects_reuse(self): + self.check(''' +local j=require('code/command-journal').new(); local c=command() +j:queue(1,c); c.data='FF' +assert(not pcall(function() j:queue(1,command()) end)) +local entry=j:before(1,command()); j:after(1,entry) +assert(not pcall(function() j:after(1,entry) end)) +j:queue(1,command()); assert(j:pending() and j.executed==1) +''') diff --git a/tests/test_engine.py b/tests/test_engine.py index a0e3d27..2da5eba 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -50,7 +50,9 @@ def test_full_ring_rejects_write_and_pending_slot_requires_native_completion(sel bytes[slot]=10 engine.schedule=function() bytes[slot]=1 end engine:scheduleCommand(command()); assert(engine:commandsPending()) -bytes[slot]=10; assert(not engine:commandsPending()) +bytes[slot]=10; assert(engine:commandsPending()) -- native state alone is not execution proof +local entry=engine.journal:before(0,command()); engine.journal:after(0,entry) +assert(not engine:commandsPending()) ''') def test_native_size_mismatch_clears_slot_and_guard(self): @@ -86,4 +88,67 @@ def test_multiplayer_cannot_enable_scope_or_be_paused_by_recorder(self): engine:pause(); assert(memory[engine.sites.paused]==0) assert(not pcall(function() engine:setScope(true) end)) engine:setScope(false); assert(memory[engine.scope]==0) +''') + + def command_hooks(self, mode): + self.check(''' +core.hookCode=function() return function() return 0 end end +recorder={active=true,status='recording',mode='record',manifest={player=1,variant='SHC'},commands={}} +function recorder:guard(f) + local ok,reason=pcall(f); if not ok then self.status='error'; self.error=reason end + return ok +end +function recorder:onExecutedCommand(c) self.commands[#self.commands+1]=c end +engine:install(recorder) +address=engine.base+0x3c67c +memory[engine.base+0x2d824]=0; memory[address]=10; bytes[address+8]=28 +memory[engine.base+engine.sites.actorOffset]=1 +memory[0x1fe7da8]=10 +memory[engine.base+0x2d830]=1; bytes[engine.buffer]=1; bytes[address+10]=1 +function before() + return callbacks[engine.sites.execute.address]({ESI=engine.base,ECX=0,EDX=999}) +end +function after() callbacks[engine.sites.executed.address]({ESI=engine.base}) end +''') + if mode == 'play': + self.check("recorder.mode='play'; recorder.status='playing'; engine.journal:queue(0,command(10))") + + def test_capture_observes_execution_not_just_receipt(self): + self.command_hooks('record') + self.check(''' +callbacks[engine.sites.copySize.address]({ESI=engine.base,EDX=engine.buffer}) +assert(#recorder.commands==0) +memory[0x1fe7da8]=12 -- late delivery: record the actual execution boundary +assert(before().EDX==28 and #recorder.commands==0) +after() +assert(#recorder.commands==1 and recorder.commands[1].time==12 and recorder.commands[1].data=='01') +assert(not engine.received[0]) +''') + + def test_valid_playback_counts_only_returned_handlers(self): + self.command_hooks('play') + self.check(''' +assert(before().EDX==28 and engine.journal.executed==0 and engine:commandsPending()) +after(); assert(engine.journal.executed==1 and not engine:commandsPending()) +''') + + def test_bad_execution_becomes_noop_without_consuming_expected_command(self): + for corrupt in ("memory[engine.base+engine.sites.actorOffset]=2", "bytes[address+10]=2", + "memory[0x1fe7da8]=11", "memory[address]=9", "bytes[address+8]=39"): + with self.subTest(corrupt=corrupt): + self.setUp() + self.command_hooks('play') + self.check(corrupt) + self.check(''' +assert(before().EDX==0 and recorder.status=='error') +after(); assert(engine:commandsPending() and engine.journal.executed==0) +assert(before().EDX==0) -- later commands in this native batch are suppressed too +''') + + def test_unowned_command_is_rejected_and_multiplayer_dispatch_is_unchanged(self): + self.command_hooks('play') + self.check(''' +engine:resetCommands(); assert(before().EDX==0) +memory[engine.base+0x618]=1; bytes[address+8]=200 +assert(before().EDX==-56) -- original signed-byte conversion, no replay policy in MP ''') diff --git a/tests/test_sessions.py b/tests/test_sessions.py index f01ef36..4cd5125 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -21,6 +21,8 @@ def setUp(self): Session=require('code/session-recorder') now=0; snapshots=0; space=true engine={rng=0x1a279c0, + resetCommands=function(self) self.journal={executed=0} end, + journal={executed=0}, setScope=function(_,active) scoped=active end, singlePlayer=function() return true end, tick=function() return now end, @@ -46,11 +48,11 @@ def setUp(self): def test_arming_does_not_record_lobby_commands(self): self.check(''' local r=session(); r:startRecording() -r:onCommand(28,10,0,4) +r:onExecutedCommand(command(10)) assert(not r.active and snapshots==0 and r.manifest.commandCount==0) r:activateRecording() assert(r.active and snapshots==1 and r.manifest.status=='recording') -r:onCommand(28,10,0,4) +r:onExecutedCommand(command(10)) assert(r.manifest.commandCount==1) ''') @@ -131,7 +133,7 @@ def test_write_failure_closes_streams_and_marks_failed(self): local r=session(); r:startRecording(); r:activateRecording() local closed=0 r.commandsFile={write=function() return nil,'disk full' end,close=function() closed=closed+1; return true end} -assert(not r:guard(function() r:onCommand(28,10,0,4) end)) +assert(not r:guard(function() r:onExecutedCommand(command(10)) end)) assert(r.manifest.status=='failed' and not r.commandsFile and closed==1) r:reset(); assert(savedManifest.status=='failed') ''') From c6423c19bc52051f5c289dee7d749f02874b486a Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:34:21 +0200 Subject: [PATCH 10/19] Add explicit Automarket settings replay integration (0.7.0) --- CHANGELOG.md | 9 +++++ README.md | 6 ++-- code/automarket-replay.lua | 60 +++++++++++++++++++++++++++++++++ code/engine-sites.lua | 2 ++ code/engine.lua | 23 +++++++++++-- code/session-recorder.lua | 2 ++ code/sessions.lua | 1 + code/validation.lua | 7 +++- definition.yml | 2 +- docs/automarket-replay.md | 28 +++++++++++++++ docs/replay-sessions.md | 2 +- tests/test_automarket_replay.py | 49 +++++++++++++++++++++++++++ tests/test_engine.py | 46 +++++++++++++++++++++++++ 13 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 code/automarket-replay.lua create mode 100644 docs/automarket-replay.md create mode 100644 tests/test_automarket_replay.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cd2e00a..e7a57a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.7.0 + +- Add an explicit Automarket 1.1.0 adapter for settings commits through protocol 1.0.0. Validate the registered protocol number, payload size, owning player, fee and Boolean fields. Unknown custom protocols remain excluded. +- Stage replay parameters in protocol's native receive buffer and restore that buffer after success or failure. +- Preserve Map Extensions 1.0.0's save wrapper so the starting save follows its custom-state serialization path. Require recorder to load after protocol before modifying dispatch code. +- Include the Automarket adapter and protocol registration in replay compatibility checks. Weekly automatic trades continue through the simulation without duplicate replay injection. + +Validation: 73 automated tests, Lua syntax checks and original-executable checks for both variants. The adapter, custom save restoration and actual automated trades still require live testing; this is not a multiplayer replay release. + ## 0.6.0 - Capture timed commands when their native handlers return, including inputs delivered directly to the received-command scheduler. Record their actual execution tick and resolved actor. diff --git a/README.md b/README.md index a1f53fc..d92b857 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.6.0 verifies native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. +Version 0.7.0 verifies native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. + +Automarket 1.1.0 has an experimental replay adapter for its settings commits and native custom save section. Use protocol 1.0.0 and map-extensions 1.0.0, with **recorder after protocol in the extension order**. The normal weekly trades run in the simulation; they are not replayed as extra trades. Other custom protocols remain unsupported. See [Automarket replay notes](docs/automarket-replay.md). ## Testing this build @@ -30,4 +32,4 @@ python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.6.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.7.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/automarket-replay.lua b/code/automarket-replay.lua new file mode 100644 index 0000000..519cba9 --- /dev/null +++ b/code/automarket-replay.lua @@ -0,0 +1,60 @@ +-- An explicit adapter for the reviewed Automarket 1.1.0 wire format. +-- Weekly trades are deterministic simulation work and must not be injected twice. +local M={SIZE=272} +local function integer(value,lo,hi,label) + return require('code/validation').integer(value,lo,hi,label) +end + +function M.version(name) + for _,extension in ipairs(allActiveExtensions or {}) do + if extension.name==name then return extension.version end + end +end + +function M.saveHookAvailable() + return M.version('map-extensions')=='1.0.0' and modules and modules['map-extensions']~=nil +end + +function M.current() + local versions={} + for _,extension in ipairs(allActiveExtensions or {}) do versions[extension.name]=extension.version end + if not versions.automarket then return nil end + assert(versions.automarket=='1.1.0','Replay adapter requires Automarket 1.1.0') + assert(versions.protocol=='1.0.0' and versions['map-extensions']=='1.0.0', + 'Automarket replay adapter requires protocol and map-extensions 1.0.0') + local protocol=assert(modules and modules.protocol,'Automarket replay protocol is unavailable') + local id=protocol:getProtocolNumber('automarket','commitSingle') + integer(id,130,2147483647,'Automarket protocol number') + return {version='1.1.0',protocol=id} +end + +function M.descriptor(value) + if value==nil then return end + assert(type(value)=='table' and value.version=='1.1.0','Unsupported Automarket replay adapter') + integer(value.protocol,130,2147483647,'Automarket protocol number') +end + +function M.compatible(recorded) + local ok,current=pcall(M.current) + if not ok then return false end + if not recorded or not current then return recorded==current end + return recorded.version==current.version and recorded.protocol==current.protocol +end + +local function word(bytes,offset) + return bytes[offset+1]+bytes[offset+2]*256+bytes[offset+3]*65536+bytes[offset+4]*16777216 +end + +function M.command(command,descriptor) + M.descriptor(descriptor) + assert(descriptor,'Custom replay command has no Automarket adapter') + assert(command.size==M.SIZE,'Invalid Automarket replay payload size') + local bytes=require('code/utils').hexToTable(command.data) + assert(word(bytes,0)==descriptor.protocol,'Unknown custom replay protocol') + assert(word(bytes,4)==command.player,'Automarket replay payload belongs to another player') + integer(word(bytes,268),0,100,'Automarket fee') + integer(bytes[9],0,1,'Automarket enabled flag') + for i=17,66 do integer(bytes[i],0,1,'Automarket goods flag') end +end + +return M diff --git a/code/engine-sites.lua b/code/engine-sites.lua index 5ec9d9d..e22d0b0 100644 --- a/code/engine-sites.lua +++ b/code/engine-sites.lua @@ -11,6 +11,7 @@ return { execute = {address=0x0048937F, bytes={15,190,148,49,132,198,3,0}}, executed = {address=0x00489390, bytes={139,142,36,216,2,0}}, actorOffset = 0x00109E70, + receivedParameters = {address=0x004908BB, bytes={141,150,220,12,0,0,82,139,19}}, switchMenu = {address=0x0046B350, bytes={0,0,139,68,36}}, resources = 0x011BF130, packager = 0x00F2B3D0, @@ -31,6 +32,7 @@ return { execute = {address=0x0048948F, bytes={15,190,148,49,132,198,3,0}}, executed = {address=0x004894A0, bytes={139,142,36,216,2,0}}, actorOffset = 0x00166300, + receivedParameters = {address=0x00490A1B, bytes={141,150,220,12,0,0,82,139,19}}, switchMenu = {address=0x0046B570, bytes={0,0,139,68,36}}, resources = 0x01293C20, packager = 0x00F2B850, diff --git a/code/engine.lua b/code/engine.lua index 88888a5..5842496 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -11,11 +11,19 @@ end function M.verify() local sites = assert(allSites[native.profile.name]) + local adapter=require('code/automarket-replay') + if adapter.version('protocol') then + assert(core.readByte(sites.execute.address+8)==0xE9, + 'Enable recorder after protocol in the UCP extension order') + end for name, site in pairs(sites) do if type(site)=='table' then local actual=core.readBytes(site.address, #site.bytes) + -- map-extensions wraps this callable entry to include extension save data. + -- We call that entry, never bypass or overwrite its five-byte hook. + local wrappedSave=name=='save' and adapter.saveHookAvailable() and actual[1]==0xE9 for i, value in ipairs(site.bytes) do - assert(actual[i]==value, 'Recorder session hook conflicts at ' .. name) + assert((wrappedSave and i<=5) or actual[i]==value, 'Recorder session hook conflicts at ' .. name) end end end @@ -117,9 +125,20 @@ function M:scheduleCommand(command) local slot=core.readInteger(self.base+self.sites.writeIndexOffset) self.expectedSize=command.size self.copyError=nil + -- protocol 1.0.0's receive callback reads this native packet buffer even when + -- the scheduler was given a different pointer. Restore it after the copy. + local received=self.base+0xcdc + local oldReceived + if command.commandCategory==122 then + oldReceived=core.readBytes(received,1260) + core.writeBytes(received,bytes) + end temporarily(function() self.schedule(self.base,command.commandCategory,command.player,command.time,self.buffer) - end,function() self.expectedSize=nil end) + end,function() + if oldReceived then core.writeBytes(received,oldReceived) end + self.expectedSize=nil + end) if self.copyError then core.writeByte(self.base+0x3c67c+slot*1272+9,10) error(self.copyError) diff --git a/code/session-recorder.lua b/code/session-recorder.lua index 1a59562..ad9a19f 100644 --- a/code/session-recorder.lua +++ b/code/session-recorder.lua @@ -38,7 +38,9 @@ end function Session:startRecording() assert(self.mode=='none','A replay session is already active') assert(self.engine:singlePlayer(),'Recording currently supports single-player Skirmish') + local automarket=require('code/automarket-replay').current() self.manifest=store.new(native.profile) + self.manifest.automarket=automarket self:setName(store.path(self.manifest.id)..'/stream') self:openFiles('w') self.mode='record'; self.status='armed'; self.active=false diff --git a/code/sessions.lua b/code/sessions.lua index 1c896b7..177d4f2 100644 --- a/code/sessions.lua +++ b/code/sessions.lua @@ -163,6 +163,7 @@ end function M.compatible(manifest) local current=M.settings() return current.hash==manifest.settingsHash and current.environmentHash==manifest.environmentHash + and require('code/automarket-replay').compatible(manifest.automarket) end M.read, M.write = read, write diff --git a/code/validation.lua b/code/validation.lua index ee01e6e..a7267a5 100644 --- a/code/validation.lua +++ b/code/validation.lua @@ -41,8 +41,12 @@ for _,id in ipairs({14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29, function M.sessionCommand(command, manifest) M.command(command) - assert(gameplay[command.commandCategory], + if command.commandCategory==122 then + require('code/automarket-replay').command(command,manifest.automarket) + else + assert(gameplay[command.commandCategory], 'Unsupported replay command category '..command.commandCategory..' (save/load and network commands are excluded)') + end assert(command.commandCategory~=119 or manifest.variant=='Extreme','Tactical powers require Extreme') assert(command.player==manifest.player,'Replay command uses a different player slot') return command @@ -58,6 +62,7 @@ end function M.manifest(value) assert(type(value)=='table','Invalid replay manifest') + require('code/automarket-replay').descriptor(value.automarket) M.integer(value.player,1,8,'player slot') M.integer(value.startTick,0,2147483647,'starting tick') M.integer(value.lastTick,value.startTick,2147483647,'ending tick') diff --git a/definition.yml b/definition.yml index 2b32ace..6cd5db2 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.6.0 +version: 0.7.0 type: module author: Corax meta: diff --git a/docs/automarket-replay.md b/docs/automarket-replay.md new file mode 100644 index 0000000..58a8195 --- /dev/null +++ b/docs/automarket-replay.md @@ -0,0 +1,28 @@ +# Automarket replay adapter (0.7.0) + +This adapter targets Automarket 1.1.0, protocol 1.0.0 and map-extensions 1.0.0. Recorder must load after protocol: its post-dispatch observer changes bytes used by protocol's original signature scan. Startup checks that protocol's dispatch hook is already present before installing recorder hooks. Other extension versions are not silently assumed compatible. + +## Commands and ownership + +Automarket's `init.lua` registers `automarket.commitSingle` as a LOCKSTEP protocol. Its 272-byte packet contains the four-byte discriminator, four-byte controlling player, 260-byte AutoMarketPlayerData and four-byte committed fee. The adapter resolves the registered discriminator through protocol's public API, stores it in the manifest, and checks it again for playback. Payload length, discriminator, owning player, fee 0..100 and all Boolean fields are validated before native scheduling. Arbitrary category 122 payloads and immediate category 121 are not enabled. + +`protocols/lockstepProtocol.lua` uses `COMMAND_FIXED_RECEIVED_PARAMETER_LOCATION_ADDRESS` when inferring the received size. In both executables the address is GameSynchronyState + 0xcdc (LEA at Crusader 0x4908bb / Extreme 0x490a1b). Recorder's allocated payload buffer alone does not satisfy this callback. The adapter stages the padded packet into the native buffer for the call and restores all 1260 bytes afterwards, including on failure. The existing native inferred-size guard remains active. + +The native pre/post-dispatch journal captures the settings commit and verifies its execution order and actor. Automarket's `ui/market/process.lua` performs weekly trades directly during simulation; those trades are not extra player commands and must not be injected a second time. The queue guard prevents a spectator from committing new market settings during playback while leaving this weekly simulation callback alone. Other automation extensions that generate commands need separate ownership policies. + +## Starting state + +Automarket registers `automarketplayerdata.bin` with Map Extensions. It includes settings, fee and fractional credit. `mapextensions/game.lua` wraps FilePackager's write entry and substitutes an extended section array; `callbacks.lua` serializes registered data into that section. Loading follows the corresponding read wrapper. + +Recorder calls the public native write entry so this wrapper executes. Verification permits its five-byte jump only when Map Extensions 1.0.0 is active and the untouched sixth byte matches; other native sites retain exact checks. This is an explicit integration contract with an installed module, not proof that no other extension changed that jump. Recorder neither overwrites the wrapper nor calls its unwrapped trampoline. + +## Validation and remaining limits + +Automated tests cover malformed custom payloads, mismatched registrations/versions, failed callbacks, exact receive-buffer restoration, save-hook acceptance and load ordering. Both native profiles have original-binary checks. The following still require a live test in each game: + +1. Record a match with enabled buy/sell settings and nonzero fees; change settings mid-match and leave through Quit Mission. +2. Confirm the starting save restores settings, fee and fractional credit, then replay to completion twice. +3. Confirm weekly purchases/sales and resource totals match, including rounding/refunds and full storage. +4. Try committing new settings during playback and confirm no extra trade/settings action occurs. + +RNG and command checks do not prove complete world-state equality. Version checks do not fingerprint edited extension files. Protocol 1.0.0 logs some handler failures internally instead of propagating them; returning from dispatch proves the call happened, not that the extension accepted every action. These limits remain visible rather than being called multiplayer compatibility. diff --git a/docs/replay-sessions.md b/docs/replay-sessions.md index e3f0bfc..5af3cf4 100644 --- a/docs/replay-sessions.md +++ b/docs/replay-sessions.md @@ -30,7 +30,7 @@ Version equality does not prove that unpacked extensions have identical source f ## Unsupported cases -Multiplayer capture/playback, resynchronization packets, lobby commands, native save/load commands, unknown extension commands and cross-variant playback are excluded. Unsupported commands during recording fail the capture with an error instead of silently omitting actions. Game speed and pause behaviour, automated actions, native save/load RNG side effects, repeated playback, match-end transitions and Extreme matches still require live verification. +Multiplayer capture/playback, resynchronization packets, lobby commands, native save/load commands, unknown extension commands (except the explicit Automarket 1.1.0 adapter) and cross-variant playback are excluded. Unsupported commands during recording fail the capture with an error instead of silently omitting actions. Game speed and pause behaviour, automated actions, native save/load RNG side effects, repeated playback, match-end transitions and Extreme matches still require live verification. ## Native evidence diff --git a/tests/test_automarket_replay.py b/tests/test_automarket_replay.py new file mode 100644 index 0000000..6b310cd --- /dev/null +++ b/tests/test_automarket_replay.py @@ -0,0 +1,49 @@ +import unittest +import test_recorder as fixture + + +class AutomarketReplayTests(unittest.TestCase): + check = fixture.RecorderTests.check + + def setUp(self): + fixture.RecorderTests.setUp(self) + self.check(''' +adapter=require('code/automarket-replay'); validation=require('code/validation') +allActiveExtensions={{name='automarket',version='1.1.0'},{name='protocol',version='1.0.0'}, + {name='map-extensions',version='1.0.0'}} +modules={protocol={getProtocolNumber=function(_,extension,name) + assert(extension=='automarket' and name=='commitSingle'); return 131 +end},['map-extensions']={}} +function marketCommand() + local payload={}; for i=1,272 do payload[i]=0 end + payload[1]=131; payload[5]=1; payload[9]=1; payload[17]=1; payload[269]=25 + return {commandCategory=122,time=10,player=1,size=272,data=require('code/utils').tableToHex(payload)} +end +manifest={player=1,variant='SHC',automarket=adapter.current()} +''') + + def test_known_commit_requires_matching_registration_and_versions(self): + self.check(''' +assert(pcall(validation.sessionCommand,marketCommand(),manifest)) +assert(adapter.compatible(manifest.automarket)) +modules.protocol.getProtocolNumber=function() return 132 end +assert(not adapter.compatible(manifest.automarket)) +allActiveExtensions[1].version='1.0.0' +assert(not pcall(adapter.current)) +allActiveExtensions={}; assert(not adapter.compatible(manifest.automarket)) +assert(adapter.compatible(nil)) +''') + + def test_unknown_custom_protocol_invalid_actor_fee_flags_and_sizes_fail(self): + self.check(''' +for _,change in ipairs({{1,132},{5,2},{269,101},{272,255},{9,2},{17,2},{66,2}}) do + local c=marketCommand(); local payload=require('code/utils').hexToTable(c.data) + payload[change[1]]=change[2]; c.data=require('code/utils').tableToHex(payload) + assert(not pcall(validation.sessionCommand,c,manifest)) +end +local c=marketCommand(); c.commandCategory=121 +assert(not pcall(validation.sessionCommand,c,manifest)) +c=marketCommand(); c.size=271; c.data=c.data:sub(1,-3) +assert(not pcall(validation.sessionCommand,c,manifest)) +assert(not pcall(validation.sessionCommand,marketCommand(),{player=1,variant='SHC'})) +''') diff --git a/tests/test_engine.py b/tests/test_engine.py index 2da5eba..689ddae 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -151,4 +151,50 @@ def test_unowned_command_is_rejected_and_multiplayer_dispatch_is_unchanged(self) engine:resetCommands(); assert(before().EDX==0) memory[engine.base+0x618]=1; bytes[address+8]=200 assert(before().EDX==-56) -- original signed-byte conversion, no replay policy in MP +''') + + def test_custom_receive_buffer_is_staged_and_restored_on_success_and_failure(self): + self.check(''' +local address=engine.base+0xcdc +for i=0,1260 do bytes[address+i]=42 end +local c=command(10); c.commandCategory=122; c.size=272; c.data='83000000'..string.rep('00',268) +for _,fail in ipairs({false,true}) do + engine:resetCommands() + engine.schedule=function() + assert(bytes[address]==131 and bytes[address+1]==0 and bytes[address+1259]==0) + assert(bytes[address+1260]==42) + if fail then error('receive callback failed') end + end + local ok=pcall(function() engine:scheduleCommand(c) end) + assert(ok~=fail and not engine.expectedSize) + for i=0,1260 do assert(bytes[address+i]==42) end + assert(engine:commandsPending()==not fail) +end +''') + + def test_save_wrapper_acceptance_requires_known_extension_and_preserved_tail(self): + self.check(''' +realNative.profile.name='SHC' +for _,site in pairs(engine.sites) do + if type(site)=='table' then core.writeBytes(site.address,site.bytes) end +end +assert(Engine.verify()) +bytes[engine.sites.save.address]=0xE9 +assert(not pcall(Engine.verify)) +allActiveExtensions={{name='map-extensions',version='1.0.0'}}; modules={['map-extensions']={}} +assert(Engine.verify()) +bytes[engine.sites.save.address+5]=0 +assert(not pcall(Engine.verify)) +''') + + def test_protocol_must_install_its_dispatch_before_recorder(self): + self.check(''' +realNative.profile.name='SHC' +for _,site in pairs(engine.sites) do + if type(site)=='table' then core.writeBytes(site.address,site.bytes) end +end +allActiveExtensions={{name='protocol',version='1.0.0'}} +local ok,reason=pcall(Engine.verify) +assert(not ok and tostring(reason):find('after protocol',1,true)) +bytes[engine.sites.execute.address+8]=0xE9; assert(Engine.verify()) ''') From dd3f173263db76c1d68558b39b0fc97eedfdcc6d Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:37:47 +0200 Subject: [PATCH 11/19] Stop invalid captured lengths before the native payload copy --- CHANGELOG.md | 2 +- code/engine.lua | 3 ++- tests/test_engine.py | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7a57a9..79865d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Preserve Map Extensions 1.0.0's save wrapper so the starting save follows its custom-state serialization path. Require recorder to load after protocol before modifying dispatch code. - Include the Automarket adapter and protocol registration in replay compatibility checks. Weekly automatic trades continue through the simulation without duplicate replay injection. -Validation: 73 automated tests, Lua syntax checks and original-executable checks for both variants. The adapter, custom save restoration and actual automated trades still require live testing; this is not a multiplayer replay release. +Validation: 74 automated tests, Lua syntax checks and original-executable checks for both variants. The adapter, custom save restoration and actual automated trades still require live testing; this is not a multiplayer replay release. ## 0.6.0 diff --git a/code/engine.lua b/code/engine.lua index 5842496..76e3404 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -204,7 +204,7 @@ function M:install(recorder) end registers.EAX=size if recorder.active and recorder.status=='recording' and self:singlePlayer() then - recorder:guard(function() + local ok=recorder:guard(function() require('code/validation').integer(size,0,1260,'native payload size') local slot=core.readInteger(self.base+0x2d824) require('code/validation').integer(slot,0,199,'native ring slot') @@ -212,6 +212,7 @@ function M:install(recorder) self.received[slot]={command={commandCategory=core.readByte(address+8),time=core.readInteger(address), size=size,data=require('code/utils').tableToHex(core.readBytes(registers.EDX,size))}} end) + if not ok then registers.EAX=0 end -- do not copy an invalid native length after stopping end return registers end,self.sites.copySize.address,6) diff --git a/tests/test_engine.py b/tests/test_engine.py index 689ddae..1417fea 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -123,6 +123,14 @@ def test_capture_observes_execution_not_just_receipt(self): after() assert(#recorder.commands==1 and recorder.commands[1].time==12 and recorder.commands[1].data=='01') assert(not engine.received[0]) +''') + + def test_invalid_captured_size_cannot_reach_native_copy(self): + self.command_hooks('record') + self.check(''' +memory[engine.base+0x2d830]=1261 +local result=callbacks[engine.sites.copySize.address]({ESI=engine.base,EDX=engine.buffer}) +assert(result.EAX==0 and recorder.status=='error' and not engine.received[0]) ''') def test_valid_playback_counts_only_returned_handlers(self): From e6eaa4f5bc077477ab2285b4d87f4f47ed425837 Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:15:56 +0200 Subject: [PATCH 12/19] Automatically publish separately testable PR prereleases --- .github/workflows/publish-release.yml | 109 +++++++++++++++++++++++++ docs/pr-releases.md | 11 +++ tools/pr_releases.py | 111 ++++++++++++++++++++++++++ 3 files changed, 231 insertions(+) create mode 100644 .github/workflows/publish-release.yml create mode 100644 docs/pr-releases.md create mode 100644 tools/pr_releases.py diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml new file mode 100644 index 0000000..b895fac --- /dev/null +++ b/.github/workflows/publish-release.yml @@ -0,0 +1,109 @@ +name: Publish PR releases +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + schedule: + - cron: '*/5 * * * *' + push: + branches: [main] + workflow_dispatch: + inputs: + pr: + description: PR number to backfill, or empty for all open PRs + required: false + type: string +permissions: + contents: read +concurrency: + group: recorder-pr-releases + cancel-in-progress: false +env: + SOURCE_REPO: Corax34/ucp_recorder +jobs: + discover: + runs-on: ubuntu-latest + outputs: + builds: ${{ steps.prs.outputs.builds }} + tests: ${{ steps.prs.outputs.tests }} + count: ${{ steps.prs.outputs.count }} + steps: + # Publisher code always comes from the trusted default branch. + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - id: prs + env: + GH_TOKEN: ${{ github.token }} + REQUESTED_PR: ${{ inputs.pr || github.event.pull_request.number }} + run: python tools/pr_releases.py discover + verify: + needs: discover + if: needs.discover.outputs.count != '0' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.discover.outputs.tests) }} + runs-on: ${{ matrix.os }} + # PR code runs only here, without write permissions, secrets or shared cache. + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ matrix.repo }} + ref: ${{ matrix.sha }} + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install lupa==2.6 unicorn==2.1.4 + - run: python -m unittest discover -s tests -v + package: + needs: [discover, verify] + if: needs.discover.outputs.count != '0' + strategy: + matrix: ${{ fromJSON(needs.discover.outputs.builds) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + path: publisher + persist-credentials: false + - uses: actions/checkout@v4 + with: + repository: ${{ matrix.repo }} + ref: ${{ matrix.sha }} + path: source + persist-credentials: false + - name: Package clean source with the trusted builder + env: + PR_BUILD: ${{ toJSON(matrix) }} + run: python publisher/tools/pr_releases.py package + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.tag }} + path: out/* + if-no-files-found: error + publish: + needs: [discover, package] + if: needs.discover.outputs.count != '0' + strategy: + matrix: ${{ fromJSON(needs.discover.outputs.builds) }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: ${{ matrix.tag }} + path: out + - name: Publish immutable PR test release + env: + GH_TOKEN: ${{ github.token }} + PR_BUILD: ${{ toJSON(matrix) }} + run: python tools/pr_releases.py publish diff --git a/docs/pr-releases.md b/docs/pr-releases.md new file mode 100644 index 0000000..65dbaa2 --- /dev/null +++ b/docs/pr-releases.md @@ -0,0 +1,11 @@ +# Downloadable builds for each PR + +`Publish PR releases` creates a separate GitHub prerelease for every published PR head from this project's maintained repositories. Each title names the PR, module version and commit; each release contains the installable `recorder-VERSION.zip` and its SHA-256 checksum. Updating a PR creates another immutable release, preserving older test results and downloads. + +PR opened, updated, reopened and ready-for-review events trigger the workflow once it is installed on the target repository's default branch. Until upstream accepts the automation, the author's fork polls the upstream PR list every five minutes; GitHub may delay scheduled runs. Manual dispatch accepts a PR number for backfilling, or an empty value for all open PRs. Existing releases are skipped. + +The workflow tests each selected commit on Windows and Linux with read-only permissions. A separate job packages clean source using the publisher script from the default branch. The write-enabled publishing job verifies the artifact identity and checksum and never executes PR code or its build scripts. No shared cache or secret is passed to PR tests. The current repository allowlist covers upstream and Krarilotus's fork; expanding publication to other contributors is an explicit maintenance decision. + +The fork publishes its own releases; it cannot install workflows or publish releases in Corax's repository. Links from the upstream PRs point to these fork test builds. Publishing does not merge a PR or certify live replay/multiplayer behaviour. Read the README and changelog inside each version, since earlier stages have fewer features and safeguards. + +For a backfill batch, all selected Windows/Linux test jobs must pass before packaging begins. A failed test keeps that batch unpublished and is visible in Actions; dispatching one PR permits an independent retry. diff --git a/tools/pr_releases.py b/tools/pr_releases.py new file mode 100644 index 0000000..417adae --- /dev/null +++ b/tools/pr_releases.py @@ -0,0 +1,111 @@ +"""Publish immutable PR packages, without executing PR build code.""" +from pathlib import Path +import hashlib +import json +import os +import re +import subprocess +import sys +import zipfile + + +def gh(*args): + return subprocess.check_output(['gh', *args], text=True, encoding='utf-8') + + +def output(**values): + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as stream: + for key, value in values.items(): + stream.write(f'{key}={json.dumps(value, separators=(",", ":"))}\n') + + +def discover(): + source = os.environ['SOURCE_REPO'] + publisher = os.environ['GITHUB_REPOSITORY'] + requested = os.environ.get('REQUESTED_PR', '') + if requested: + assert requested.isdecimal(), 'PR must be a number' + pulls = [json.loads(gh('api', f'repos/{source}/pulls/{requested}'))] + else: + pages = json.loads(gh('api', f'repos/{source}/pulls?state=open&per_page=100', '--paginate', '--slurp')) + pulls = [pr for page in pages for pr in page] + pages = json.loads(gh('api', f'repos/{publisher}/releases?per_page=100', '--paginate', '--slurp')) + existing = {release['tag_name'] for page in pages for release in page if not release['draft']} + builds = [] + for pr in pulls: + head = pr['head'] + if pr['draft'] or not head['repo'] or head['repo']['full_name'] not in { + publisher, source, 'Krarilotus/ucp_recorder' + }: + continue + sha = head['sha'] + assert re.fullmatch('[0-9a-f]{40}', sha) + tag = f'pr-{pr["number"]}-{sha[:12]}' + if tag not in existing: + builds.append(dict(pr=pr['number'], sha=sha, repo=head['repo']['full_name'], tag=tag)) + tests = [dict(build, os=runner) for build in builds for runner in ('ubuntu-latest', 'windows-latest')] + output(builds={'include': builds}, tests={'include': tests}, count=len(builds)) + print(f'{len(builds)} PR releases need publication') + + +def package(): + build = json.loads(os.environ['PR_BUILD']) + source = Path('source').resolve() + destination = Path('out') + destination.mkdir(exist_ok=True) + definition = (source / 'definition.yml').read_text(encoding='utf-8') + assert re.search(r'^name: recorder\s*$', definition, re.M), 'Unexpected module identity' + version = re.search(r'^version: (\d+\.\d+\.\d+)\s*$', definition, re.M).group(1) + asset = destination / f'recorder-{version}.zip' + files = [source / name for name in ('definition.yml', 'options.yml', 'init.lua', 'README.md', 'CHANGELOG.md')] + files += sorted((source / 'code').rglob('*.lua')) + files += sorted((source / 'docs').rglob('*.md')) + with zipfile.ZipFile(asset, 'w', zipfile.ZIP_DEFLATED) as archive: + for path in files: + assert path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(source) + archive.write(path, path.relative_to(source).as_posix()) + checksum = hashlib.sha256(asset.read_bytes()).hexdigest() + asset.with_suffix('.zip.sha256').write_text(f'{checksum} {asset.name}\n', encoding='utf-8') + (destination / 'build.json').write_text(json.dumps(dict(build, version=version, sha256=checksum)), encoding='utf-8') + + +def publish(): + build = json.loads(os.environ['PR_BUILD']) + metadata = json.loads(Path('out/build.json').read_text(encoding='utf-8')) + assert all(metadata[key] == value for key, value in build.items()), 'Artifact belongs to another PR/commit' + version = metadata['version'] + assert re.fullmatch(r'\d+\.\d+\.\d+', version) + asset = Path('out') / f'recorder-{version}.zip' + digest = hashlib.sha256(asset.read_bytes()).hexdigest() + assert digest == metadata['sha256'], 'Artifact checksum differs' + with zipfile.ZipFile(asset) as archive: + assert archive.testzip() is None + definition = archive.read('definition.yml').decode('utf-8') + assert re.search(rf'^version: {re.escape(version)}\s*$', definition, re.M) + changelog = archive.read('CHANGELOG.md').decode('utf-8') + asset.with_suffix('.zip.sha256').write_text(f'{digest} {asset.name}\n', encoding='utf-8') + section = re.search(rf'^## {re.escape(version)}\s*\n(.*?)(?=^## |\Z)', changelog, re.M | re.S) + source = os.environ['SOURCE_REPO'] + notes = f'''Experimental test build for [{source} PR #{build['pr']}](https://github.com/{source}/pull/{build['pr']}). + +**Exact commit:** `{build['sha']}`. **Module version:** `{version}`. + +Download **{asset.name}** below. GitHub's source archives are not installable modules. This PR includes preceding stacked changes; test the PR stages in order. Later updates produce a separate release for the new commit. + +Use a separate UCP3 developer-mode installation. Preserve Graphics API Replacer and its dependencies if required. Consult the packaged README for the features and installation instructions of this exact stage. Later stages require recorder after protocol in extension order. + +Automated tests passed on Windows and Linux. No live game is launched by this workflow. These builds are not certified for complete replay or multiplayer playback; earlier stages deliberately have fewer features and safeguards. Publishing a fork prerelease does not merge or approve its upstream PR. + +## Changelog for this stage + +{section.group(1).strip() if section else 'See CHANGELOG.md in the module archive.'} +''' + body = Path('out/release-notes.md') + body.write_text(notes, encoding='utf-8') + print(gh('release', 'create', build['tag'], str(asset), str(asset.with_suffix('.zip.sha256')), + '--repo', os.environ['GITHUB_REPOSITORY'], '--target', build['sha'], '--prerelease', + '--title', f'PR #{build["pr"]} - Recorder {version} - {build["sha"][:12]}', '--notes-file', str(body))) + + +if __name__ == '__main__': + {'discover': discover, 'package': package, 'publish': publish}[sys.argv[1]]() From d8f60e9a5b292fd6fa03ef0b0a9391aa1eb5e3c5 Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:28:15 +0200 Subject: [PATCH 13/19] Verify replay resource state for all eight players (0.8.0) --- CHANGELOG.md | 9 +++++++++ README.md | 4 ++-- code/engine-sites.lua | 4 ++++ code/engine.lua | 10 ++++++++++ code/session-recorder.lua | 25 +++++++++++++++++++++++-- code/sessions.lua | 3 ++- code/validation.lua | 7 +++++++ definition.yml | 2 +- docs/replay-sessions.md | 4 ++-- docs/resource-verification.md | 9 +++++++++ tests/test_engine.py | 16 ++++++++++++++++ tests/test_recorder.py | 1 + tests/test_session_files.py | 19 ++++++++++++++++++- tests/test_sessions.py | 35 ++++++++++++++++++++++++++++++++--- 14 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 docs/resource-verification.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 79865d5..85bb78b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.8.0 + +- Verify all 25 native resource slots for each of the eight players when loading the starting save, at periodic checkpoints and at the ending boundary. +- Detect wrong gold/stockpile totals even when commands and RNG values still match; report the first differing player, resource, expected amount and actual amount. +- Validate resource evidence before loading a replay, and preserve it in the completion report. +- Use simulation profile `recorder-sp-v4`; earlier experimental captures require their original version. + +Validation: 79 automated tests and native operand checks for Crusader and Extreme. This adds resource-state verification, not a complete world-state checksum or multiplayer playback. Live replay tests remain pending. + ## 0.7.0 - Add an explicit Automarket 1.1.0 adapter for settings commits through protocol 1.0.0. Validate the registered protocol number, payload size, owning player, fee and Boolean fields. Unknown custom protocols remain excluded. diff --git a/README.md b/README.md index d92b857..2fcc941 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.7.0 verifies native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. +Version 0.8.0 verifies player resources and native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. Automarket 1.1.0 has an experimental replay adapter for its settings commits and native custom save section. Use protocol 1.0.0 and map-extensions 1.0.0, with **recorder after protocol in the extension order**. The normal weekly trades run in the simulation; they are not replayed as extra trades. Other custom protocols remain unsupported. See [Automarket replay notes](docs/automarket-replay.md). @@ -32,4 +32,4 @@ python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.7.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.8.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/engine-sites.lua b/code/engine-sites.lua index e22d0b0..b0a5c0f 100644 --- a/code/engine-sites.lua +++ b/code/engine-sites.lua @@ -11,6 +11,8 @@ return { execute = {address=0x0048937F, bytes={15,190,148,49,132,198,3,0}}, executed = {address=0x00489390, bytes={139,142,36,216,2,0}}, actorOffset = 0x00109E70, + playerResources = 0x0115C2C8, + resourceReset = {address=0x0040C334, bytes={137,20,157,200,194,21,1,131,192,1,131,248,25}}, receivedParameters = {address=0x004908BB, bytes={141,150,220,12,0,0,82,139,19}}, switchMenu = {address=0x0046B350, bytes={0,0,139,68,36}}, resources = 0x011BF130, @@ -32,6 +34,8 @@ return { execute = {address=0x0048948F, bytes={15,190,148,49,132,198,3,0}}, executed = {address=0x004894A0, bytes={139,142,36,216,2,0}}, actorOffset = 0x00166300, + playerResources = 0x011EEF08, + resourceReset = {address=0x0040C344, bytes={137,20,157,8,239,30,1,131,192,1,131,248,25}}, receivedParameters = {address=0x00490A1B, bytes={141,150,220,12,0,0,82,139,19}}, switchMenu = {address=0x0046B570, bytes={0,0,139,68,36}}, resources = 0x01293C20, diff --git a/code/engine.lua b/code/engine.lua index 76e3404..1d84dc3 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -67,6 +67,16 @@ function M:rngState() core.readInteger(self.rng+0x9c48),core.readInteger(self.rng+0x9c4c)} end +function M:resourceState() + local values={} + for player=1,8 do + for resource=0,24 do + values[#values+1]=core.readInteger(self.sites.playerResources+player*0x39f4+resource*4) + end + end + return values +end + function M:saveSnapshot(path) assert(#path<500 and self:singlePlayer(), 'Snapshot requires a single-player game') local resource=self.sites.resources diff --git a/code/session-recorder.lua b/code/session-recorder.lua index ad9a19f..5a4b6ca 100644 --- a/code/session-recorder.lua +++ b/code/session-recorder.lua @@ -61,6 +61,8 @@ function Session:activateRecording() self.manifest.player=self.engine:player() self.manifest.startTick=self.engine:tick() self.manifest.lastTick=self.manifest.startTick + self.manifest.startResources=self.engine:resourceState() + self.manifest.finalResources=self.manifest.startResources local r=self.engine:rngState() local seed=core.readInteger(self.engine.rng+4) self:saveInfo(0,seed,seed,r[1],r[2],r[4],r[3]) @@ -98,6 +100,7 @@ function Session:startPlayback(id) core.writeBytes(self.engine.rng,bytes) assert(self.engine:tick()==manifest.startTick,'Loaded save has a different starting tick') assert(self.engine:player()==manifest.player,'Loaded save has a different player slot') + self:checkResources(manifest.startResources,'starting save') self.active=true; self.status='playing'; self.playedCommands=0 self.nextCheckpoint=nil print('Playing '..id) @@ -133,9 +136,10 @@ function Session:onTick() if self.status=='recording' then self.manifest.lastTick=now self.manifest.finalRng=self.engine:rngState() + self.manifest.finalResources=self.engine:resourceState() self.observedTick=true if now%64==0 then - local line=json:encode({time=now,rng=self.engine:rngState()}) + local line=json:encode({time=now,rng=self.engine:rngState(),resources=self.manifest.finalResources}) assert(self.rngFile:write(line..'\n')); assert(self.rngFile:flush()) end elseif self.status=='playing' then @@ -153,6 +157,7 @@ function Session:onTick() error('RNG divergence at tick '..now..' (field '..i..')') end end + self:checkResources(expected.resources,'checkpoint') end if now>=self.manifest.lastTick then assert(self.playedCommands==self.manifest.commandCount,'Replay ended before all commands were scheduled') @@ -160,9 +165,25 @@ function Session:onTick() assert(self.engine.journal.executed==self.manifest.commandCount,'Replay native execution count differs') local actual=self.engine:rngState() for i=1,4 do assert(actual[i]==self.manifest.finalRng[i],'Final RNG state differs at tick '..now) end + self:checkResources(self.manifest.finalResources,'ending state') self.status='finished'; core.writeInteger(self.halt,1); self.engine:pause() store.write(store.path(self.manifest.id)..'/last-playback.json',json:encode({ - status='finished',lastTick=now,commands=self.playedCommands,rngCheckpoints='matched'})) + status='finished',lastTick=now,commands=self.playedCommands,rngCheckpoints='matched',resourceCheckpoints='matched'})) + end + end +end + +function Session:checkResources(expected,phase) + validation.resources(expected) + local actual=self.engine:resourceState() + for i=1,200 do + if actual[i]~=expected[i] then + local player=math.floor((i-1)/25)+1 + local resource=(i-1)%25 + self.firstDesync={kind='resources',time=self.engine:tick(),phase=phase, + player=player,resource=resource,expected=expected[i],actual=actual[i]} + store.write(store.path(self.manifest.id)..'/desync.json',json:encode(self.firstDesync)) + error('Resource divergence at tick '..self.engine:tick()..' (player '..player..', resource '..resource..')') end end end diff --git a/code/sessions.lua b/code/sessions.lua index 177d4f2..43f78a1 100644 --- a/code/sessions.lua +++ b/code/sessions.lua @@ -1,6 +1,6 @@ local platform = require('code/platform') local validation = require('code/validation') -local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v3'} +local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v4'} local activeSettings local function read(path) @@ -154,6 +154,7 @@ function M.preflight(manifest) local checkpoint=json:decode(line) assert(type(checkpoint)=='table' and checkpoint.time==tick and tick<=manifest.lastTick,'Invalid replay checkpoint timeline') validation.rng(checkpoint.rng) + validation.resources(checkpoint.resources) tick=tick+64 end assert(tick>manifest.lastTick,'Replay verification data ended early') diff --git a/code/validation.lua b/code/validation.lua index a7267a5..f685267 100644 --- a/code/validation.lua +++ b/code/validation.lua @@ -68,6 +68,8 @@ function M.manifest(value) M.integer(value.lastTick,value.startTick,2147483647,'ending tick') M.integer(value.commandCount,0,2147483647,'command count') M.rng(value.finalRng) + M.resources(value.startResources) + M.resources(value.finalResources) for _,key in ipairs({'settingsHash','environmentHash','snapshotHash','rngHash','commandsHash','checkpointsHash','infoHash'}) do assert(type(value[key])=='string' and #value[key]==64 and not value[key]:find('[^%x]'), 'Invalid replay '..key) @@ -75,4 +77,9 @@ function M.manifest(value) return value end +function M.resources(values) + assert(type(values)=='table' and #values==200,'Invalid replay resource state') + for i=1,200 do M.integer(values[i],-2147483648,2147483647,'resource amount') end +end + return M diff --git a/definition.yml b/definition.yml index 6cd5db2..46dd79a 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.7.0 +version: 0.8.0 type: module author: Corax meta: diff --git a/docs/replay-sessions.md b/docs/replay-sessions.md index 5af3cf4..db635d8 100644 --- a/docs/replay-sessions.md +++ b/docs/replay-sessions.md @@ -10,7 +10,7 @@ Each session reserves `ucp/replays/YYYYMMDD-HHMMSS-NNNN/`. Its manifest records - `rng.bin`: the complete 40,016-byte RNG structure, including its table. - `ucp-config.yml` and `environment.json`: startup settings and resolved environment. - `stream-commands.json`, `stream-rng-sync.json`, `stream-infself.json`: commands, 64-tick RNG checkpoints and initial match information. -- `manifest.json`: status, timeline, command count, final RNG values/indices and SHA-256 hashes. +- `manifest.json`: status, timeline, command count, final RNG values/indices, starting/final player resource arrays and SHA-256 hashes. The manifest moves through armed, recording and complete. Cancelled and failed sessions cannot play. Stream close failures and validation failures prevent completion. Metadata is replaced through a temporary file; this prevents a partial JSON document becoming the published manifest, but does not promise recovery from every power-loss scenario. @@ -18,7 +18,7 @@ The manifest moves through armed, recording and complete. Cancelled and failed s Capture starts after the native Skirmish launch function initializes the game. Checkpoints observe the same point immediately before each simulation tick, independent of render-frame count. The ending state is the last observed boundary, not the later time at which an exit dialog runs. Future commands still queued when recording ends are excluded from the finalized stream. -Playback preflights all files, loads the native starting save, restores the full RNG structure, and checks the loaded tick and local player. The native scheduler receives recorded commands with their recorded execution tick. It must have a free slot; its inferred payload length must match the recorded length. At completion, all recorded commands must have been scheduled and observed returning from their native handlers. Ownership, dispatch order, actor, tick, category and payload are checked before execution. RNG verification covers two values and two indices, **not a full world-state checksum**. +Playback preflights all files, loads the native starting save, restores the full RNG structure, and checks the loaded tick and local player. The native scheduler receives recorded commands with their recorded execution tick. It must have a free slot; its inferred payload length must match the recorded length. At completion, all recorded commands must have been scheduled and observed returning from their native handlers. Ownership, dispatch order, actor, tick, category and payload are checked before execution. Verification covers two RNG values, two RNG indices and all 25 resource integers for players 1 through 8. Resource checks run after loading the starting save, at the periodic checkpoints and at the final boundary. This is **not a full world-state checksum**: units, buildings, economy variables outside these resource slots and private extension state need additional coverage. The local single-player identity stays intact. New commands queued by a spectator are blocked while playback is active, finished or failed. This hook may also block commands from automation extensions; compatibility with those extensions requires separate analysis and testing. diff --git a/docs/resource-verification.md b/docs/resource-verification.md new file mode 100644 index 0000000..be146ef --- /dev/null +++ b/docs/resource-verification.md @@ -0,0 +1,9 @@ +# Resource verification (0.8.0) + +Command execution and RNG equality can miss an incorrect trade or resource refund. Capture now snapshots the 25 signed resource integers of each real player (slots 1..8), including gold, and compares those integers during replay. Slot 0, UI buffers, padding and unrelated structure fields are excluded. No native hash function or RNG call is invoked by the observer. + +Automarket 1.1.0's `ui/market/init.lua` derives the resource base from the native resource-reset instruction and uses a player stride of 0x39f4. Original executable checks confirm the operand at Crusader 0x40c334 references 0x115c2c8, and Extreme 0x40c344 references 0x11eef08. The same 25-slot layout and stride apply to both profiles. The observer reads these addresses; it does not patch the reset function. + +The manifest stores the starting and final arrays. Each 64-tick checkpoint includes an array alongside its RNG values. Before playback starts, malformed or missing arrays fail preflight; after native loading, the actual resources must match the starting array. During playback the first mismatch writes desync.json with the tick, phase, player, zero-based resource ID and expected/actual amounts, then halts the session. Completion between checkpoint ticks still checks the final array. + +Tests cover all eight player strides in both profiles, signed values, detached snapshots, incorrect gold with matching RNG, the last resource of player 8, starting-save mismatch and missing/malformed stream evidence. All native operand checks are read-only. Full match playback and Automarket credit/refund behaviour remain live-test requirements. Unit/building state and private extension state are still outside this check. diff --git a/tests/test_engine.py b/tests/test_engine.py index 1417fea..a5fdb2e 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -205,4 +205,20 @@ def test_protocol_must_install_its_dispatch_before_recorder(self): local ok,reason=pcall(Engine.verify) assert(not ok and tostring(reason):find('after protocol',1,true)) bytes[engine.sites.execute.address+8]=0xE9; assert(Engine.verify()) +''') + + def test_resource_snapshot_covers_eight_players_and_ignores_ui_slot_zero(self): + self.check(''' +for _,sites in pairs(require('code/engine-sites')) do + local e=Engine.new(sites) + for player=0,8 do + for resource=0,24 do memory[sites.playerResources+player*0x39f4+resource*4]=player*1000+resource end + end + local state=e:resourceState(); assert(#state==200) + for player=1,8 do + for resource=0,24 do assert(state[(player-1)*25+resource+1]==player*1000+resource) end + end + memory[sites.playerResources+0x39f4+15*4]=-123 + assert(e:resourceState()[16]==-123 and state[16]==1015) +end ''') diff --git a/tests/test_recorder.py b/tests/test_recorder.py index 5ca50c6..f671802 100644 --- a/tests/test_recorder.py +++ b/tests/test_recorder.py @@ -59,6 +59,7 @@ def setUp(self): files[r.infoFileName]={{gameType=0,mapSeed=1,matchSeed=1,RNGvalue1=1,RNGvalue2=1,RNGindex1=1,RNGindex2=1}} return r end +function resourceState(value) local t={}; for i=1,200 do t[i]=value or 0 end; return t end function command(tick) return {time=tick or 1,commandCategory=28,player=1,size=1,data='01'} end function allClosed() for _,h in ipairs(handles) do assert(h.closed) end end ''') diff --git a/tests/test_session_files.py b/tests/test_session_files.py index f578304..68f02cf 100644 --- a/tests/test_session_files.py +++ b/tests/test_session_files.py @@ -53,14 +53,16 @@ def mkdir(path): CONFIG_FILE=temp_root..'/current.yml' store.write(CONFIG_FILE,'load-order: [recorder-0.3.0]\\n') profile={name='SHC',sha256=string.rep('a',64)} +function resourceState(value) local t={}; for i=1,200 do t[i]=value or 0 end; return t end function recording() local m=store.new(profile) m.player=1; m.startTick=0; m.lastTick=64; m.finalRng={11,22,3,4} + m.startResources=resourceState(); m.finalResources=resourceState() m.snapshotHash=string.rep('b',64); m.rngHash=string.rep('c',64) local path=store.path(m.id) local function command(t) return {commandCategory=28,player=1,time=t,size=1,data='01'} end store.write(path..'/stream-commands.json',json:encode(command(10))..'\\n'..json:encode(command(65))..'\\n') - store.write(path..'/stream-rng-sync.json',json:encode({time=0,rng={1,2,3,4}})..'\\n'..json:encode({time=64,rng=m.finalRng})..'\\n') + store.write(path..'/stream-rng-sync.json',json:encode({time=0,rng={1,2,3,4},resources=resourceState()})..'\\n'..json:encode({time=64,rng=m.finalRng,resources=resourceState()})..'\\n') store.write(path..'/stream-infself.json',json:encode({gameType=0,mapSeed=123,matchSeed=123,RNGvalue1=1,RNGvalue2=2,RNGindex1=4,RNGindex2=3})..'\\n') return m end @@ -120,4 +122,19 @@ def test_invalid_manifest_and_path_do_not_load(self): local m=recording(); store.finish(m) m.lastTick=-1; store.save(m) assert(not pcall(store.load,m.id,profile)) +''') + + def test_missing_or_malformed_resource_evidence_prevents_completion(self): + self.lua.execute(''' +local m=recording(); m.finalResources[200]=nil +assert(not pcall(store.finish,m)) +m=recording() +local path=store.path(m.id)..'/stream-rng-sync.json' +store.write(path,json:encode({time=0,rng={1,2,3,4}})..'\\n') +assert(not pcall(store.finish,m)) +local validation=require('code/validation') +for _,invalid in ipairs({1.5,2147483648,-2147483649}) do + local values=resourceState(); values[100]=invalid + assert(not pcall(validation.resources,values)) +end ''') diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 4cd5125..e1ff84f 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -21,6 +21,7 @@ def setUp(self): Session=require('code/session-recorder') now=0; snapshots=0; space=true engine={rng=0x1a279c0, + resourceState=function() return resourceState() end, resetCommands=function(self) self.journal={executed=0} end, journal={executed=0}, setScope=function(_,active) scoped=active end, @@ -106,9 +107,9 @@ def test_end_of_replay_requires_all_commands(self): def test_final_checkpoint_boundary_finishes_without_next_checkpoint(self): self.check(''' local r=session(); r.status='playing'; r.mode='play'; r.active=true; r.playedCommands=0 -r.manifest={id='test',lastTick=64,commandCount=0,finalRng={11,22,3,4}} +r.manifest={id='test',lastTick=64,commandCount=0,finalRng={11,22,3,4},finalResources=resourceState()} local reads=0 -r.rngFile={read=function() reads=reads+1; assert(reads==1); return {time=64,rng={11,22,3,4}} end} +r.rngFile={read=function() reads=reads+1; assert(reads==1); return {time=64,rng={11,22,3,4},resources=resourceState()} end} now=64; r:onTick(); r:onTick() assert(r.status=='finished' and paused and reads==1) ''') @@ -116,7 +117,7 @@ def test_final_checkpoint_boundary_finishes_without_next_checkpoint(self): def test_queued_commands_are_not_reported_as_executed(self): self.check(''' local r=session(); r.status='playing'; r.mode='play'; r.active=true; r.playedCommands=2 -r.manifest={id='test',lastTick=65,commandCount=2,finalRng={11,22,3,4}} +r.manifest={id='test',lastTick=65,commandCount=2,finalRng={11,22,3,4},finalResources=resourceState()} engine.commandsPending=function() return true end now=65; assert(not r:guard(function() r:onTick() end)); assert(r.status=='error') ''') @@ -195,4 +196,32 @@ def test_error_in_stale_playback_cannot_pause_multiplayer(self): engine.singlePlayer=function() return false end assert(not r:guard(function() error('stale session') end)) assert(not paused and memory[r.halt]==0) +''') + + def test_resource_divergence_stops_even_when_rng_is_identical(self): + self.check(''' +local r=session(); r.mode='play'; r.status='playing'; r.active=true +r.manifest={id='test',lastTick=128} +r.rngFile={read=function() return {time=64,rng={11,22,3,4},resources=resourceState()} end} +engine.resourceState=function() local state=resourceState(); state[41]=5; return state end +now=64; assert(not r:guard(function() r:onTick() end)) +assert(paused and r.status=='error' and r.firstDesync.kind=='resources') +assert(r.firstDesync.player==2 and r.firstDesync.resource==15 and r.firstDesync.actual==5) +''') + + def test_ending_resource_state_is_checked_between_checkpoints(self): + self.check(''' +local r=session(); r.mode='play'; r.status='playing'; r.active=true; r.playedCommands=0 +r.manifest={id='test',lastTick=65,commandCount=0,finalRng={11,22,3,4},finalResources=resourceState()} +r.manifest.finalResources[200]=1 +now=65; assert(not r:guard(function() r:onTick() end)) +assert(r.status=='error' and r.firstDesync.player==8 and r.firstDesync.resource==24) +assert(r.firstDesync.phase=='ending state') +''') + + def test_starting_resource_mismatch_has_its_own_diagnostic(self): + self.check(''' +local r=session(); r.mode='play'; r.status='loading'; r.manifest={id='test'} +assert(not r:guard(function() r:checkResources(resourceState(1),'starting save') end)) +assert(paused and r.firstDesync.phase=='starting save' and r.firstDesync.player==1) ''') From 1789366e79d713ffb157664ac8a10e26db8025f2 Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:44:16 +0200 Subject: [PATCH 14/19] Include the multiplayer comparator in PR test packages --- tools/pr_releases.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/pr_releases.py b/tools/pr_releases.py index 417adae..49eb824 100644 --- a/tools/pr_releases.py +++ b/tools/pr_releases.py @@ -60,6 +60,9 @@ def package(): files = [source / name for name in ('definition.yml', 'options.yml', 'init.lua', 'README.md', 'CHANGELOG.md')] files += sorted((source / 'code').rglob('*.lua')) files += sorted((source / 'docs').rglob('*.md')) + comparator = source / 'tools/compare_multiplayer.py' + if comparator.exists(): + files.append(comparator) with zipfile.ZipFile(asset, 'w', zipfile.ZIP_DEFLATED) as archive: for path in files: assert path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(source) From 54b7190e2a57401ffcf79b16bb89431a0225a6cf Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:44:16 +0200 Subject: [PATCH 15/19] Add opt-in multiplayer execution diagnostics (0.9.0) --- CHANGELOG.md | 9 +++ README.md | 6 +- code/engine.lua | 4 + code/multiplayer-trace.lua | 102 +++++++++++++++++++++++++ definition.yml | 2 +- docs/multiplayer-diagnostics.md | 24 ++++++ init.lua | 2 + options.yml | 10 ++- tests/test_multiplayer_trace.py | 130 ++++++++++++++++++++++++++++++++ tools/build.py | 3 + tools/compare_multiplayer.py | 97 ++++++++++++++++++++++++ 11 files changed, 385 insertions(+), 4 deletions(-) create mode 100644 code/multiplayer-trace.lua create mode 100644 docs/multiplayer-diagnostics.md create mode 100644 tests/test_multiplayer_trace.py create mode 100644 tools/compare_multiplayer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 85bb78b..ce04832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.9.0 + +- Add opt-in multiplayer command diagnostics, disabled by default. Observe the resolved actor, payload, scheduled/executed tick and post-command RNG/resource state without injecting, suppressing or pausing multiplayer commands. +- Isolate logging failures from native dispatch, and mark untracked or interrupted captures incomplete. +- Include a standalone trace comparator that reports the first difference between two peers and rejects incomplete evidence. +- Clarify that a fixed starting seed alone does not guarantee deterministic gameplay. + +Validation: 87 automated tests, including native callback pass-through on logging failure and peer trace comparison. This is multiplayer diagnostic support, not multiplayer replay; live multiplayer traces are still needed. + ## 0.8.0 - Verify all 25 native resource slots for each of the eight players when loading the starting save, at periodic checkpoints and at the ending boundary. diff --git a/README.md b/README.md index 2fcc941..a040244 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,12 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.8.0 verifies player resources and native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. +Version 0.9.0 verifies player resources and native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. Automarket 1.1.0 has an experimental replay adapter for its settings commits and native custom save section. Use protocol 1.0.0 and map-extensions 1.0.0, with **recorder after protocol in the extension order**. The normal weekly trades run in the simulation; they are not replayed as extra trades. Other custom protocols remain unsupported. See [Automarket replay notes](docs/automarket-replay.md). +Optional **Multiplayer diagnostics** logs actual command execution for comparing two peers. It is disabled by default and does not enable multiplayer replay. See [multiplayer trace instructions](docs/multiplayer-diagnostics.md). + ## Testing this build Use a separate game installation with UCP3 developer mode and this module enabled. If your game needs Graphics API Replacer, keep it and its dependencies enabled in the test configuration. @@ -32,4 +34,4 @@ python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.8.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.9.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/engine.lua b/code/engine.lua index 1d84dc3..da46fe0 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -43,6 +43,7 @@ function M.new(sites) end function M:resetCommands() + if self.trace then self.trace:observe('stop','game/session transition') end self.journal=require('code/command-journal').new() self.received={} self.executing=nil @@ -213,6 +214,7 @@ function M:install(recorder) size=0 end registers.EAX=size + if self.trace then self.trace:observe('receivedCommand',registers.EDX,size) end if recorder.active and recorder.status=='recording' and self:singlePlayer() then local ok=recorder:guard(function() require('code/validation').integer(size,0,1260,'native payload size') @@ -232,6 +234,7 @@ function M:install(recorder) core.detourCode(function(registers) local category=core.readByte(registers.ESI+registers.ECX+0x3c684) registers.EDX=category<128 and category or category-256 -- original MOVSX + if self.trace then self.trace:observe('beforeCommand') end if self:singlePlayer() and not self.loading and recorder.active then local ok=recorder:guard(function() assert(recorder.status=='playing' or recorder.status=='recording','Replay session is stopped') @@ -242,6 +245,7 @@ function M:install(recorder) return registers end,self.sites.execute.address,8) core.detourCode(function(registers) + if self.trace then self.trace:observe('afterCommand') end if self:singlePlayer() and not self.loading and recorder.active then recorder:guard(function() self:afterCommand(recorder) end) end diff --git a/code/multiplayer-trace.lua b/code/multiplayer-trace.lua new file mode 100644 index 0000000..d998361 --- /dev/null +++ b/code/multiplayer-trace.lua @@ -0,0 +1,102 @@ +-- Opt-in observation only: never pause, patch state, inject or suppress commands. +local store=require('code/sessions') +local platform=require('code/platform') +local native=require('code/native') +local validation=require('code/validation') +local utils=require('code/utils') +local unpack=table.unpack or unpack +local M={ROOT='ucp/replay-diagnostics'} + +function M.new(engine) + return setmetatable({engine=engine,received={},count=0},{__index=M}) +end + +function M:write(value) + assert(self.file:write(json:encode(value)..'\n')) + assert(self.file:flush()) +end + +function M:open() + if self.file then return end + platform.mkdir(M.ROOT) + local prefix=os.date('!%Y%m%d-%H%M%S') + for i=1,9999 do + local path=M.ROOT..'/'..prefix..'-'..string.format('%04d',i) + if platform.mkdir(path) then self.path=path; break end + assert(i<9999,'Cannot allocate multiplayer trace folder') + end + self.file=assert(io.open(self.path..'/commands.jsonl','w')) + self.count=0 + self:write({kind='header',format=1,variant=native.profile.name,executable=native.profile.sha256, + localPlayer=self.engine:player(),firstTick=self.engine:tick()}) + print('Multiplayer diagnostics: '..self.path) +end + +function M:receivedCommand(address,size) + validation.integer(size,0,1260,'multiplayer trace payload length') + local slot=validation.integer(core.readInteger(self.engine.base+0x2d824),0,199,'trace ring slot') + self.received[slot]={size=size,data=utils.tableToHex(core.readBytes(address,size))} +end + +function M:beforeCommand() + assert(not self.executing,'Nested multiplayer command dispatch') + local slot=validation.integer(core.readInteger(self.engine.base+0x2d824),0,199,'trace ring slot') + local address=self.engine.base+0x3c67c+slot*1272 + self:open() + local source=self.received[slot] + local event={kind=source and 'command' or 'untracked',time=self.engine:tick(),slot=slot, + scheduledTime=core.readInteger(address),handle=core.readInteger(address+4), + player=core.readInteger(self.engine.base+self.engine.sites.actorOffset),category=core.readByte(address+8)} + if source then + event.size=source.size + event.data=utils.tableToHex(core.readBytes(address+10,source.size)) + event.changedSinceReceive=event.data~=source.data + else self.incomplete=true end + self.executing=event +end + +function M:afterCommand() + local event=self.executing + if not event then return end + self.executing=nil + self.received[event.slot]=nil + self.count=self.count+1 + event.sequence=self.count + event.rng=self.engine:rngState() + event.resources=self.engine:resourceState() + self:write(event) +end + +function M:stop(reason) + local f=self.file + if f then + local ok,err=pcall(function() + self:write({kind='end',status=(self.incomplete or self.executing) and 'incomplete' or 'complete',commands=self.count,reason=reason}) + end) + self.file=nil + local closed,closeError=f:close() + assert(ok and closed,err or closeError) + end + self.received={}; self.executing=nil; self.failed=false; self.incomplete=false; self.path=nil +end + +function M:observe(event,...) + if self.failed and event~='stop' then return end + local args={...} + local ok,reason=pcall(function() + if event~='stop' and self.engine:singlePlayer() then + if self.file or next(self.received) then self:stop('left multiplayer') end + return + end + self[event](self,unpack(args)) + end) + if not ok then + self.failed=true; self.executing=nil + local f=self.file; self.file=nil + if f then pcall(f.close,f) end + if self.path then pcall(store.write,self.path..'/error.txt',tostring(reason)) end + print('Multiplayer diagnostics stopped: '..tostring(reason)) + end +end + +return M diff --git a/definition.yml b/definition.yml index 46dd79a..d9b02ea 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.8.0 +version: 0.9.0 type: module author: Corax meta: diff --git a/docs/multiplayer-diagnostics.md b/docs/multiplayer-diagnostics.md new file mode 100644 index 0000000..c3eea55 --- /dev/null +++ b/docs/multiplayer-diagnostics.md @@ -0,0 +1,24 @@ +# Multiplayer diagnostics (0.9.0) + +This is the first multiplayer-specific implementation stage. It observes execution on each peer without changing the simulation scope, RNG seeds, pause flag, command ownership or dispatch decisions. Multiplayer recording/playback remains disabled. The existing single-player replay checks and isolation gates remain in effect. + +## Collect and compare + +1. Use the same test build on both peers, with the usual multiplayer extensions and Graphics API Replacer where required. Enable `multiplayerDiagnostics: true` in recorder's UCP configuration (the option defaults to false). Keep recorder after protocol in extension order. +2. Play a short match with commands from both players. Exit the match normally so the trace can write its completion record. +3. Each game writes `ucp/replay-diagnostics/TIMESTAMP-NNNN/commands.jsonl`. Copy the two files into separately named peer-A and peer-B folders to preserve their identity. +4. Extract `tools/compare_multiplayer.py` from the module ZIP and run `python compare_multiplayer.py peer-A/commands.jsonl peer-B/commands.jsonl` with Python 3.10 or newer. + +Exit codes: 0 means the compared command/RNG/resource evidence matched; 1 means it differed; 2 means the evidence was incomplete or incompatible. The JSON result identifies the first execution sequence/field difference, or the player/resource and amounts for an economy difference. Keep both original traces for analysis. + +## What is observed + +The existing checked receive-copy and pre/post-dispatch sites observe timed commands from all senders. The actor comes from the native resolved-player field, rather than assigning every command to the local spectator. Each returned handler produces one row with the execution and scheduled ticks, category, payload, raw sender handle, native slot, resolved actor, post-command RNG and all eight players' resources. The comparator ignores peer-local player identity, raw handles and ring slots, comparing the resolved actions and resulting evidence instead. + +A missing payload receipt produces an untracked row and an incomplete completion status. Interrupted handlers, write errors, missing footers and malformed records cannot become a successful comparison. Logging errors close the trace and disable further diagnostic writes for that session; they do not halt multiplayer or replace its command category. Trace files are flushed synchronously, so this opt-in instrumentation has I/O and execution overhead that still needs live measurement. + +## Boundaries + +This is command-boundary evidence, not a full replay or a full world-state checksum. It does not yet capture untimed commands, periodic no-command simulation boundaries, resynchronization/save-transfer packets or private extension state. It does not make a multiplayer save playable offline, replace networking, or change human/AI identity rules. Equal traces therefore do not prove a complete multiplayer match is deterministic. Different post-command RNG/resource evidence locates a mismatch; further analysis is needed to establish its cause. + +Automated tests use native callback fixtures to verify original registers/dispatch survive logging failures, remote actors are recorded, single-player does not start a multiplayer trace, and incomplete evidence is rejected. The comparator has matching/differing/corrupt trace tests. No live multiplayer match was launched for this stage. diff --git a/init.lua b/init.lua index a3d2ec5..2399b1c 100644 --- a/init.lua +++ b/init.lua @@ -15,6 +15,7 @@ function module:enable(config) end require('code/sessions').captureSettings() local engine=Engine.new(sites) + if config.multiplayerDiagnostics then engine.trace=require('code/multiplayer-trace').new(engine) end fixes.install(fixSites,engine.scope,engine.base+0x618,seed) local recorder=Session:new(engine) self.recorder=recorder @@ -32,6 +33,7 @@ function module:enable(config) end,address,size) end observe(native.addr(0x4428c6),10,function() + if engine.trace then engine.trace:observe('stop','new match') end recorder:activateRecording() ui.resetButtons() end) diff --git a/options.yml b/options.yml index 77fee6c..adb3068 100644 --- a/options.yml +++ b/options.yml @@ -11,7 +11,7 @@ options: value: "trace" - url: recorder.useFixedSeed header: "Use fixed seed" - text: "Using fixed seeds guarantees deterministic gameplay" + text: "Use the configured starting seed for single-player recorder sessions" hasHeader: true display: contents: @@ -25,3 +25,11 @@ options: contents: type: integer value: 1 +- url: recorder.multiplayerDiagnostics + header: "Multiplayer diagnostics" + text: "Log executed multiplayer commands and resource/RNG evidence for comparison. This does not enable multiplayer replay." + hasHeader: true + display: + contents: + type: boolean + value: false diff --git a/tests/test_multiplayer_trace.py b/tests/test_multiplayer_trace.py new file mode 100644 index 0000000..b5e9baa --- /dev/null +++ b/tests/test_multiplayer_trace.py @@ -0,0 +1,130 @@ +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +import test_engine + + +class MultiplayerTraceTests(unittest.TestCase): + check = test_engine.EngineTests.check + command_hooks = test_engine.EngineTests.command_hooks + + def setUp(self): + test_engine.EngineTests.setUp(self) + self.command_hooks('record') + self.check(''' +package.loaded['code/platform']={mkdir=function() return true end} +traceRows={}; json.encode=function(_,v) traceRows[#traceRows+1]=v; return 'json' end +realNative.profile.name='SHC'; realNative.profile.sha256=string.rep('a',64) +engine.trace=require('code/multiplayer-trace').new(engine) +memory[engine.base+0x618]=1 +memory[engine.base+engine.sites.actorOffset]=3 +recorder.active=false; recorder.mode='none' +engine.resourceState=function() return resourceState() end +core.writeInteger=function() error('diagnostics attempted native state write') end +function receive(size) + memory[engine.base+0x2d830]=size or 1 + return callbacks[engine.sites.copySize.address]({ESI=engine.base,EDX=engine.buffer}) +end +''') + + def test_trace_observes_resolved_remote_actor_without_changing_dispatch(self): + self.check(''' +assert(receive().EAX==1) +assert(before().EDX==28) +after() +engine.trace:observe('stop','test ended') +assert(#traceRows==3 and traceRows[1].kind=='header' and traceRows[3].status=='complete') +assert(traceRows[2].player==3 and traceRows[2].category==28 and traceRows[2].data=='01') +assert(traceRows[2].sequence==1 and #traceRows[2].resources==200) +assert(recorder.status=='recording' and #recorder.commands==0) +''') + + def test_trace_write_failure_cannot_suppress_multiplayer_command(self): + self.check(''' +receive(); assert(before().EDX==28) +engine.trace.file.write=function() return nil,'disk full' end +after() +assert(engine.trace.failed and not engine.trace.file) +assert(before().EDX==28 and receive().EAX==1) +assert(recorder.mode=='none' and not recorder.error) +''') + + def test_invalid_diagnostic_size_leaves_original_native_register_value(self): + self.check(''' +assert(receive(1261).EAX==1261 and engine.trace.failed) +assert(before().EDX==28) +''') + + def test_missing_receipt_or_interrupted_handler_marks_trace_incomplete(self): + self.check(''' +before(); after(); engine.trace:observe('stop','missing receipt') +assert(traceRows[2].kind=='untracked' and traceRows[3].status=='incomplete') +traceRows={}; receive(); before(); engine.trace:observe('stop','interrupted') +assert(traceRows[2].kind=='end' and traceRows[2].status=='incomplete') +''') + + def test_singleplayer_does_not_start_multiplayer_trace(self): + self.check(''' +memory[engine.base+0x618]=0; receive(); before(); after() +assert(#traceRows==0 and not engine.trace.file) +''') + + +class CompareTraceTests(unittest.TestCase): + def setUp(self): + module_path = Path(__file__).resolve().parents[1] / 'tools/compare_multiplayer.py' + spec = importlib.util.spec_from_file_location('compare_multiplayer', module_path) + self.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.module) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def trace(self): + return [dict(kind='header', format=1, variant='SHC', executable='a', localPlayer=1), + dict(kind='command', sequence=1, time=10, scheduledTime=10, player=3, + category=28, size=1, data='01', handle=123, slot=0, rng=[1, 2, 3, 4], resources=[0] * 200), + dict(kind='end', status='complete', commands=1)] + + def compare(self, a, b): + paths = [self.root / 'a.jsonl', self.root / 'b.jsonl'] + for path, records in zip(paths, (a, b)): + path.write_text(''.join(json.dumps(r) + '\n' for r in records), encoding='utf-8') + return self.module.compare(*paths) + + def test_peer_local_identity_and_ring_slot_do_not_create_false_differences(self): + a, b = self.trace(), self.trace() + b[0]['localPlayer'] = 3 + b[1].update(handle=456, slot=199) + self.assertEqual(self.compare(a, b)['status'], 'matched') + + def test_command_order_actor_tick_payload_rng_and_resources_are_compared(self): + for key, value in [('time', 11), ('player', 2), ('data', 'FF'), ('rng', [1, 2, 3, 5]), + ('resources', [0] * 40 + [7] + [0] * 159)]: + with self.subTest(key=key): + a, b = self.trace(), self.trace() + b[1][key] = value + result = self.compare(a, b) + self.assertEqual(result['status'], 'different') + self.assertEqual(result['firstDifference']['field'], key) + if key == 'resources': + self.assertEqual(result['firstDifference']['player'], 2) + self.assertEqual(result['firstDifference']['resource'], 15) + + def test_missing_footer_untracked_invalid_payload_and_empty_trace_are_incomplete(self): + for change in ('footer', 'untracked', 'data', 'empty', 'sequence'): + a, b = self.trace(), self.trace() + if change == 'footer': + b.pop() + elif change == 'untracked': + b[1]['kind'] = 'untracked' + elif change == 'data': + b[1]['data'] = 'GG' + elif change == 'empty': + b = [] + else: + b[1]['sequence'] = 2 + self.assertEqual(self.compare(a, b)['status'], 'incomplete', change) diff --git a/tools/build.py b/tools/build.py index c2f0bef..9d5dcf9 100644 --- a/tools/build.py +++ b/tools/build.py @@ -12,6 +12,9 @@ files = [root/p for p in ('definition.yml', 'options.yml', 'init.lua', 'README.md', 'CHANGELOG.md')] files.extend(sorted((root/'code').rglob('*.lua'))) files.extend(sorted((root/'docs').rglob('*.md'))) +comparator = root/'tools/compare_multiplayer.py' +if comparator.exists(): + files.append(comparator) with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as archive: for path in files: archive.write(path, path.relative_to(root).as_posix()) diff --git a/tools/compare_multiplayer.py b/tools/compare_multiplayer.py new file mode 100644 index 0000000..6ffecc1 --- /dev/null +++ b/tools/compare_multiplayer.py @@ -0,0 +1,97 @@ +"""Compare two peers' command traces. Exit 0=matched, 1=different, 2=incomplete.""" +import argparse +import itertools +import json +from pathlib import Path + + +def integer(value, lo, hi): + if type(value) is not int or not lo <= value <= hi: + raise ValueError('Invalid integer in trace') + + +def rows(stream): + while line := stream.readline(65537): + if len(line) > 65536: + raise ValueError('Oversized trace line') + value = json.loads(line) + if not isinstance(value, dict): + raise ValueError('Invalid trace record') + yield value + + +def commands(records): + count = 0 + for record in records: + if record.get('kind') == 'end': + if record.get('status') != 'complete' or record.get('commands') != count or not count: + raise ValueError('Trace is incomplete or has no commands') + if next(records, None) is not None: + raise ValueError('Data follows trace completion') + return + if record.get('kind') != 'command': + raise ValueError('Trace contains an untracked/unknown command') + count += 1 + if record.get('sequence') != count: + raise ValueError('Trace command sequence is incomplete') + for key, lo, hi in [('time', 0, 2147483647), ('scheduledTime', 1, 2147483647), + ('player', 1, 8), ('category', 0, 127), ('size', 0, 1260)]: + integer(record.get(key), lo, hi) + data = record.get('data') + if not isinstance(data, str) or len(data) != record['size'] * 2 or any(c not in '0123456789abcdefABCDEF' for c in data): + raise ValueError('Invalid trace payload') + record['data'] = data.upper() + for key, length in [('rng', 4), ('resources', 200)]: + values = record.get(key) + if not isinstance(values, list) or len(values) != length: + raise ValueError(f'Missing {key} evidence') + for value in values: + integer(value, -2147483648, 2147483647) + yield record + raise ValueError('Trace has no completion record') + + +def compare(left, right): + difference = None + count = 0 + try: + with Path(left).open(encoding='utf-8') as a, Path(right).open(encoding='utf-8') as b: + ar, br = rows(a), rows(b) + ah, bh = next(ar), next(br) + for header in (ah, bh): + if header.get('kind') != 'header' or header.get('format') != 1: + raise ValueError('Unsupported trace format') + if any(ah.get(key) != bh.get(key) for key in ('variant', 'executable')): + raise ValueError('Traces require the same game executable/variant') + for av, bv in itertools.zip_longest(commands(ar), commands(br)): + count += 1 + if difference: + continue # Still validate both complete streams before reporting. + if av is None or bv is None: + difference = dict(sequence=count, field='command count') + continue + for key in ('time', 'scheduledTime', 'player', 'category', 'size', 'data', 'rng', 'resources'): + if av[key] == bv[key]: + continue + difference = dict(sequence=count, time=av['time'], field=key) + if key in ('rng', 'resources'): + index = next(i for i, (x, y) in enumerate(zip(av[key], bv[key])) if x != y) + difference.update(index=index, left=av[key][index], right=bv[key][index]) + if key == 'resources': + difference.update(player=index // 25 + 1, resource=index % 25) + else: + difference.update(left=av[key], right=bv[key]) + break + except (OSError, ValueError, KeyError, StopIteration) as error: + return dict(status='incomplete', reason=str(error) or 'Empty trace', firstDifference=difference) + return dict(status='different' if difference else 'matched', commands=count, firstDifference=difference) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('left', type=Path) + parser.add_argument('right', type=Path) + args = parser.parse_args() + result = compare(args.left, args.right) + print(json.dumps(result, indent=2)) + raise SystemExit({'matched': 0, 'different': 1, 'incomplete': 2}[result['status']]) From 9d67d0bb46b3bac825230088aa976365a48566f2 Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:59:22 +0200 Subject: [PATCH 16/19] Observe multiplayer simulation checkpoints between commands (0.10.0) --- CHANGELOG.md | 8 +++++ README.md | 6 ++-- code/fixes.lua | 4 +-- code/multiplayer-trace.lua | 25 +++++++++++--- code/scoped-code.lua | 5 +++ definition.yml | 2 +- docs/multiplayer-diagnostics.md | 8 +++-- init.lua | 10 +++++- tests/test_multiplayer_trace.py | 26 ++++++++++++++ tests/test_scoped_code.py | 18 ++++++++++ tools/compare_multiplayer.py | 61 ++++++++++++++++++++++----------- 11 files changed, 138 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce04832..324c2f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.10.0 + +- Add periodic multiplayer RNG/resource evidence before native simulation advancement, including periods with no player commands. +- Keep the optional observer independent of the single-player replay halt/patch scope; preserve native registers, flags, stack and tick continuation. +- Deduplicate paused boundaries and compare format-2 command/checkpoint streams, rejecting missing periodic evidence. The comparator still accepts paired format-1 traces. + +Validation: 91 automated tests, including x86 emulation for both variants with a stale halt flag and register-clobbering diagnostic callbacks. Live multiplayer timing and trace comparisons remain pending; multiplayer replay is not enabled. + ## 0.9.0 - Add opt-in multiplayer command diagnostics, disabled by default. Observe the resolved actor, payload, scheduled/executed tick and post-command RNG/resource state without injecting, suppressing or pausing multiplayer commands. diff --git a/README.md b/README.md index a040244..eb556ff 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.9.0 verifies player resources and native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. +Version 0.10.0 verifies player resources and native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. Automarket 1.1.0 has an experimental replay adapter for its settings commits and native custom save section. Use protocol 1.0.0 and map-extensions 1.0.0, with **recorder after protocol in the extension order**. The normal weekly trades run in the simulation; they are not replayed as extra trades. Other custom protocols remain unsupported. See [Automarket replay notes](docs/automarket-replay.md). -Optional **Multiplayer diagnostics** logs actual command execution for comparing two peers. It is disabled by default and does not enable multiplayer replay. See [multiplayer trace instructions](docs/multiplayer-diagnostics.md). +Optional **Multiplayer diagnostics** logs actual command execution and periodic simulation evidence for comparing two peers. It is disabled by default and does not enable multiplayer replay. See [multiplayer trace instructions](docs/multiplayer-diagnostics.md). ## Testing this build @@ -34,4 +34,4 @@ python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.9.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.10.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/fixes.lua b/code/fixes.lua index 95b3a92..7d5dd8b 100644 --- a/code/fixes.lua +++ b/code/fixes.lua @@ -25,9 +25,9 @@ function M.install(sites,enabled,mode,seed) end end -function M.installTick(site,enabled,mode,halt,callback) +function M.installTick(site,enabled,mode,halt,callback,originalCallback) local tick={address=site.address,bytes=site.bytes,kind='raw',patch='tick', - halt=halt,callback=callback,skipTick=site.address+0x25} + halt=halt,callback=callback,originalCallback=originalCallback,skipTick=site.address+0x25} M.install({tick},enabled,mode) end return M diff --git a/code/multiplayer-trace.lua b/code/multiplayer-trace.lua index d998361..8c7599c 100644 --- a/code/multiplayer-trace.lua +++ b/code/multiplayer-trace.lua @@ -26,12 +26,26 @@ function M:open() assert(i<9999,'Cannot allocate multiplayer trace folder') end self.file=assert(io.open(self.path..'/commands.jsonl','w')) - self.count=0 - self:write({kind='header',format=1,variant=native.profile.name,executable=native.profile.sha256, + self.count=0; self.events=0 + self:write({kind='header',format=2,variant=native.profile.name,executable=native.profile.sha256, localPlayer=self.engine:player(),firstTick=self.engine:tick()}) print('Multiplayer diagnostics: '..self.path) end +function M:record(event) + self.events=self.events+1 + event.sequence=self.events + self:write(event) +end + +function M:onTick() + local now=self.engine:tick() + if now%64~=0 or now==self.lastTick then return end + self:open() + self.lastTick=now + self:record({kind='checkpoint',time=now,rng=self.engine:rngState(),resources=self.engine:resourceState()}) +end + function M:receivedCommand(address,size) validation.integer(size,0,1260,'multiplayer trace payload length') local slot=validation.integer(core.readInteger(self.engine.base+0x2d824),0,199,'trace ring slot') @@ -61,23 +75,24 @@ function M:afterCommand() self.executing=nil self.received[event.slot]=nil self.count=self.count+1 - event.sequence=self.count event.rng=self.engine:rngState() event.resources=self.engine:resourceState() - self:write(event) + self:record(event) end function M:stop(reason) local f=self.file if f then local ok,err=pcall(function() - self:write({kind='end',status=(self.incomplete or self.executing) and 'incomplete' or 'complete',commands=self.count,reason=reason}) + self:write({kind='end',status=(self.incomplete or self.executing) and 'incomplete' or 'complete', + commands=self.count,events=self.events,reason=reason}) end) self.file=nil local closed,closeError=f:close() assert(ok and closed,err or closeError) end self.received={}; self.executing=nil; self.failed=false; self.incomplete=false; self.path=nil + self.lastTick=nil end function M:observe(event,...) diff --git a/code/scoped-code.lua b/code/scoped-code.lua index 161d863..505322a 100644 --- a/code/scoped-code.lua +++ b/code/scoped-code.lua @@ -44,6 +44,11 @@ function M.build(site,enabled,mode,seed,origin) end rel({0xe9},site.address+#site.bytes) labels.original=#out; emit(0x9d) + if site.patch=='tick' and site.originalCallback then + emit(0x9c,0x60) + rel({0xe8},site.originalCallback) + emit(0x61,0x9d) + end if site.kind=='call' then rel({0xe8},site.target) elseif site.kind=='branch' then rel({0x0f,site.condition},site.target) diff --git a/definition.yml b/definition.yml index d9b02ea..97c5d5d 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.9.0 +version: 0.10.0 type: module author: Corax meta: diff --git a/docs/multiplayer-diagnostics.md b/docs/multiplayer-diagnostics.md index c3eea55..a6ebc9b 100644 --- a/docs/multiplayer-diagnostics.md +++ b/docs/multiplayer-diagnostics.md @@ -1,4 +1,4 @@ -# Multiplayer diagnostics (0.9.0) +# Multiplayer diagnostics (0.10.0) This is the first multiplayer-specific implementation stage. It observes execution on each peer without changing the simulation scope, RNG seeds, pause flag, command ownership or dispatch decisions. Multiplayer recording/playback remains disabled. The existing single-player replay checks and isolation gates remain in effect. @@ -13,12 +13,14 @@ Exit codes: 0 means the compared command/RNG/resource evidence matched; 1 means ## What is observed -The existing checked receive-copy and pre/post-dispatch sites observe timed commands from all senders. The actor comes from the native resolved-player field, rather than assigning every command to the local spectator. Each returned handler produces one row with the execution and scheduled ticks, category, payload, raw sender handle, native slot, resolved actor, post-command RNG and all eight players' resources. The comparator ignores peer-local player identity, raw handles and ring slots, comparing the resolved actions and resulting evidence instead. +The existing checked receive-copy and pre/post-dispatch sites observe timed commands from all senders. The actor comes from the native resolved-player field, rather than assigning every command to the local spectator. Every 64 simulation ticks, a read-only observer also records RNG/resource state immediately before native RNG advancement. It runs on the original simulation path without using the single-player halt flag. Repeated observations of a paused boundary are deduplicated. Each returned handler produces one row with the execution and scheduled ticks, category, payload, raw sender handle, native slot, resolved actor, post-command RNG and all eight players' resources. The comparator ignores peer-local player identity, raw handles and ring slots, comparing the resolved actions and resulting evidence instead. A missing payload receipt produces an untracked row and an incomplete completion status. Interrupted handlers, write errors, missing footers and malformed records cannot become a successful comparison. Logging errors close the trace and disable further diagnostic writes for that session; they do not halt multiplayer or replace its command category. Trace files are flushed synchronously, so this opt-in instrumentation has I/O and execution overhead that still needs live measurement. ## Boundaries -This is command-boundary evidence, not a full replay or a full world-state checksum. It does not yet capture untimed commands, periodic no-command simulation boundaries, resynchronization/save-transfer packets or private extension state. It does not make a multiplayer save playable offline, replace networking, or change human/AI identity rules. Equal traces therefore do not prove a complete multiplayer match is deterministic. Different post-command RNG/resource evidence locates a mismatch; further analysis is needed to establish its cause. +This is command-boundary and periodic simulation evidence, not a full replay or a full world-state checksum. It does not yet capture untimed commands, resynchronization/save-transfer packets or private extension state. It does not make a multiplayer save playable offline, replace networking, or change human/AI identity rules. Equal traces therefore do not prove a complete multiplayer match is deterministic. Different post-command RNG/resource evidence locates a mismatch; further analysis is needed to establish its cause. Automated tests use native callback fixtures to verify original registers/dispatch survive logging failures, remote actors are recorded, single-player does not start a multiplayer trace, and incomplete evidence is rejected. The comparator has matching/differing/corrupt trace tests. No live multiplayer match was launched for this stage. + +Format 2 gives every command/checkpoint an ordered evidence sequence and records both event and command totals. The comparator detects skipped checkpoint boundaries, including matching files with the same missing interval. It accepts older format-1 pairs as command-only evidence; mixing formats is rejected. A short interval before the first recorded event is outside the trace window. diff --git a/init.lua b/init.lua index 2399b1c..d2e1ffe 100644 --- a/init.lua +++ b/init.lua @@ -48,7 +48,15 @@ function module:enable(config) local tickCallback=core.allocateCode({0x90,0x90,0x90,0x90,0x90,0xC3}) observe(tickCallback,5,function() recorder:onTick() end) - fixes.installTick(sites.tick,engine.scope,engine.base+0x618,recorder.halt,tickCallback) + local multiplayerTick + if engine.trace then + multiplayerTick=core.allocateCode({0x90,0x90,0x90,0x90,0x90,0xC3}) + core.detourCode(function(registers) + engine.trace:observe('onTick') + return registers + end,multiplayerTick,5) + end + fixes.installTick(sites.tick,engine.scope,engine.base+0x618,recorder.halt,tickCallback,multiplayerTick) end function module:disable() diff --git a/tests/test_multiplayer_trace.py b/tests/test_multiplayer_trace.py index b5e9baa..28ac274 100644 --- a/tests/test_multiplayer_trace.py +++ b/tests/test_multiplayer_trace.py @@ -72,6 +72,17 @@ def test_singleplayer_does_not_start_multiplayer_trace(self): assert(#traceRows==0 and not engine.trace.file) ''') + def test_tick_evidence_does_not_need_commands_and_deduplicates_paused_boundary(self): + self.check(''' +memory[0x1fe7da8]=64 +engine.trace:observe('onTick'); engine.trace:observe('onTick') +assert(#traceRows==2 and traceRows[2].kind=='checkpoint' and traceRows[2].time==64) +memory[0x1fe7da8]=65; engine.trace:observe('onTick'); assert(#traceRows==2) +memory[0x1fe7da8]=128; engine.trace:observe('onTick') +engine.trace:observe('stop','test ended') +assert(traceRows[4].commands==0 and traceRows[4].events==2 and traceRows[4].status=='complete') +''') + class CompareTraceTests(unittest.TestCase): def setUp(self): @@ -128,3 +139,18 @@ def test_missing_footer_untracked_invalid_payload_and_empty_trace_are_incomplete else: b[1]['sequence'] = 2 self.assertEqual(self.compare(a, b)['status'], 'incomplete', change) + + def test_periodic_evidence_compares_without_any_commands(self): + a = [dict(kind='header', format=2, variant='SHC', executable='a', firstTick=64), + dict(kind='checkpoint', sequence=1, time=64, rng=[1, 2, 3, 4], resources=[0]*200), + dict(kind='end', status='complete', events=1, commands=0)] + b = json.loads(json.dumps(a)) + self.assertEqual(self.compare(a, b)['status'], 'matched') + b[1]['rng'][3] = 5 + self.assertEqual(self.compare(a, b)['firstDifference']['field'], 'rng') + + def test_missing_checkpoint_cannot_be_hidden_in_a_matching_pair(self): + a = self.trace() + a[0].update(format=2, firstTick=0) + a[-1].update(events=1) + self.assertEqual(self.compare(a, a)['status'], 'incomplete') diff --git a/tests/test_scoped_code.py b/tests/test_scoped_code.py index 1d22e35..21e4e83 100644 --- a/tests/test_scoped_code.py +++ b/tests/test_scoped_code.py @@ -35,6 +35,9 @@ def put(address,value): machine.mem_write(address,struct.pack(' next_checkpoint: + raise ValueError('Missing periodic checkpoint evidence') + fields = [('scheduledTime', 1, 2147483647), + ('player', 1, 8), ('category', 0, 127), ('size', 0, 1260)] + if record['kind'] == 'command': + for key, lo, hi in fields: + integer(record.get(key), lo, hi) + data = record.get('data') + if not isinstance(data, str) or len(data) != record['size'] * 2 or any(c not in '0123456789abcdefABCDEF' for c in data): + raise ValueError('Invalid trace payload') + record['data'] = data.upper() + elif record['time'] != next_checkpoint: + raise ValueError('Missing, repeated or invalid checkpoint boundary') + else: + next_checkpoint += 64 for key, length in [('rng', 4), ('resources', 200)]: values = record.get(key) if not isinstance(values, list) or len(values) != length: @@ -59,18 +76,22 @@ def compare(left, right): ar, br = rows(a), rows(b) ah, bh = next(ar), next(br) for header in (ah, bh): - if header.get('kind') != 'header' or header.get('format') != 1: + if header.get('kind') != 'header' or header.get('format') not in (1, 2): raise ValueError('Unsupported trace format') - if any(ah.get(key) != bh.get(key) for key in ('variant', 'executable')): - raise ValueError('Traces require the same game executable/variant') - for av, bv in itertools.zip_longest(commands(ar), commands(br)): + if any(ah.get(key) != bh.get(key) for key in ('variant', 'executable', 'format')): + raise ValueError('Traces require the same game executable, variant and format') + for av, bv in itertools.zip_longest(evidence(ar, ah['format'], ah.get('firstTick', 0)), + evidence(br, bh['format'], bh.get('firstTick', 0))): count += 1 if difference: continue # Still validate both complete streams before reporting. if av is None or bv is None: - difference = dict(sequence=count, field='command count') + difference = dict(sequence=count, field='event count') continue - for key in ('time', 'scheduledTime', 'player', 'category', 'size', 'data', 'rng', 'resources'): + fields = ('kind', 'time') + if av['kind'] == bv['kind'] == 'command': + fields += ('scheduledTime', 'player', 'category', 'size', 'data') + for key in fields + ('rng', 'resources'): if av[key] == bv[key]: continue difference = dict(sequence=count, time=av['time'], field=key) @@ -84,7 +105,7 @@ def compare(left, right): break except (OSError, ValueError, KeyError, StopIteration) as error: return dict(status='incomplete', reason=str(error) or 'Empty trace', firstDifference=difference) - return dict(status='different' if difference else 'matched', commands=count, firstDifference=difference) + return dict(status='different' if difference else 'matched', events=count, firstDifference=difference) if __name__ == '__main__': From b792534cd3618328b7d942f0fbdc7f534877c4a7 Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:28:35 +0200 Subject: [PATCH 17/19] Verify complete RNG state and multiplayer configuration evidence --- CHANGELOG.md | 9 ++++++++ README.md | 4 ++-- code/engine.lua | 6 +++++ code/multiplayer-trace.lua | 8 +++++-- code/session-recorder.lua | 32 +++++++++++++++++++++---- code/sessions.lua | 3 ++- code/validation.lua | 9 +++++--- definition.yml | 2 +- docs/multiplayer-diagnostics.md | 12 +++++++++- docs/replay-sessions.md | 9 ++++++++ tests/test_engine.py | 10 ++++++++ tests/test_multiplayer_trace.py | 41 +++++++++++++++++++++++++++++++++ tests/test_session_files.py | 18 +++++++++++++-- tests/test_sessions.py | 38 +++++++++++++++++++++++++++--- tools/compare_multiplayer.py | 21 +++++++++++++---- 15 files changed, 199 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 324c2f0..5a1abf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.11.0 + +- Compare SHA-256 of the complete 40,016-byte RNG structure after restoring the starting save, at periodic checkpoints and at the ending boundary. Detect different stored random arrays even when current RNG values and indices match. +- Retain the last observed RNG bytes for completion, so leaving the match cannot change the state being certified. Hash at checkpoints/completion, while copying the current RNG bytes at each recorded tick. +- Add full RNG hashes to periodic multiplayer diagnostics and compare resolved UCP configuration, framework and extension order/versions before comparing peers. Missing or incompatible evidence is rejected. +- Use single-player profile `recorder-sp-v5` and multiplayer trace format 3. Older recordings need their original module version; the comparator still accepts paired older trace formats with their original evidence limits. + +Validation: 98 automated tests, including altered RNG array bytes with unchanged current values/indices, final-state preservation, missing evidence and observer failure isolation. Full-world state and live multiplayer/replay validation remain outstanding; multiplayer replay is not enabled. + ## 0.10.0 - Add periodic multiplayer RNG/resource evidence before native simulation advancement, including periods with no player commands. diff --git a/README.md b/README.md index eb556ff..3ddc20a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.10.0 verifies player resources and native command execution and confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. +Version 0.11.0 verifies full RNG state, player resources and native command execution. It confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. Automarket 1.1.0 has an experimental replay adapter for its settings commits and native custom save section. Use protocol 1.0.0 and map-extensions 1.0.0, with **recorder after protocol in the extension order**. The normal weekly trades run in the simulation; they are not replayed as extra trades. Other custom protocols remain unsupported. See [Automarket replay notes](docs/automarket-replay.md). @@ -34,4 +34,4 @@ python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.10.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.11.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/engine.lua b/code/engine.lua index da46fe0..e69feb2 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -68,6 +68,12 @@ function M:rngState() core.readInteger(self.rng+0x9c48),core.readInteger(self.rng+0x9c4c)} end +function M:rngData() + local data=core.readString(self.rng,0x9c50) + assert(type(data)=='string' and #data==0x9c50,'Incomplete native RNG state') + return data +end + function M:resourceState() local values={} for player=1,8 do diff --git a/code/multiplayer-trace.lua b/code/multiplayer-trace.lua index 8c7599c..f2b5ccc 100644 --- a/code/multiplayer-trace.lua +++ b/code/multiplayer-trace.lua @@ -27,7 +27,10 @@ function M:open() end self.file=assert(io.open(self.path..'/commands.jsonl','w')) self.count=0; self.events=0 - self:write({kind='header',format=2,variant=native.profile.name,executable=native.profile.sha256, + local environmentHash=store.settings().environmentHash + validation.hash(environmentHash,'diagnostic environment hash') + self:write({kind='header',format=3,variant=native.profile.name,executable=native.profile.sha256, + environmentHash=environmentHash, localPlayer=self.engine:player(),firstTick=self.engine:tick()}) print('Multiplayer diagnostics: '..self.path) end @@ -43,7 +46,8 @@ function M:onTick() if now%64~=0 or now==self.lastTick then return end self:open() self.lastTick=now - self:record({kind='checkpoint',time=now,rng=self.engine:rngState(),resources=self.engine:resourceState()}) + self:record({kind='checkpoint',time=now,rng=self.engine:rngState(),resources=self.engine:resourceState(), + rngHash=sha.sha256(self.engine:rngData())}) end function M:receivedCommand(address,size) diff --git a/code/session-recorder.lua b/code/session-recorder.lua index 5a4b6ca..c242700 100644 --- a/code/session-recorder.lua +++ b/code/session-recorder.lua @@ -45,6 +45,7 @@ function Session:startRecording() self:openFiles('w') self.mode='record'; self.status='armed'; self.active=false self.error=nil; self.observedTick=false + self.finalRngData=nil self.engine:resetCommands() self.engine:setScope(true) core.writeInteger(self.halt,0) @@ -55,7 +56,7 @@ function Session:activateRecording() if self.mode~='record' or self.status~='armed' then return end local path=store.path(self.manifest.id) self.engine:saveSnapshot(path..'/start.sav') - store.write(path..'/rng.bin',core.readString(self.engine.rng,0x9c50)) + store.write(path..'/rng.bin',self.engine:rngData()) self.manifest.snapshotHash=sha.sha256(store.read(path..'/start.sav')) self.manifest.rngHash=sha.sha256(store.read(path..'/rng.bin')) self.manifest.player=self.engine:player() @@ -98,6 +99,7 @@ function Session:startPlayback(id) self.engine:loadSnapshot(path..'/start.sav') local bytes={}; for i=1,#rng do bytes[i]=rng:byte(i) end core.writeBytes(self.engine.rng,bytes) + self:checkRngData(manifest.rngHash,'starting save') assert(self.engine:tick()==manifest.startTick,'Loaded save has a different starting tick') assert(self.engine:player()==manifest.player,'Loaded save has a different player slot') self:checkResources(manifest.startResources,'starting save') @@ -137,9 +139,13 @@ function Session:onTick() self.manifest.lastTick=now self.manifest.finalRng=self.engine:rngState() self.manifest.finalResources=self.engine:resourceState() + -- Keep the exact last observed boundary; quitting may already change native state. + -- Hash only checkpoints and completion, not every simulation tick. + self.finalRngData=self.engine:rngData() self.observedTick=true if now%64==0 then - local line=json:encode({time=now,rng=self.engine:rngState(),resources=self.manifest.finalResources}) + local line=json:encode({time=now,rng=self.engine:rngState(),resources=self.manifest.finalResources, + rngHash=sha.sha256(self.finalRngData)}) assert(self.rngFile:write(line..'\n')); assert(self.rngFile:flush()) end elseif self.status=='playing' then @@ -158,6 +164,7 @@ function Session:onTick() end end self:checkResources(expected.resources,'checkpoint') + self:checkRngData(expected.rngHash,'checkpoint') end if now>=self.manifest.lastTick then assert(self.playedCommands==self.manifest.commandCount,'Replay ended before all commands were scheduled') @@ -166,13 +173,25 @@ function Session:onTick() local actual=self.engine:rngState() for i=1,4 do assert(actual[i]==self.manifest.finalRng[i],'Final RNG state differs at tick '..now) end self:checkResources(self.manifest.finalResources,'ending state') + self:checkRngData(self.manifest.finalRngHash,'ending state') self.status='finished'; core.writeInteger(self.halt,1); self.engine:pause() store.write(store.path(self.manifest.id)..'/last-playback.json',json:encode({ - status='finished',lastTick=now,commands=self.playedCommands,rngCheckpoints='matched',resourceCheckpoints='matched'})) + status='finished',lastTick=now,commands=self.playedCommands,rngCheckpoints='matched', + fullRngCheckpoints='matched',resourceCheckpoints='matched'})) end end end +function Session:checkRngData(expected,phase) + validation.hash(expected,'full RNG hash') + local actual=sha.sha256(self.engine:rngData()) + if actual~=expected then + self.firstDesync={kind='rng-state',time=self.engine:tick(),phase=phase,expected=expected,actual=actual} + store.write(store.path(self.manifest.id)..'/desync.json',json:encode(self.firstDesync)) + error('Full RNG state divergence at tick '..self.engine:tick()..' ('..phase..')') + end +end + function Session:checkResources(expected,phase) validation.resources(expected) local actual=self.engine:resourceState() @@ -198,7 +217,11 @@ function Session:reset() local closed,reason=pcall(Base.reset,self) if manifest then if complete and closed then - local ok,finishError=pcall(store.finish,manifest) + local ok,finishError=pcall(function() + assert(self.finalRngData,'Missing ending RNG state') + manifest.finalRngHash=sha.sha256(self.finalRngData) + store.finish(manifest) + end) if not ok then manifest.status='failed'; pcall(store.save,manifest) error(finishError) @@ -211,6 +234,7 @@ function Session:reset() assert(closed,reason) self.status='idle'; self.manifest=nil; self.nextCheckpoint=nil self.observedTick=false; self.error=nil + self.finalRngData=nil core.writeInteger(self.halt,0) end diff --git a/code/sessions.lua b/code/sessions.lua index 43f78a1..17fd3c7 100644 --- a/code/sessions.lua +++ b/code/sessions.lua @@ -1,6 +1,6 @@ local platform = require('code/platform') local validation = require('code/validation') -local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v4'} +local M = {FORMAT = 1, ROOT = 'ucp/replays', PROFILE = 'recorder-sp-v5'} local activeSettings local function read(path) @@ -155,6 +155,7 @@ function M.preflight(manifest) assert(type(checkpoint)=='table' and checkpoint.time==tick and tick<=manifest.lastTick,'Invalid replay checkpoint timeline') validation.rng(checkpoint.rng) validation.resources(checkpoint.resources) + validation.hash(checkpoint.rngHash,'checkpoint RNG hash') tick=tick+64 end assert(tick>manifest.lastTick,'Replay verification data ended early') diff --git a/code/validation.lua b/code/validation.lua index f685267..93ef22e 100644 --- a/code/validation.lua +++ b/code/validation.lua @@ -70,13 +70,16 @@ function M.manifest(value) M.rng(value.finalRng) M.resources(value.startResources) M.resources(value.finalResources) - for _,key in ipairs({'settingsHash','environmentHash','snapshotHash','rngHash','commandsHash','checkpointsHash','infoHash'}) do - assert(type(value[key])=='string' and #value[key]==64 and not value[key]:find('[^%x]'), - 'Invalid replay '..key) + for _,key in ipairs({'settingsHash','environmentHash','snapshotHash','rngHash','finalRngHash','commandsHash','checkpointsHash','infoHash'}) do + M.hash(value[key],key) end return value end +function M.hash(value,label) + assert(type(value)=='string' and #value==64 and not value:find('[^%x]'),'Invalid replay '..label) +end + function M.resources(values) assert(type(values)=='table' and #values==200,'Invalid replay resource state') for i=1,200 do M.integer(values[i],-2147483648,2147483647,'resource amount') end diff --git a/definition.yml b/definition.yml index 97c5d5d..420ceea 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.10.0 +version: 0.11.0 type: module author: Corax meta: diff --git a/docs/multiplayer-diagnostics.md b/docs/multiplayer-diagnostics.md index a6ebc9b..6ddfb55 100644 --- a/docs/multiplayer-diagnostics.md +++ b/docs/multiplayer-diagnostics.md @@ -1,4 +1,4 @@ -# Multiplayer diagnostics (0.10.0) +# Multiplayer diagnostics (0.11.0) This is the first multiplayer-specific implementation stage. It observes execution on each peer without changing the simulation scope, RNG seeds, pause flag, command ownership or dispatch decisions. Multiplayer recording/playback remains disabled. The existing single-player replay checks and isolation gates remain in effect. @@ -24,3 +24,13 @@ This is command-boundary and periodic simulation evidence, not a full replay or Automated tests use native callback fixtures to verify original registers/dispatch survive logging failures, remote actors are recorded, single-player does not start a multiplayer trace, and incomplete evidence is rejected. The comparator has matching/differing/corrupt trace tests. No live multiplayer match was launched for this stage. Format 2 gives every command/checkpoint an ordered evidence sequence and records both event and command totals. The comparator detects skipped checkpoint boundaries, including matching files with the same missing interval. It accepts older format-1 pairs as command-only evidence; mixing formats is rejected. A short interval before the first recorded event is outside the trace window. + +Format 3 additionally records SHA-256 of the complete 40,016-byte RNG structure at +each periodic checkpoint, detecting differences in stored random arrays before +they appear in the current values/indices. Command rows retain their smaller +post-handler RNG summary. The header includes the resolved UCP settings, +framework and extension order/version environment hash; different environments +are reported as incompatible. This does not fingerprint edited extension files +or external assets. Older format-1/2 pairs can still be compared with their +original evidence limits. Full RNG hashing adds periodic CPU work; live overhead +and two-peer comparisons still need measurement. diff --git a/docs/replay-sessions.md b/docs/replay-sessions.md index db635d8..bc66c68 100644 --- a/docs/replay-sessions.md +++ b/docs/replay-sessions.md @@ -24,6 +24,15 @@ The local single-player identity stays intact. New commands queued by a spectato ## Configuration compatibility +Version 0.11.0 uses simulation profile `recorder-sp-v5`. It additionally compares +SHA-256 of all `0x9c50` bytes of the native RNG structure: current values, seed, +stored arrays and indices. Checks run after restoration, every 64 ticks and at +the ending boundary. A mismatch writes a `rng-state` diagnostic with its phase, +tick and expected/actual hashes. Hash evidence is validated during preflight. +The recorder retains the last observed RNG bytes at each tick and hashes that +retained state when finishing, rather than reading possibly reset state after +quitting. Older experimental recordings require their original recorder version. + Settings are captured when the module enables, not reread when the user begins a recording. Playback requires the captured configuration bytes and the resolved configuration/extension-version environment to match. A restart with the saved configuration is necessary when these differ. This stage does not relaunch automatically. Version equality does not prove that unpacked extensions have identical source files, or that map assets and external resources are unchanged. No extension download, installation or version switching is performed. Replays are local test artifacts; native save/payload handling has not been audited as a parser for hostile downloaded files. diff --git a/tests/test_engine.py b/tests/test_engine.py index a5fdb2e..eeba98c 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -5,6 +5,16 @@ class EngineTests(unittest.TestCase): check = fixture.RecorderTests.check + def test_full_rng_read_keeps_zero_bytes_and_rejects_short_reads(self): + self.check(''' +local data=string.rep('x',0x9c4f)..string.char(0) +core.readString=function(address,size) + assert(address==engine.rng and size==0x9c50); return data +end +assert(engine:rngData()==data) +data=data:sub(1,-2); assert(not pcall(function() engine:rngData() end)) +''') + def setUp(self): fixture.RecorderTests.setUp(self) self.check(''' diff --git a/tests/test_multiplayer_trace.py b/tests/test_multiplayer_trace.py index 28ac274..d8cc0fe 100644 --- a/tests/test_multiplayer_trace.py +++ b/tests/test_multiplayer_trace.py @@ -16,6 +16,9 @@ def setUp(self): self.command_hooks('record') self.check(''' package.loaded['code/platform']={mkdir=function() return true end} +require('code/sessions').settings=function() return {environmentHash=string.rep('a',64)} end +sha={sha256=function() return string.rep('b',64) end} +engine.rngData=function() return string.rep('x',0x9c50) end traceRows={}; json.encode=function(_,v) traceRows[#traceRows+1]=v; return 'json' end realNative.profile.name='SHC'; realNative.profile.sha256=string.rep('a',64) engine.trace=require('code/multiplayer-trace').new(engine) @@ -77,12 +80,24 @@ def test_tick_evidence_does_not_need_commands_and_deduplicates_paused_boundary(s memory[0x1fe7da8]=64 engine.trace:observe('onTick'); engine.trace:observe('onTick') assert(#traceRows==2 and traceRows[2].kind=='checkpoint' and traceRows[2].time==64) +assert(traceRows[1].format==3 and traceRows[1].environmentHash==string.rep('a',64)) +assert(traceRows[2].rngHash==string.rep('b',64)) memory[0x1fe7da8]=65; engine.trace:observe('onTick'); assert(#traceRows==2) memory[0x1fe7da8]=128; engine.trace:observe('onTick') engine.trace:observe('stop','test ended') assert(traceRows[4].commands==0 and traceRows[4].events==2 and traceRows[4].status=='complete') ''') + def test_full_rng_capture_failure_disables_observer_without_blocking_dispatch(self): + self.check(''' +memory[0x1fe7da8]=64 +engine.rngData=function() error('RNG read failed') end +engine.trace:observe('onTick') +assert(engine.trace.failed and not engine.trace.file) +assert(receive().EAX==1 and before().EDX==28) +assert(recorder.mode=='none' and not recorder.error) +''') + class CompareTraceTests(unittest.TestCase): def setUp(self): @@ -154,3 +169,29 @@ def test_missing_checkpoint_cannot_be_hidden_in_a_matching_pair(self): a[0].update(format=2, firstTick=0) a[-1].update(events=1) self.assertEqual(self.compare(a, a)['status'], 'incomplete') + + def full_trace(self): + return [dict(kind='header', format=3, variant='SHC', executable='a', firstTick=64, + environmentHash='a'*64), + dict(kind='checkpoint', sequence=1, time=64, rng=[1, 2, 3, 4], + resources=[0]*200, rngHash='b'*64), + dict(kind='end', status='complete', events=1, commands=0)] + + def test_full_rng_difference_is_found_with_matching_values_and_indices(self): + a, b = self.full_trace(), self.full_trace() + self.assertEqual(self.compare(a, b)['status'], 'matched') + b[1]['rngHash'] = 'c'*64 + self.assertEqual(self.compare(a, b)['firstDifference']['field'], 'rngHash') + + def test_different_environment_or_missing_full_rng_cannot_match(self): + for change in ('environment', 'header hash', 'state hash', 'invalid hash'): + a, b = self.full_trace(), self.full_trace() + if change == 'environment': + b[0]['environmentHash'] = 'c'*64 + elif change == 'header hash': + del b[0]['environmentHash'] + elif change == 'state hash': + del b[1]['rngHash'] + else: + b[1]['rngHash'] = 'z'*64 + self.assertEqual(self.compare(a, b)['status'], 'incomplete', change) diff --git a/tests/test_session_files.py b/tests/test_session_files.py index 68f02cf..a6ffcd0 100644 --- a/tests/test_session_files.py +++ b/tests/test_session_files.py @@ -12,6 +12,20 @@ class SessionFileTests(unittest.TestCase): + def test_missing_full_rng_evidence_prevents_completion(self): + self.lua.execute(''' +local m=recording(); m.finalRngHash=nil +assert(not pcall(store.finish,m)) +m=recording() +local path=store.path(m.id)..'/stream-rng-sync.json' +local lines={} +for line in store.read(path):gmatch('[^\\n]+') do + local row=json:decode(line); row.rngHash=nil; lines[#lines+1]=json:encode(row) +end +store.write(path,table.concat(lines,'\\n')..'\\n') +assert(not pcall(store.finish,m)) +''') + def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) @@ -58,11 +72,11 @@ def mkdir(path): local m=store.new(profile) m.player=1; m.startTick=0; m.lastTick=64; m.finalRng={11,22,3,4} m.startResources=resourceState(); m.finalResources=resourceState() - m.snapshotHash=string.rep('b',64); m.rngHash=string.rep('c',64) + m.snapshotHash=string.rep('b',64); m.rngHash=string.rep('c',64); m.finalRngHash=string.rep('d',64) local path=store.path(m.id) local function command(t) return {commandCategory=28,player=1,time=t,size=1,data='01'} end store.write(path..'/stream-commands.json',json:encode(command(10))..'\\n'..json:encode(command(65))..'\\n') - store.write(path..'/stream-rng-sync.json',json:encode({time=0,rng={1,2,3,4},resources=resourceState()})..'\\n'..json:encode({time=64,rng=m.finalRng,resources=resourceState()})..'\\n') + store.write(path..'/stream-rng-sync.json',json:encode({time=0,rng={1,2,3,4},resources=resourceState(),rngHash=m.rngHash})..'\\n'..json:encode({time=64,rng=m.finalRng,resources=resourceState(),rngHash=m.finalRngHash})..'\\n') store.write(path..'/stream-infself.json',json:encode({gameType=0,mapSeed=123,matchSeed=123,RNGvalue1=1,RNGvalue2=2,RNGindex1=4,RNGindex2=3})..'\\n') return m end diff --git a/tests/test_sessions.py b/tests/test_sessions.py index e1ff84f..a09d58f 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -1,3 +1,4 @@ +import hashlib import unittest import test_recorder as fixture @@ -5,11 +6,41 @@ class SessionTests(unittest.TestCase): check = fixture.RecorderTests.check + def test_full_rng_array_mismatch_halts_at_checkpoint_and_ending_boundary(self): + self.lua.globals().hash_string = lambda value: hashlib.sha256(value.encode()).hexdigest() + self.check(''' +sha.sha256=hash_string +local original=engine:rngData() +local expected=sha.sha256(original) +-- Change an unconsumed byte while rngState() still returns the same four values. +engine.rngData=function() return original:sub(1,1000)..'y'..original:sub(1002) end +for _,tick in ipairs({64,65}) do + local r=session(); r.mode='play'; r.status='playing'; r.active=true; r.playedCommands=0 + r.manifest={id='test',lastTick=tick,commandCount=0,finalRng={11,22,3,4}, + finalResources=resourceState(),finalRngHash=expected} + r.rngFile={read=function() return {time=64,rng={11,22,3,4},resources=resourceState(),rngHash=expected} end} + now=tick; assert(not r:guard(function() r:onTick() end)) + assert(r.firstDesync.kind=='rng-state' and r.firstDesync.time==tick and paused) + assert(r.firstDesync.phase==(tick==64 and 'checkpoint' or 'ending state')) +end +''') + + def test_completion_hash_uses_last_observed_state_and_releases_buffer(self): + self.lua.globals().hash_string = lambda value: hashlib.sha256(value.encode()).hexdigest() + self.check(''' +sha.sha256=hash_string +local r=session(); r:startRecording(); r:activateRecording(); now=65; r:onTick() +local expected=sha.sha256(engine:rngData()) +engine.rngData=function() error('Must not read game state after leaving match') end +r:reset() +assert(savedManifest.finalRngHash==expected and not r.finalRngData) +''') + def setUp(self): fixture.RecorderTests.setUp(self) self.check(''' json.encode=function(_,value) lastEncoded=value; return 'json' end -sha={sha256=function(value) return 'hash' end} +sha={sha256=function(value) return string.rep('a',64) end} savedManifest=nil package.loaded['code/sessions']={ new=function() return {id='test',variant='SHC',commandCount=0,lastTick=0} end, @@ -21,6 +52,7 @@ def setUp(self): Session=require('code/session-recorder') now=0; snapshots=0; space=true engine={rng=0x1a279c0, + rngData=function() return string.rep('x',0x9c50) end, resourceState=function() return resourceState() end, resetCommands=function(self) self.journal={executed=0} end, journal={executed=0}, @@ -107,9 +139,9 @@ def test_end_of_replay_requires_all_commands(self): def test_final_checkpoint_boundary_finishes_without_next_checkpoint(self): self.check(''' local r=session(); r.status='playing'; r.mode='play'; r.active=true; r.playedCommands=0 -r.manifest={id='test',lastTick=64,commandCount=0,finalRng={11,22,3,4},finalResources=resourceState()} +r.manifest={id='test',lastTick=64,commandCount=0,finalRng={11,22,3,4},finalResources=resourceState(),finalRngHash=string.rep('a',64)} local reads=0 -r.rngFile={read=function() reads=reads+1; assert(reads==1); return {time=64,rng={11,22,3,4},resources=resourceState()} end} +r.rngFile={read=function() reads=reads+1; assert(reads==1); return {time=64,rng={11,22,3,4},resources=resourceState(),rngHash=string.rep('a',64)} end} now=64; r:onTick(); r:onTick() assert(r.status=='finished' and paused and reads==1) ''') diff --git a/tools/compare_multiplayer.py b/tools/compare_multiplayer.py index 0c53443..9488fc2 100644 --- a/tools/compare_multiplayer.py +++ b/tools/compare_multiplayer.py @@ -10,6 +10,11 @@ def integer(value, lo, hi): raise ValueError('Invalid integer in trace') +def hash_value(value): + if not isinstance(value, str) or len(value) != 64 or any(c not in '0123456789abcdef' for c in value): + raise ValueError('Missing or invalid SHA256 evidence') + + def rows(stream): while line := stream.readline(65537): if len(line) > 65536: @@ -28,12 +33,12 @@ def evidence(records, format_version, first_tick=0): for record in records: if record.get('kind') == 'end': if (record.get('status') != 'complete' or record.get('commands') != command_count or not count - or (format_version == 2 and record.get('events') != count)): + or (format_version >= 2 and record.get('events') != count)): raise ValueError('Trace is incomplete or has no evidence') if next(records, None) is not None: raise ValueError('Data follows trace completion') return - allowed = ('command', 'checkpoint') if format_version == 2 else ('command',) + allowed = ('command', 'checkpoint') if format_version >= 2 else ('command',) if record.get('kind') not in allowed: raise ValueError('Trace contains an untracked/unknown command') count += 1 @@ -43,7 +48,7 @@ def evidence(records, format_version, first_tick=0): previous_tick = record['time'] if record['kind'] == 'command': command_count += 1 - if format_version == 2 and record['time'] > next_checkpoint: + if format_version >= 2 and record['time'] > next_checkpoint: raise ValueError('Missing periodic checkpoint evidence') fields = [('scheduledTime', 1, 2147483647), ('player', 1, 8), ('category', 0, 127), ('size', 0, 1260)] @@ -58,6 +63,8 @@ def evidence(records, format_version, first_tick=0): raise ValueError('Missing, repeated or invalid checkpoint boundary') else: next_checkpoint += 64 + if format_version >= 3: + hash_value(record.get('rngHash')) for key, length in [('rng', 4), ('resources', 200)]: values = record.get(key) if not isinstance(values, list) or len(values) != length: @@ -76,10 +83,14 @@ def compare(left, right): ar, br = rows(a), rows(b) ah, bh = next(ar), next(br) for header in (ah, bh): - if header.get('kind') != 'header' or header.get('format') not in (1, 2): + if header.get('kind') != 'header' or header.get('format') not in (1, 2, 3): raise ValueError('Unsupported trace format') + if header['format'] >= 3: + hash_value(header.get('environmentHash')) if any(ah.get(key) != bh.get(key) for key in ('variant', 'executable', 'format')): raise ValueError('Traces require the same game executable, variant and format') + if ah['format'] >= 3 and ah['environmentHash'] != bh['environmentHash']: + raise ValueError('Traces require the same resolved UCP settings, extension order/versions and framework') for av, bv in itertools.zip_longest(evidence(ar, ah['format'], ah.get('firstTick', 0)), evidence(br, bh['format'], bh.get('firstTick', 0))): count += 1 @@ -91,6 +102,8 @@ def compare(left, right): fields = ('kind', 'time') if av['kind'] == bv['kind'] == 'command': fields += ('scheduledTime', 'player', 'category', 'size', 'data') + elif av['kind'] == bv['kind'] == 'checkpoint' and ah['format'] >= 3: + fields += ('rngHash',) for key in fields + ('rng', 'resources'): if av[key] == bv[key]: continue From 31a26e445a0ef2750badc87a866e815009a7254c Mon Sep 17 00:00:00 2001 From: Krarilotus <51748815+Krarilotus@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:14:28 +0200 Subject: [PATCH 18/19] Repair live replay integration and isolate mood music RNG --- CHANGELOG.md | 12 +++++ README.md | 8 ++-- code/engine.lua | 24 +++++----- code/native-ui.lua | 8 ++-- code/platform.lua | 70 ++++++++++++++++++++++++--- code/resource-hooks.lua | 38 +++++++++++++++ code/scoped-code.lua | 1 + code/scoped-sites.lua | 18 ++++++- code/session-recorder.lua | 11 +++++ code/sessions.lua | 2 +- code/ui.lua | 8 ++-- definition.yml | 2 +- docs/live-validation.md | 55 +++++++++++++++++++++ docs/replay-sessions.md | 6 +-- init.lua | 2 +- tests/check_executables.py | 15 +++++- tests/test_engine.py | 6 +-- tests/test_native_ui.py | 2 +- tests/test_platform.py | 93 ++++++++++++++++++++++++++++++++++++ tests/test_resource_hooks.py | 55 +++++++++++++++++++++ tests/test_scoped_code.py | 28 +++++++++++ tests/test_sessions.py | 20 ++++++++ tests/test_win32_abi.py | 8 ++-- 23 files changed, 447 insertions(+), 45 deletions(-) create mode 100644 code/resource-hooks.lua create mode 100644 docs/live-validation.md create mode 100644 tests/test_platform.py create mode 100644 tests/test_resource_hooks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a1abf3..00341fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.12.0 + +- Fix the lobby crash caused by Lua filename hooks clobbering registers used by native callers. Use native filename overrides only while loading the replay's starting save. +- Fix the first-tick crash by moving the pause guard away from an incoming branch target, preserving the original unpaused entry path. +- Bootstrap Windows file operations from loaded system exports, including named forwarders, and compile the stdcall bridge through the supported UCP assembly API. +- Terminate native path/text buffers explicitly, use the initialized game font with a shadow, and place Record/Replays in gaps between existing Skirmish controls. +- Capture the starting save at the first simulation boundary following match initialization. +- Guard all seven mood-selection calls that advance simulation RNG, in addition to the previous music guards, for Crusader and Extreme. Music keeps using the current RNG value; this can change track variety. These changes apply only during single-player recorder sessions; idle and multiplayer paths retain native behavior. +- Use simulation profile `recorder-sp-v6`; older captures require their original module version. + +Validation: 104 automated tests and original-executable checks for both variants, including complete mood-function RNG-call coverage. Live Crusader tests verified recording, browser selection and starting-save loading, and exposed RNG drift traced to music selection. The final music fix has not yet been retested in game. Complete playback, player commands, Extreme, settings restart and multiplayer still need live validation; multiplayer replay remains disabled. See [live evidence](docs/live-validation.md). + ## 0.11.0 - Compare SHA-256 of the complete 40,016-byte RNG structure after restoring the starting save, at periodic checkpoints and at the ending boundary. Detect different stored random arrays even when current RNG values and indices match. diff --git a/README.md b/README.md index 3ddc20a..5650a8b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.11.0 verifies full RNG state, player resources and native command execution. It confines recorder simulation changes to requested single-player sessions and hides its controls in multiplayer. It includes a native replay browser, settings restart, starting saves and settings snapshots. **The save/load playback path, browser rendering and real restart still need in-game verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. +Version 0.12.0 repairs crashes and file/UI integration failures found in live Crusader tests, and guards seven previously missed music calls that advance the match RNG. Recordings, the native replay browser and loading a starting save have been exercised in game. **A complete replay with the final music fix, Extreme gameplay and settings restart still need live verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. See [live findings and next tests](docs/live-validation.md). Automarket 1.1.0 has an experimental replay adapter for its settings commits and native custom save section. Use protocol 1.0.0 and map-extensions 1.0.0, with **recorder after protocol in the extension order**. The normal weekly trades run in the simulation; they are not replayed as extra trades. Other custom protocols remain unsupported. See [Automarket replay notes](docs/automarket-replay.md). @@ -13,7 +13,7 @@ Optional **Multiplayer diagnostics** logs actual command execution and periodic Use a separate game installation with UCP3 developer mode and this module enabled. If your game needs Graphics API Replacer, keep it and its dependencies enabled in the test configuration. 1. Open a single-player Skirmish lobby and configure the match. -2. Click **Record next match** near the bottom. This arms a new recording; **Cancel recording** cancels it. +2. Click **Record** near the bottom. This arms a new recording; **Cancel** cancels it. 3. Start the match. The module captures the starting save and begins recording. 4. Leave through **Quit Mission** to finalize the recording. Do not terminate the process to finish a replay. 5. Return to a Skirmish lobby, open **Replays**, select a completed session and click **Play**. The saved starting state replaces manual map/AI setup. @@ -28,10 +28,10 @@ See [session format and limitations](docs/replay-sessions.md), [browser and rest ## Development ```sh -python -m pip install lupa==2.6 unicorn==2.1.4 +python -m pip install lupa==2.6 unicorn==2.1.4 capstone==5.0.7 python -m unittest discover -s tests -v python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.11.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.12.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/engine.lua b/code/engine.lua index e69feb2..4ff51d2 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -37,6 +37,7 @@ function M.new(sites) e.loadNative=core.exposeCode(sites.load.address,1,0) e.buffer=core.allocate(1260,true) e.pathBuffer=core.allocate(512,true) + e.pathOverride=core.allocate(4,true) e.scope=core.allocate(4,true) M.resetCommands(e) return setmetatable(e,{__index=M}) @@ -92,10 +93,10 @@ function M:saveSnapshot(path) local oldName=core.readBytes(filename,1001) local oldProgress=core.readInteger(self.sites.packager+0x20) core.writeInteger(resource+0xbc4,1) - core.writeString(filename,path) + core.writeString(filename,path..'\0') core.writeInteger(self.sites.packager+0x20,0) -- no progress callback/audio during capture temporarily(function() self.saveNative(self.sites.packager,self.sites.sections) end,function() - core.writeBytes(filename,oldName) -- writeString would add a byte past this fixed array. + core.writeBytes(filename,oldName) -- Restore all bytes of the fixed native array. core.writeInteger(resource+0xbc4,oldType) core.writeInteger(self.sites.packager+0x20,oldProgress) end) @@ -110,13 +111,15 @@ function M:loadSnapshot(path) local old={} for _,offset in ipairs({0x58,0x7c,0x80,0x884}) do old[offset]=core.readInteger(state+offset) end self.overridePath=path - core.writeString(self.pathBuffer,path) + core.writeString(self.pathBuffer,path..'\0') core.writeInteger(state+0x58,31) -- native Load action core.writeInteger(state+0x7c,1) core.writeInteger(state+0x80,0) core.writeInteger(state+0x884,0) self.loading=true + core.writeInteger(self.pathOverride,1) temporarily(function() self.loadNative(0) end,function() + core.writeInteger(self.pathOverride,0) self.loading=false self.overridePath=nil -- These four fields belong to the load dialog; preserve the game's menu transition. @@ -196,16 +199,11 @@ function M:afterCommand(recorder) end function M:install(recorder) - local originalFileName, originalMapName, originalQueue - originalFileName=core.hookCode(function(this) - if self.overridePath and core.readInteger(this+0xbc4)==1 then return self.pathBuffer end - return originalFileName(this) - end,self.sites.fileName.address,1,1,#self.sites.fileName.bytes) - local dummy=core.allocate(16,true); core.writeString(dummy,'replay') - originalMapName=core.hookCode(function(this,index) - if self.overridePath then return dummy end - return originalMapName(this,index) - end,self.sites.mapName.address,2,1,#self.sites.mapName.bytes) + local originalQueue + local resources=require('code/resource-hooks') + resources.install(self.sites.fileName,self.pathOverride,self.pathBuffer,true) + local dummy=core.allocate(16,true); core.writeString(dummy,'replay\0') + resources.install(self.sites.mapName,self.pathOverride,dummy,false) originalQueue=core.hookCode(function(this,category) if recorder.mode=='play' and self:singlePlayer() and not self.loading then return 0 end return originalQueue(this,category) diff --git a/code/native-ui.lua b/code/native-ui.lua index 741165d..2884c88 100644 --- a/code/native-ui.lua +++ b/code/native-ui.lua @@ -33,9 +33,11 @@ end function M:text(label,x,y,alignment) label=tostring(label):gsub('[\r\n%z]',' '):sub(1,150) - core.writeString(self.textBuffer,label) - local color=core.readSmallInteger(self.sites.gold.value)%65536 - self.textNative(self.sites.textManager.value,self.textBuffer,x,y,alignment or 0,color,0,0,0) + core.writeString(self.textBuffer,label..'\0') + -- Font slots 0..14 are uninitialized in the HD menus. Slot 19 is the + -- game's 16-pixel antialiased font; text expects BGR24, unlike RGB15 borders. + self.textNative(self.sites.textManager.value,self.textBuffer,x+1,y+1,alignment or 0,0,19,0,0) + self.textNative(self.sites.textManager.value,self.textBuffer,x,y,alignment or 0,0x9AD7E4,19,0,0) end function M:button(address,x,y,width,height,label,action,selected) diff --git a/code/platform.lua b/code/platform.lua index 29f255c..4dab3f9 100644 --- a/code/platform.lua +++ b/code/platform.lua @@ -1,24 +1,82 @@ -- Small Win32 adapter; stdcall is wrapped explicitly because RPS exposeCode -- supports cdecl/thiscall, not stdcall. No shell command construction is used. local M = {} -local libraries = {} local buffers = {} local function buffer(key,value) assert(type(value)=='string' and #value<4096 and not value:find('\0',1,true),'Invalid replay path') buffers[key]=buffers[key] or core.allocate(4096,true) - core.writeString(buffers[key],value) + core.writeString(buffers[key],value..'\0') return buffers[key] end -function M.stdcall(library, name, count) - if not libraries[library] then libraries[library] = assert(core.openLibraryHandle(library)) end - local address = assert(libraries[library]:getProcAddress(name)) +function M.stdcallAddress(address,count) local code = {0x55, 0x8B, 0xEC} -- push ebp; mov ebp,esp for i = count, 1, -1 do code[#code+1] = 0xFF; code[#code+1] = 0x75; code[#code+1] = 4+i*4 end code[#code+1] = core.callTo(address) code[#code+1] = 0x5D; code[#code+1] = 0xC3 - return core.exposeCode(core.allocateCode(code), count, 0) + local target=core.allocateCode(core.calculateCodeSize(code)) + core.writeCode(target,code) + return core.exposeCode(target,count,0) +end +local wrap=M.stdcallAddress + +-- UCP's library loader accepts extension DLLs, not Windows system libraries. +-- Bootstrap GetProcAddress from the already loaded kernel32 export directory; +-- use the game's verified GetModuleHandleA import (ASLR-resolved by Windows). +local getModule,getProc +local function bootstrap() + local profile=require('code/native').profile.name + local imports={SHC=0x59e128,Extreme=0x59e12c} + getModule=wrap(core.readInteger(assert(imports[profile],'Unknown Windows import profile')),1) + local function exported(library,name,depth) + assert(depth<8,'Cyclic Windows export forwarder') + local base=getModule(buffer('library',library))%4294967296 + assert(base~=0,'Windows library is not loaded: '..library) + local function u32(a) return core.readInteger(a)%4294967296 end + local function u16(a) return core.readSmallInteger(a)%65536 end + assert(u16(base)==0x5a4d,'Invalid Windows library header') + local pe=base+u32(base+0x3c) + assert(u32(pe)==0x4550 and u16(pe+24)==0x10b,'Expected PE32 Windows library') + local start,size=u32(pe+120),u32(pe+124) + local exports=base+start + local count=u32(exports+24) + assert(start>0 and count<65536,'Invalid Windows exports') + local names,ordinals,functions=base+u32(exports+32),base+u32(exports+36),base+u32(exports+28) + local function cstring(a) + local chars={} + for i=0,255 do + local byte=core.readByte(a+i) + if byte==0 then return table.concat(chars) end + chars[#chars+1]=string.char(byte) + end + error('Windows export name is too long') + end + for i=0,count-1 do + if cstring(base+u32(names+i*4))==name then + local ordinal=u16(ordinals+i*2) + assert(ordinal=start and rva=8 or (enabled and mode in (0,99))) + self.assertEqual(machine.reg_read(UC_X86_REG_EIP),address+0x4f if skip else address+14) + if not skip: self.assertEqual(machine.reg_read(UC_X86_REG_ECX),0x12345678) + def test_optional_tick_diagnostics_preserve_native_multiplayer_execution(self): engines=self.lua.execute((ROOT/'code/engine-sites.lua').read_text()) for variant,engine in engines.items(): diff --git a/tests/test_sessions.py b/tests/test_sessions.py index a09d58f..f3e37b3 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -6,6 +6,26 @@ class SessionTests(unittest.TestCase): check = fixture.RecorderTests.check + def test_start_callback_defers_snapshot_to_one_simulation_boundary(self): + self.check(''' +local r=session(); r:startRecording(); r:prepareRecording() +assert(snapshots==0 and not r.active) +now=1; r:onTick() +assert(snapshots==1 and r.active and r.manifest.startTick==1) +now=2; r:onTick(); assert(snapshots==1) +r:reset(); r:startRecording(); r:prepareRecording(); r:reset() +r:startRecording(); now=3; r:onTick() +assert(snapshots==1 and not r.active and not r.capturePending) +''') + + def test_pending_snapshot_is_cancelled_by_multiplayer_transition(self): + self.check(''' +local r=session(); r:startRecording(); r:prepareRecording() +engine.singlePlayer=function() return false end +now=1; r:onTick() +assert(snapshots==0 and not r.capturePending and r.mode=='none') +''') + def test_full_rng_array_mismatch_halts_at_checkpoint_and_ending_boundary(self): self.lua.globals().hash_string = lambda value: hashlib.sha256(value.encode()).hexdigest() self.check(''' diff --git a/tests/test_win32_abi.py b/tests/test_win32_abi.py index 3a0666f..808939c 100644 --- a/tests/test_win32_abi.py +++ b/tests/test_win32_abi.py @@ -22,14 +22,15 @@ def test_generated_bridges_preserve_argument_order_and_caller_stack(self): lua = LuaRuntime() lua.execute(''' core={ - openLibraryHandle=function() return {getProcAddress=function() return 0x102000 end} end, callTo=function(address) return {target=address} end, - allocateCode=function(code) captured=code; return 0x100000 end, + calculateCodeSize=function(code) return #code+4 end, + allocateCode=function(size) assert(type(size)=='number'); allocated=size; return 0x100000 end, + writeCode=function(address,code) assert(address==0x100000); captured=code end, exposeCode=function(address,count,convention) return {address,count,convention} end, } ''') platform = lua.execute((ROOT/'code/platform.lua').read_text()) - bridge = platform.stdcall('fake.dll', 'fake', count) + bridge = platform.stdcallAddress(0x102000, count) self.assertEqual(list(bridge.values()), [0x100000, count, 0]) code = bytearray() for instruction in lua.globals().captured.values(): @@ -37,6 +38,7 @@ def test_generated_bridges_preserve_argument_order_and_caller_stack(self): code.append(instruction) else: code += b'\xe8'+struct.pack(' Date: Sat, 5 Sep 2026 22:35:47 +0200 Subject: [PATCH 19/19] Capture local multiplayer commands and expose network coverage gaps --- CHANGELOG.md | 11 ++++ README.md | 4 +- code/engine.lua | 13 ++++ code/multiplayer-trace.lua | 50 ++++++++++++++- code/network-observer.lua | 20 ++++++ code/network-sites.lua | 13 ++++ definition.yml | 2 +- docs/multiplayer-diagnostics.md | 47 +++++++++++++-- init.lua | 1 + tests/check_executables.py | 2 +- tests/test_multiplayer_trace.py | 104 +++++++++++++++++++++++++++++++- tools/compare_multiplayer.py | 47 +++++++++++++-- 12 files changed, 298 insertions(+), 16 deletions(-) create mode 100644 code/network-observer.lua create mode 100644 code/network-sites.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 00341fb..e6715d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.13.0 + +- Capture locally queued multiplayer payloads before native transmission; these do not pass through the received-command copy hook and were previously reported as untracked. +- Record the native eight-slot human/AI roster and transport identity in multiplayer diagnostic format 4. Compare logical rosters across peers, while checking each command's handle against its recorded actor. +- Observe both local and received immediate-command execution paths when multiplayer diagnostics are enabled. Mark them as coverage gaps instead of silently omitting them from a supposedly complete trace. +- Detect roster/identity changes and native synchronization phase changes between periodic checkpoints. Traces starting during resync, ambiguous/system handles, uncovered events and incompatible rosters cannot report a successful comparison. +- Preserve normal dispatch instructions/registers and isolate diagnostic errors. The three new native hooks are installed only when the optional diagnostics setting is enabled. +- Document the remaining network requirements, including system-message transitions, chunked resync, 24-bit wire timestamps and private extension state. Single-player profile remains `recorder-sp-v6`; older paired trace formats retain their original evidence limits. + +Validation: 112 automated tests, both original executable profiles and ZIP packaging pass. New tests cover local queue capture, roster differences, handle renumbering/ownership, between-checkpoint resync, immediate dispatch, hook conflicts and write-failure pass-through. No live multiplayer validation was performed; this improves diagnostics and does not enable multiplayer recording/playback. + ## 0.12.0 - Fix the lobby crash caused by Lua filename hooks clobbering registers used by native callers. Use native filename overrides only while loading the replay's starting save. diff --git a/README.md b/README.md index 5650a8b..e455799 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Work in progress: recording and playback of single-player Stronghold Crusader and Crusader Extreme Skirmishes using UCP3. -Version 0.12.0 repairs crashes and file/UI integration failures found in live Crusader tests, and guards seven previously missed music calls that advance the match RNG. Recordings, the native replay browser and loading a starting save have been exercised in game. **A complete replay with the final music fix, Extreme gameplay and settings restart still need live verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. See [live findings and next tests](docs/live-validation.md). +Version 0.13.0 adds multiplayer roster evidence and detects immediate-command/resync gaps in diagnostic traces. It includes 0.12.0's repairs for live Crusader crashes, file/UI integration and music RNG calls. Recordings, the native replay browser and loading a starting save have been exercised in game. **A complete replay with the final music fix, Extreme gameplay and settings restart still need live verification.** Multiplayer recording is not supported yet; normal multiplayer isolation has automated coverage but still needs a live comparison. See [live findings and next tests](docs/live-validation.md). Automarket 1.1.0 has an experimental replay adapter for its settings commits and native custom save section. Use protocol 1.0.0 and map-extensions 1.0.0, with **recorder after protocol in the extension order**. The normal weekly trades run in the simulation; they are not replayed as extra trades. Other custom protocols remain unsupported. See [Automarket replay notes](docs/automarket-replay.md). @@ -34,4 +34,4 @@ python tests/check_executables.py "PATH/TO/ORIGINAL/GAME" python tools/build.py ``` -The builder creates `dist/recorder-0.12.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. +The builder creates `dist/recorder-0.13.0.zip` with a flat module layout. `definition.yml` uses metadata schema version `1.0.0`; that is separate from the extension version. diff --git a/code/engine.lua b/code/engine.lua index 4ff51d2..ddf957d 100644 --- a/code/engine.lua +++ b/code/engine.lua @@ -85,6 +85,19 @@ function M:resourceState() return values end +function M:networkState() + local state={mode=core.readInteger(self.base+0x618),localPlayer=self:player(), + syncStatus=core.readInteger(self.base+0xb98),handles={},roster={}} + for slot=1,8 do + local handle=core.readInteger(self.base+0x6a8+slot*4) + local ai=core.readInteger(self.base+0x714+slot*4) + state.handles[slot]=handle + state.roster[slot]={slot=slot,kind=handle~=-1 and 'human' or (ai~=0 and 'ai' or 'empty'), + ai=ai,variation=core.readInteger(self.base+0x738+slot*4)} + end + return state +end + function M:saveSnapshot(path) assert(#path<500 and self:singlePlayer(), 'Snapshot requires a single-player game') local resource=self.sites.resources diff --git a/code/multiplayer-trace.lua b/code/multiplayer-trace.lua index f2b5ccc..d73fe99 100644 --- a/code/multiplayer-trace.lua +++ b/code/multiplayer-trace.lua @@ -29,12 +29,46 @@ function M:open() self.count=0; self.events=0 local environmentHash=store.settings().environmentHash validation.hash(environmentHash,'diagnostic environment hash') - self:write({kind='header',format=3,variant=native.profile.name,executable=native.profile.sha256, + self.network=self.engine:networkState() + self:write({kind='header',format=4,variant=native.profile.name,executable=native.profile.sha256, environmentHash=environmentHash, + network=self.network, localPlayer=self.engine:player(),firstTick=self.engine:tick()}) + if self.network.syncStatus~=0 then self:gap('capture began during native synchronization') end print('Multiplayer diagnostics: '..self.path) end +function M:gap(reason,details) + self.incomplete=true + self:record({kind='gap',time=self.engine:tick(),reason=reason,details=details}) +end + +function M:checkNetwork() + local state=self.engine:networkState() + local old=self.network + local changed=state.mode~=old.mode or state.localPlayer~=old.localPlayer + for slot=1,8 do + local a,b=state.roster[slot],old.roster[slot] + changed=changed or state.handles[slot]~=old.handles[slot] or a.ai~=b.ai or a.variation~=b.variation + end + if changed then self:gap('player roster or identity changed',state) end + if state.syncStatus~=old.syncStatus then self:gap('native synchronization phase changed',state) end + self.network=state +end + +function M:immediateCommand(source) + -- Lobby commands before the first observed simulation boundary are outside + -- this diagnostic window. Once open, do not silently omit immediate dispatch. + if not self.file then return end + self:checkNetwork() + local slot=validation.integer(core.readInteger(self.engine.base+0x2d824),0,199,'immediate ring slot') + local address=self.engine.base+0x3c67c+slot*1272 + self:gap('immediate command is outside timed replay coverage',{ + source=source,category=core.readByte(address+8),scheduledTime=core.readInteger(address), + player=core.readInteger(self.engine.base+self.engine.sites.actorOffset), + size=core.readInteger(self.engine.base+0x2d830)}) +end + function M:record(event) self.events=self.events+1 event.sequence=self.events @@ -43,6 +77,7 @@ end function M:onTick() local now=self.engine:tick() + if self.file then self:checkNetwork() end if now%64~=0 or now==self.lastTick then return end self:open() self.lastTick=now @@ -50,10 +85,16 @@ function M:onTick() rngHash=sha.sha256(self.engine:rngData())}) end -function M:receivedCommand(address,size) +function M:receivedCommand(address,size,origin) validation.integer(size,0,1260,'multiplayer trace payload length') local slot=validation.integer(core.readInteger(self.engine.base+0x2d824),0,199,'trace ring slot') - self.received[slot]={size=size,data=utils.tableToHex(core.readBytes(address,size))} + self.received[slot]={size=size,data=utils.tableToHex(core.readBytes(address,size)),origin=origin or 'received'} +end + +function M:locallyQueuedCommand() + local slot=validation.integer(core.readInteger(self.engine.base+0x2d824),0,199,'local ring slot') + local size=core.readInteger(self.engine.base+0x2d830) + self:receivedCommand(self.engine.base+0x3c67c+slot*1272+10,size,'local') end function M:beforeCommand() @@ -61,11 +102,13 @@ function M:beforeCommand() local slot=validation.integer(core.readInteger(self.engine.base+0x2d824),0,199,'trace ring slot') local address=self.engine.base+0x3c67c+slot*1272 self:open() + self:checkNetwork() local source=self.received[slot] local event={kind=source and 'command' or 'untracked',time=self.engine:tick(),slot=slot, scheduledTime=core.readInteger(address),handle=core.readInteger(address+4), player=core.readInteger(self.engine.base+self.engine.sites.actorOffset),category=core.readByte(address+8)} if source then + event.origin=source.origin event.size=source.size event.data=utils.tableToHex(core.readBytes(address+10,source.size)) event.changedSinceReceive=event.data~=source.data @@ -97,6 +140,7 @@ function M:stop(reason) end self.received={}; self.executing=nil; self.failed=false; self.incomplete=false; self.path=nil self.lastTick=nil + self.network=nil end function M:observe(event,...) diff --git a/code/network-observer.lua b/code/network-observer.lua new file mode 100644 index 0000000..eeb7f7f --- /dev/null +++ b/code/network-observer.lua @@ -0,0 +1,20 @@ +-- Opt-in diagnostics; detours retain all original instructions/registers. +local M={} +function M.install(trace) + local sites=require('code/network-sites')[require('code/native').profile.name] + for name,site in pairs(sites) do + local actual=core.readBytes(site.address,#site.bytes) + for i,byte in ipairs(site.bytes) do + assert(actual[i]==byte,'Multiplayer observer hook conflicts at '..name) + end + end + for name,site in pairs(sites) do + local source=name + local event=name=='localTimed' and 'locallyQueuedCommand' or 'immediateCommand' + core.detourCode(function(registers) + trace:observe(event,source) + return registers + end,site.address,#site.bytes) + end +end +return M diff --git a/code/network-sites.lua b/code/network-sites.lua new file mode 100644 index 0000000..090bb32 --- /dev/null +++ b/code/network-sites.lua @@ -0,0 +1,13 @@ +-- Local queue and immediate execution paths bypass receive-copy observation. +return { + SHC={ + localTimed={address=0x00489216,bytes={139,150,48,216,2,0}}, + remoteImmediate={address=0x00480417,bytes={15,190,135,132,198,3,0}}, + localImmediate={address=0x004892BE,bytes={15,190,148,49,132,198,3,0}}, + }, + Extreme={ + localTimed={address=0x00489326,bytes={139,150,48,216,2,0}}, + remoteImmediate={address=0x004805E7,bytes={15,190,135,132,198,3,0}}, + localImmediate={address=0x004893CE,bytes={15,190,148,49,132,198,3,0}}, + }, +} diff --git a/definition.yml b/definition.yml index aa2f25f..e45f513 100644 --- a/definition.yml +++ b/definition.yml @@ -1,6 +1,6 @@ name: recorder display-name: UCP-Recorder -version: 0.12.0 +version: 0.13.0 type: module author: Corax meta: diff --git a/docs/multiplayer-diagnostics.md b/docs/multiplayer-diagnostics.md index 6ddfb55..1417d7f 100644 --- a/docs/multiplayer-diagnostics.md +++ b/docs/multiplayer-diagnostics.md @@ -1,6 +1,6 @@ -# Multiplayer diagnostics (0.11.0) +# Multiplayer diagnostics (0.13.0) -This is the first multiplayer-specific implementation stage. It observes execution on each peer without changing the simulation scope, RNG seeds, pause flag, command ownership or dispatch decisions. Multiplayer recording/playback remains disabled. The existing single-player replay checks and isolation gates remain in effect. +This stage observes execution and identity on each peer without changing the simulation scope, RNG seeds, pause flag, command ownership or dispatch decisions. Multiplayer recording/playback remains disabled. The existing single-player replay checks and isolation gates remain in effect. ## Collect and compare @@ -13,13 +13,13 @@ Exit codes: 0 means the compared command/RNG/resource evidence matched; 1 means ## What is observed -The existing checked receive-copy and pre/post-dispatch sites observe timed commands from all senders. The actor comes from the native resolved-player field, rather than assigning every command to the local spectator. Every 64 simulation ticks, a read-only observer also records RNG/resource state immediately before native RNG advancement. It runs on the original simulation path without using the single-player halt flag. Repeated observations of a paused boundary are deduplicated. Each returned handler produces one row with the execution and scheduled ticks, category, payload, raw sender handle, native slot, resolved actor, post-command RNG and all eight players' resources. The comparator ignores peer-local player identity, raw handles and ring slots, comparing the resolved actions and resulting evidence instead. +The existing checked receive-copy and pre/post-dispatch sites observe timed commands from all senders. The actor comes from the native resolved-player field, rather than assigning every command to the local spectator. Every 64 simulation ticks, a read-only observer also records RNG/resource state immediately before native RNG advancement. It runs on the original simulation path without using the single-player halt flag. Repeated observations of a paused boundary are deduplicated. Each returned handler produces one row with the execution and scheduled ticks, category, payload, raw sender handle, native slot, resolved actor, post-command RNG and all eight players' resources. The comparator compares logical actors across peers, ignoring peer-local ring slots and transport-handle numbering differences. In format 4, each peer's handle-to-actor mapping must agree with its own captured roster. A missing payload receipt produces an untracked row and an incomplete completion status. Interrupted handlers, write errors, missing footers and malformed records cannot become a successful comparison. Logging errors close the trace and disable further diagnostic writes for that session; they do not halt multiplayer or replace its command category. Trace files are flushed synchronously, so this opt-in instrumentation has I/O and execution overhead that still needs live measurement. ## Boundaries -This is command-boundary and periodic simulation evidence, not a full replay or a full world-state checksum. It does not yet capture untimed commands, resynchronization/save-transfer packets or private extension state. It does not make a multiplayer save playable offline, replace networking, or change human/AI identity rules. Equal traces therefore do not prove a complete multiplayer match is deterministic. Different post-command RNG/resource evidence locates a mismatch; further analysis is needed to establish its cause. +This is command-boundary and periodic simulation evidence, not a full replay or a full world-state checksum. Immediate commands are detected but their full effects/payloads are not replayed; resynchronization/save transfers and private extension state remain unsupported. It does not make a multiplayer save playable offline, replace networking, or change human/AI identity rules. Equal traces therefore do not prove a complete multiplayer match is deterministic. Different post-command RNG/resource evidence locates a mismatch; further analysis is needed to establish its cause. Automated tests use native callback fixtures to verify original registers/dispatch survive logging failures, remote actors are recorded, single-player does not start a multiplayer trace, and incomplete evidence is rejected. The comparator has matching/differing/corrupt trace tests. No live multiplayer match was launched for this stage. @@ -34,3 +34,42 @@ are reported as incompatible. This does not fingerprint edited extension files or external assets. Older format-1/2 pairs can still be compared with their original evidence limits. Full RNG hashing adds periodic CPU work; live overhead and two-peer comparisons still need measurement. + +## Format 4: ownership and uncovered network events + +Locally queued timed payloads are captured before transmission at `0x489216` +(Extreme `0x489326`). They enter the command ring directly and never traverse +the receive-copy hook. Each command row identifies its local/received origin; +the comparator ignores that peer-relative origin and compares actual execution. + +The header records mode, local slot, synchronization status, eight transport +handles and a logical roster (human/AI/empty, AI ID and variation). Native +classification treats handle -1 plus a nonzero AI ID as AI. Duplicate human +handles, system-message handle zero, missing roster data and an active resync +phase are rejected by the comparator. Human handle numbering may differ between +the two files, but each command must map to its actor in that file's own roster. + +While a trace is open, identity and synchronization state are checked at every +simulation boundary and before timed dispatch. Changes produce `gap` rows and +an incomplete footer. Separate read-only detours cover immediate dispatch from +both received commands (`0x480417`, Extreme `0x4805E7`) and locally queued +commands (`0x4892BE`, Extreme `0x4893CE`). These hooks retain the original MOVSX +instructions and do not suppress execution. Any such dispatch in the trace +window produces a gap with its category, actor, declared size and source. +Harmless immediate commands can therefore also make this conservative stage +incomplete; there is no unaudited whitelist. + +Lobby events before the trace opens are outside its window. DirectPlay system +messages are not yet individually logged: persistent roster/sync changes can be +noticed at later boundaries, but a transient change between observations can +escape that sampling. Host migration, a synchronized starting snapshot, offline +roster restoration, resync payload reconstruction and native-versus-extension +state coverage remain implementation work. See the [native network-flow +contribution](https://github.com/sourcehold/OpenSHC/pull/220) for the evidence. + +The source investigation also confirms that the wire timestamp uses three bytes, +while command-ring ticks use four. A replay file must not silently equate these +formats. The reviewed RedirectPlay source forwards reliable flags but omits the +ReceiveData size output; its exact correspondence to the shipped DLL remains +unverified. Neither replacing transport nor recording raw packets alone closes +the replay ownership and resync gaps. diff --git a/init.lua b/init.lua index e37e64c..0bf761c 100644 --- a/init.lua +++ b/init.lua @@ -20,6 +20,7 @@ function module:enable(config) local recorder=Session:new(engine) self.recorder=recorder engine:install(recorder) + if engine.trace then require('code/network-observer').install(engine.trace) end local ui=require('code/ui') ui.createButtons(recorder,uiSites) diff --git a/tests/check_executables.py b/tests/check_executables.py index a571305..0343f97 100644 --- a/tests/check_executables.py +++ b/tests/check_executables.py @@ -43,7 +43,7 @@ def check(folder): menu = native.addr(0x59ab30) assert reader(menu, 1) == b'\x68' assert struct.unpack(' 65536: @@ -25,12 +54,14 @@ def rows(stream): yield value -def evidence(records, format_version, first_tick=0): +def evidence(records, format_version, first_tick=0, network=None): count, command_count = 0, 0 integer(first_tick, 0, 2147483647) previous_tick = first_tick next_checkpoint = (first_tick + 63) // 64 * 64 for record in records: + if record.get('kind') == 'gap': + raise ValueError(f'Uncovered network event at tick {record.get("time")}: {record.get("reason")}') if record.get('kind') == 'end': if (record.get('status') != 'complete' or record.get('commands') != command_count or not count or (format_version >= 2 and record.get('events') != count)): @@ -59,6 +90,10 @@ def evidence(records, format_version, first_tick=0): if not isinstance(data, str) or len(data) != record['size'] * 2 or any(c not in '0123456789abcdefABCDEF' for c in data): raise ValueError('Invalid trace payload') record['data'] = data.upper() + if network is not None: + player = record['player'] - 1 + if network['roster'][player]['kind'] != 'human' or record.get('handle') != network['handles'][player]: + raise ValueError('Command handle does not match its recorded logical player') elif record['time'] != next_checkpoint: raise ValueError('Missing, repeated or invalid checkpoint boundary') else: @@ -83,7 +118,7 @@ def compare(left, right): ar, br = rows(a), rows(b) ah, bh = next(ar), next(br) for header in (ah, bh): - if header.get('kind') != 'header' or header.get('format') not in (1, 2, 3): + if header.get('kind') != 'header' or header.get('format') not in (1, 2, 3, 4): raise ValueError('Unsupported trace format') if header['format'] >= 3: hash_value(header.get('environmentHash')) @@ -91,8 +126,12 @@ def compare(left, right): raise ValueError('Traces require the same game executable, variant and format') if ah['format'] >= 3 and ah['environmentHash'] != bh['environmentHash']: raise ValueError('Traces require the same resolved UCP settings, extension order/versions and framework') - for av, bv in itertools.zip_longest(evidence(ar, ah['format'], ah.get('firstTick', 0)), - evidence(br, bh['format'], bh.get('firstTick', 0))): + an = network_state(ah) if ah['format'] >= 4 else None + bn = network_state(bh) if bh['format'] >= 4 else None + if an and (an['roster'] != bn['roster'] or an['mode'] != bn['mode']): + raise ValueError('Traces require the same logical player roster and game mode') + for av, bv in itertools.zip_longest(evidence(ar, ah['format'], ah.get('firstTick', 0), an), + evidence(br, bh['format'], bh.get('firstTick', 0), bn)): count += 1 if difference: continue # Still validate both complete streams before reporting.